From 141d5361db0b90542bf3c0d8e9dd777833711253 Mon Sep 17 00:00:00 2001 From: "Oleksandr K." Date: Wed, 7 Aug 2024 18:38:08 +0200 Subject: [PATCH] feat: update handling vendor charts and add linters (#413) * feat: update handling vendor charts and add linters * remove makefile tasks * fix chart path * use chart-vendor * set charts path for chart-vendor depends-on: https://github.com/vexxhost/chart-vendor/pull/1 * eliminate make from gh actions * fix duplicated patch applies --------- Co-authored-by: okozachenko1203 --- .charts.yml | 9 + .github/workflows/publish.yml | 5 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 12 +- .gitignore | 4 +- Earthfile | 15 +- Makefile | 26 - hack/add-omt-to-clusterrole.patch | 17 - hack/stack.sh | 6 - magnum_cluster_api/charts/cilium/.helmignore | 24 + magnum_cluster_api/charts/cilium/Chart.yaml | 98 + magnum_cluster_api/charts/cilium/LICENSE | 202 + magnum_cluster_api/charts/cilium/README.md | 801 ++ .../charts/cilium/README.md.gotmpl | 54 + .../cilium/files/agent/poststart-eni.bash | 19 + .../dashboards/cilium-dashboard.json | 8479 +++++++++++++++++ .../configmap/bootstrap-config.json | 331 + .../dashboards/cilium-operator-dashboard.json | 998 ++ .../hubble/dashboards/hubble-dashboard.json | 3232 +++++++ .../dashboards/hubble-dns-namespace.json | 602 ++ .../hubble-l7-http-metrics-by-workload.json | 1385 +++ .../hubble-network-overview-namespace.json | 1001 ++ .../charts/cilium/files/nodeinit/prestop.bash | 60 + .../charts/cilium/files/nodeinit/startup.bash | 206 + .../charts/cilium/files/spire/init.bash | 48 + .../cilium/files/spire/wait-for-spire.bash | 35 + .../charts/cilium/templates/NOTES.txt | 22 + .../charts/cilium/templates/_helpers.tpl | 156 + .../templates/cilium-agent/clusterrole.yaml | 150 + .../cilium-agent/clusterrolebinding.yaml | 20 + .../templates/cilium-agent/daemonset.yaml | 981 ++ .../cilium-agent/dashboards-configmap.yaml | 30 + .../cilium/templates/cilium-agent/role.yaml | 116 + .../templates/cilium-agent/rolebinding.yaml | 110 + .../templates/cilium-agent/service.yaml | 56 + .../cilium-agent/serviceaccount.yaml | 16 + .../cilium-agent/servicemonitor.yaml | 61 + .../templates/cilium-ca-bundle-configmap.yaml | 11 + .../cilium/templates/cilium-ca-secret.yaml | 17 + .../cilium/templates/cilium-configmap.yaml | 1253 +++ .../templates/cilium-envoy/configmap.yaml | 16 + .../templates/cilium-envoy/daemonset.yaml | 266 + .../templates/cilium-envoy/service.yaml | 32 + .../cilium-envoy/serviceaccount.yaml | 16 + .../cilium-envoy/servicemonitor.yaml | 45 + .../templates/cilium-flowlog-configmap.yaml | 12 + .../templates/cilium-gateway-api-class.yaml | 10 + .../templates/cilium-ingress-class.yaml | 12 + .../templates/cilium-ingress-service.yaml | 57 + .../templates/cilium-nodeinit/daemonset.yaml | 114 + .../cilium-nodeinit/serviceaccount.yaml | 16 + .../templates/cilium-operator/_helpers.tpl | 36 + .../cilium-operator/clusterrole.yaml | 294 + .../cilium-operator/clusterrolebinding.yaml | 20 + .../cilium-operator/dashboards-configmap.yaml | 30 + .../templates/cilium-operator/deployment.yaml | 337 + .../cilium-operator/poddisruptionbudget.yaml | 28 + .../templates/cilium-operator/role.yaml | 49 + .../cilium-operator/rolebinding.yaml | 45 + .../templates/cilium-operator/secret.yaml | 17 + .../templates/cilium-operator/service.yaml | 27 + .../cilium-operator/serviceaccount.yaml | 20 + .../cilium-operator/servicemonitor.yaml | 48 + .../cilium-preflight/clusterrole.yaml | 150 + .../cilium-preflight/clusterrolebinding.yaml | 20 + .../templates/cilium-preflight/daemonset.yaml | 220 + .../cilium-preflight/deployment.yaml | 107 + .../cilium-preflight/poddisruptionbudget.yaml | 28 + .../cilium-preflight/serviceaccount.yaml | 16 + .../templates/cilium-resource-quota.yaml | 35 + .../templates/cilium-secrets-namespace.yaml | 14 + .../clustermesh-apiserver/_helpers.tpl | 11 + .../clustermesh-apiserver/clusterrole.yaml | 72 + .../clusterrolebinding.yaml | 20 + .../clustermesh-apiserver/deployment.yaml | 431 + .../metrics-service.yaml | 43 + .../poddisruptionbudget.yaml | 26 + .../clustermesh-apiserver/service.yaml | 38 + .../clustermesh-apiserver/serviceaccount.yaml | 16 + .../clustermesh-apiserver/servicemonitor.yaml | 77 + .../tls-certmanager/admin-secret.yaml | 20 + .../tls-certmanager/client-secret.yaml | 18 + .../tls-certmanager/remote-secret.yaml | 18 + .../tls-certmanager/server-secret.yaml | 31 + .../tls-cronjob/_job-spec.tpl | 73 + .../tls-cronjob/cronjob.yaml | 19 + .../tls-cronjob/job.yaml | 20 + .../tls-cronjob/role.yaml | 40 + .../tls-cronjob/rolebinding.yaml | 21 + .../tls-cronjob/serviceaccount.yaml | 16 + .../tls-helm/admin-secret.yaml | 21 + .../tls-helm/client-secret.yaml | 20 + .../tls-helm/remote-secret.yaml | 20 + .../tls-helm/server-secret.yaml | 22 + .../tls-provided/admin-secret.yaml | 16 + .../tls-provided/client-secret.yaml | 16 + .../tls-provided/remote-secret.yaml | 16 + .../tls-provided/server-secret.yaml | 16 + .../users-configmap.yaml | 25 + .../templates/clustermesh-config/_helpers.tpl | 24 + .../clustermesh-secret.yaml | 26 + .../kvstoremesh-secret.yaml | 24 + .../cilium-etcd-operator-clusterrole.yaml | 79 + ...lium-etcd-operator-clusterrolebinding.yaml | 20 + .../cilium-etcd-operator-deployment.yaml | 128 + .../cilium-etcd-operator-serviceaccount.yaml | 16 + .../etcd-operator-clusterrole.yaml | 60 + .../etcd-operator-clusterrolebinding.yaml | 20 + .../etcd-operator-serviceaccount.yaml | 16 + .../etcd-operator/poddisruptionbudget.yaml | 28 + .../templates/hubble-relay/configmap.yaml | 51 + .../templates/hubble-relay/deployment.yaml | 201 + .../hubble-relay/metrics-service.yaml | 24 + .../hubble-relay/poddisruptionbudget.yaml | 26 + .../templates/hubble-relay/service.yaml | 30 + .../hubble-relay/serviceaccount.yaml | 16 + .../hubble-relay/servicemonitor.yaml | 39 + .../cilium/templates/hubble-ui/_nginx.tpl | 61 + .../templates/hubble-ui/clusterrole.yaml | 50 + .../hubble-ui/clusterrolebinding.yaml | 20 + .../cilium/templates/hubble-ui/configmap.yaml | 14 + .../templates/hubble-ui/deployment.yaml | 216 + .../cilium/templates/hubble-ui/ingress.yaml | 40 + .../hubble-ui/poddisruptionbudget.yaml | 26 + .../cilium/templates/hubble-ui/service.yaml | 31 + .../templates/hubble-ui/serviceaccount.yaml | 16 + .../hubble/dashboards-configmap.yaml | 30 + .../templates/hubble/metrics-service.yaml | 32 + .../cilium/templates/hubble/peer-service.yaml | 30 + .../templates/hubble/servicemonitor.yaml | 44 + .../tls-certmanager/relay-client-secret.yaml | 22 + .../tls-certmanager/relay-server-secret.yaml | 31 + .../hubble/tls-certmanager/server-secret.yaml | 32 + .../tls-certmanager/ui-client-certs.yaml | 22 + .../hubble/tls-cronjob/_job-spec.tpl | 68 + .../hubble/tls-cronjob/clusterrole.yaml | 38 + .../tls-cronjob/clusterrolebinding.yaml | 20 + .../templates/hubble/tls-cronjob/cronjob.yaml | 25 + .../templates/hubble/tls-cronjob/job.yaml | 21 + .../hubble/tls-cronjob/serviceaccount.yaml | 16 + .../templates/hubble/tls-helm/_helpers.tpl | 31 + .../hubble/tls-helm/relay-client-secret.yaml | 21 + .../hubble/tls-helm/relay-server-secret.yaml | 22 + .../hubble/tls-helm/server-secret.yaml | 22 + .../hubble/tls-helm/ui-client-certs.yaml | 21 + .../tls-provided/relay-client-secret.yaml | 16 + .../tls-provided/relay-server-secret.yaml | 16 + .../hubble/tls-provided/server-secret.yaml | 16 + .../hubble/tls-provided/ui-client-certs.yaml | 16 + .../templates/spire/agent/clusterrole.yaml | 15 + .../spire/agent/clusterrolebinding.yaml | 19 + .../templates/spire/agent/configmap.yaml | 53 + .../templates/spire/agent/daemonset.yaml | 126 + .../templates/spire/agent/serviceaccount.yaml | 11 + .../templates/spire/bundle-configmap.yaml | 11 + .../cilium/templates/spire/namespace.yaml | 10 + .../templates/spire/server/clusterrole.yaml | 26 + .../spire/server/clusterrolebinding.yaml | 18 + .../templates/spire/server/configmap.yaml | 72 + .../cilium/templates/spire/server/role.yaml | 16 + .../templates/spire/server/rolebinding.yaml | 37 + .../templates/spire/server/service.yaml | 29 + .../spire/server/serviceaccount.yaml | 11 + .../templates/spire/server/statefulset.yaml | 129 + .../charts/cilium/templates/validate.yaml | 103 + magnum_cluster_api/charts/cilium/values.yaml | 3516 +++++++ .../charts/cilium/values.yaml.tmpl | 3513 +++++++ .../charts/cluster-autoscaler/.helmignore | 23 + .../charts/cluster-autoscaler/Chart.yaml | 13 + .../charts/cluster-autoscaler/README.md | 437 + .../cluster-autoscaler/README.md.gotmpl | 339 + .../cluster-autoscaler/templates/NOTES.txt | 18 + .../cluster-autoscaler/templates/_helpers.tpl | 129 + .../templates/clusterrole.yaml | 174 + .../templates/clusterrolebinding.yaml | 16 + .../templates/deployment.yaml | 315 + .../cluster-autoscaler/templates/pdb.yaml | 16 + .../templates/podsecuritypolicy.yaml | 42 + .../priority-expander-configmap.yaml | 25 + .../templates/prometheusrule.yaml | 15 + .../cluster-autoscaler/templates/role.yaml | 80 + .../templates/rolebinding.yaml | 17 + .../cluster-autoscaler/templates/secret.yaml | 21 + .../cluster-autoscaler/templates/service.yaml | 39 + .../templates/serviceaccount.yaml | 13 + .../templates/servicemonitor.yaml | 30 + .../cluster-autoscaler/templates/vpa.yaml | 20 + .../charts/cluster-autoscaler/values.yaml | 398 + .../001-add-omt-to-clusterrole.patch | 19 + magnum_cluster_api/resources.py | 4 +- tox.ini | 9 + vendir.lock.yml | 13 - vendir.yml | 18 - zuul.d/jobs.yaml | 3 + .../playbooks/linters/pre.yml | 19 +- 195 files changed, 35770 insertions(+), 113 deletions(-) create mode 100644 .charts.yml delete mode 100644 Makefile delete mode 100644 hack/add-omt-to-clusterrole.patch create mode 100644 magnum_cluster_api/charts/cilium/.helmignore create mode 100644 magnum_cluster_api/charts/cilium/Chart.yaml create mode 100644 magnum_cluster_api/charts/cilium/LICENSE create mode 100644 magnum_cluster_api/charts/cilium/README.md create mode 100644 magnum_cluster_api/charts/cilium/README.md.gotmpl create mode 100644 magnum_cluster_api/charts/cilium/files/agent/poststart-eni.bash create mode 100644 magnum_cluster_api/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json create mode 100644 magnum_cluster_api/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json create mode 100644 magnum_cluster_api/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json create mode 100644 magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dashboard.json create mode 100644 magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json create mode 100644 magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json create mode 100644 magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json create mode 100644 magnum_cluster_api/charts/cilium/files/nodeinit/prestop.bash create mode 100644 magnum_cluster_api/charts/cilium/files/nodeinit/startup.bash create mode 100644 magnum_cluster_api/charts/cilium/files/spire/init.bash create mode 100644 magnum_cluster_api/charts/cilium/files/spire/wait-for-spire.bash create mode 100644 magnum_cluster_api/charts/cilium/templates/NOTES.txt create mode 100644 magnum_cluster_api/charts/cilium/templates/_helpers.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/daemonset.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/role.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/rolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-agent/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-ca-bundle-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-ca-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-envoy/configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-envoy/daemonset.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-envoy/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-envoy/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-envoy/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-flowlog-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-gateway-api-class.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-ingress-class.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-ingress-service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/daemonset.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/_helpers.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/deployment.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/role.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/rolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-operator/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-preflight/daemonset.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-preflight/deployment.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-preflight/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-resource-quota.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/cilium-secrets-namespace.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/deployment.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-config/_helpers.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/deployment.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/metrics-service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-relay/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/_nginx.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/deployment.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/ingress.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble-ui/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/dashboards-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/metrics-service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/peer-service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/job.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/_helpers.tpl create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-client-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/ui-client-certs.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/server-secret.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/agent/configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/agent/daemonset.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/agent/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/bundle-configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/namespace.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/configmap.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/role.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/rolebinding.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/service.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/spire/server/statefulset.yaml create mode 100644 magnum_cluster_api/charts/cilium/templates/validate.yaml create mode 100644 magnum_cluster_api/charts/cilium/values.yaml create mode 100644 magnum_cluster_api/charts/cilium/values.yaml.tmpl create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/.helmignore create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/Chart.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/README.md create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/README.md.gotmpl create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/NOTES.txt create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/_helpers.tpl create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrole.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrolebinding.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/deployment.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/pdb.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/prometheusrule.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/role.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/rolebinding.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/secret.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/service.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/serviceaccount.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/servicemonitor.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/templates/vpa.yaml create mode 100644 magnum_cluster_api/charts/cluster-autoscaler/values.yaml create mode 100644 magnum_cluster_api/charts/patches/cluster-autoscaler/001-add-omt-to-clusterrole.patch delete mode 100644 vendir.lock.yml delete mode 100644 vendir.yml rename hack/setup-vendir.sh => zuul.d/playbooks/linters/pre.yml (67%) mode change 100755 => 100644 diff --git a/.charts.yml b/.charts.yml new file mode 100644 index 00000000..411be100 --- /dev/null +++ b/.charts.yml @@ -0,0 +1,9 @@ +charts: + - name: cluster-autoscaler + version: 9.29.1 + repository: + url: https://kubernetes.github.io/autoscaler + - name: cilium + version: 1.15.3 + repository: + url: https://helm.cilium.io/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4541ad8b..63eac9ae 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,8 +17,11 @@ jobs: with: cache: poetry + - name: Setup poetry + uses: abatilo/actions-poetry@v2 + - name: Build package - run: make build + run: poetry build - name: Publish a Python distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31d0c00d..00b1ad1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,7 @@ jobs: - name: Build package if: ${{ steps.release.outputs.release_created }} - run: make build + run: poetry build - name: Publish package distributions to PyPI if: ${{ steps.release.outputs.release_created }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dad3d050..aff20c29 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,16 +30,15 @@ jobs: uses: actions/checkout@v3 - name: Install Poetry - run: make poetry + run: pipx install poetry - name: Setup Python uses: actions/setup-python@v4 with: cache: poetry - python-version: ${{ matrix.python-version }} - name: Build package - run: make build + run: poetry build install: runs-on: ${{ matrix.runs-on }} @@ -60,8 +59,11 @@ jobs: - name: Checkout project uses: actions/checkout@v3 + - name: Checkout project + uses: actions/checkout@v3 + - name: Install Poetry - run: make poetry + run: pipx install poetry - name: Setup Python uses: actions/setup-python@v4 @@ -70,7 +72,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Build package - run: make build + run: poetry build - name: Install package using constraints run: pip install --constraint https://releases.openstack.org/constraints/upper/${{ matrix.openstack-version }} ./dist/*.whl diff --git a/.gitignore b/.gitignore index 784c8f82..2bd4a625 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .direnv +.idea __pycache__ dist -magnum_cluster_api/charts/vendor/* site +*.orig +*.rej diff --git a/Earthfile b/Earthfile index 3e0199c6..85357680 100644 --- a/Earthfile +++ b/Earthfile @@ -1,23 +1,10 @@ VERSION 0.7 -vendir: - FROM github.com/vexxhost/atmosphere/images/curl+image - ARG TARGETOS - ARG TARGETARCH - ARG VERSION=v0.40.0 - RUN curl -Lo vendir https://github.com/carvel-dev/vendir/releases/download/${VERSION}/vendir-${TARGETOS}-${TARGETARCH} - RUN chmod +x vendir && ./vendir version - SAVE ARTIFACT vendir - build: FROM github.com/vexxhost/atmosphere/images/magnum+build - COPY +vendir/vendir /usr/local/bin/vendir COPY github.com/vexxhost/atmosphere/images/helm+binary/helm /usr/local/bin/helm - COPY --dir magnum_cluster_api/ pyproject.toml README.md vendir.yml /src + COPY --dir magnum_cluster_api/ pyproject.toml README.md /src WORKDIR /src - RUN vendir sync - COPY hack/add-omt-to-clusterrole.patch /hack/ - RUN patch -p0 magnum_cluster_api/charts/vendor/cluster-autoscaler/templates/clusterrole.yaml < /hack/add-omt-to-clusterrole.patch DO github.com/vexxhost/atmosphere/images/openstack-service+PIP_INSTALL --PACKAGES /src SAVE ARTIFACT /var/lib/openstack venv diff --git a/Makefile b/Makefile deleted file mode 100644 index a25fcc84..00000000 --- a/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -clean: - rm -rfv magnum_cluster_api/charts/vendor - -vendir: - curl -Lo vendir https://github.com/carvel-dev/vendir/releases/download/v0.40.0/vendir-linux-amd64 - chmod +x vendir && ./vendir version - sudo mv vendir /usr/local/bin/vendir - -vendor: clean vendir - vendir sync - patch -p0 magnum_cluster_api/charts/vendor/cluster-autoscaler/templates/clusterrole.yaml < hack/add-omt-to-clusterrole.patch - -poetry: - pipx install poetry - -build: vendor poetry - poetry build - -install: build poetry - poetry install - -unit-tests: install poetry - poetry run pytest magnum_cluster_api/tests/unit/ - -functional-tests: install poetry - poetry run pytest magnum_cluster_api/tests/functional/ diff --git a/hack/add-omt-to-clusterrole.patch b/hack/add-omt-to-clusterrole.patch deleted file mode 100644 index f24edcd3..00000000 --- a/hack/add-omt-to-clusterrole.patch +++ /dev/null @@ -1,17 +0,0 @@ ---- clusterrole.yaml.orig 2023-04-04 03:30:47.711856778 +0000 -+++ clusterrole.yaml 2023-04-04 03:31:04.607355748 +0000 -@@ -161,5 +161,14 @@ rules: - - list - - update - - watch -+ - apiGroups: -+ - infrastructure.cluster.x-k8s.io -+ resources: -+ - openstackmachinetemplates -+ verbs: -+ - get -+ - list -+ - update -+ - watch - {{- end }} - {{- end -}} diff --git a/hack/stack.sh b/hack/stack.sh index 438c9a1b..ebc84c55 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -107,12 +107,6 @@ EOF # Install CAPI/CAPO ./hack/setup-capo.sh -# Install vendir -./hack/setup-vendir.sh - -# Vendor the chart -make vendor - # Install `magnum-cluster-api` pip install -U setuptools pip python-magnumclient $HOME/.local/bin/pip3 install -e . diff --git a/magnum_cluster_api/charts/cilium/.helmignore b/magnum_cluster_api/charts/cilium/.helmignore new file mode 100644 index 00000000..f9dcfa99 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.github/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/magnum_cluster_api/charts/cilium/Chart.yaml b/magnum_cluster_api/charts/cilium/Chart.yaml new file mode 100644 index 00000000..43447f2c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/Chart.yaml @@ -0,0 +1,98 @@ +annotations: + artifacthub.io/crds: "- kind: CiliumNetworkPolicy\n version: v2\n name: ciliumnetworkpolicies.cilium.io\n + \ displayName: Cilium Network Policy\n description: |\n Cilium Network Policies + provide additional functionality beyond what\n is provided by standard Kubernetes + NetworkPolicy such as the ability\n to allow traffic based on FQDNs, or to + filter at Layer 7.\n- kind: CiliumClusterwideNetworkPolicy\n version: v2\n name: + ciliumclusterwidenetworkpolicies.cilium.io\n displayName: Cilium Clusterwide + Network Policy\n description: |\n Cilium Clusterwide Network Policies support + configuring network traffic\n policiies across the entire cluster, including + applying node firewalls.\n- kind: CiliumExternalWorkload\n version: v2\n name: + ciliumexternalworkloads.cilium.io\n displayName: Cilium External Workload\n description: + |\n Cilium External Workload supports configuring the ability for external\n + \ non-Kubernetes workloads to join the cluster.\n- kind: CiliumLocalRedirectPolicy\n + \ version: v2\n name: ciliumlocalredirectpolicies.cilium.io\n displayName: Cilium + Local Redirect Policy\n description: |\n Cilium Local Redirect Policy allows + local redirects to be configured\n within a node to support use cases like + Node-Local DNS or KIAM.\n- kind: CiliumNode\n version: v2\n name: ciliumnodes.cilium.io\n + \ displayName: Cilium Node\n description: |\n Cilium Node represents a node + managed by Cilium. It contains a\n specification to control various node specific + configuration aspects\n and a status section to represent the status of the + node.\n- kind: CiliumIdentity\n version: v2\n name: ciliumidentities.cilium.io\n + \ displayName: Cilium Identity\n description: |\n Cilium Identity allows introspection + into security identities that\n Cilium allocates which identify sets of labels + that are assigned to\n individual endpoints in the cluster.\n- kind: CiliumEndpoint\n + \ version: v2\n name: ciliumendpoints.cilium.io\n displayName: Cilium Endpoint\n + \ description: |\n Cilium Endpoint represents the status of individual pods + or nodes in\n the cluster which are managed by Cilium, including enforcement + status,\n IP addressing and whether the networking is successfully operational.\n- + kind: CiliumEndpointSlice\n version: v2alpha1\n name: ciliumendpointslices.cilium.io\n + \ displayName: Cilium Endpoint Slice\n description: |\n Cilium Endpoint Slice + represents the status of groups of pods or nodes\n in the cluster which are + managed by Cilium, including enforcement status,\n IP addressing and whether + the networking is successfully operational.\n- kind: CiliumEgressGatewayPolicy\n + \ version: v2\n name: ciliumegressgatewaypolicies.cilium.io\n displayName: Cilium + Egress Gateway Policy\n description: |\n Cilium Egress Gateway Policy provides + control over the way that traffic\n leaves the cluster and which source addresses + to use for that traffic.\n- kind: CiliumClusterwideEnvoyConfig\n version: v2\n + \ name: ciliumclusterwideenvoyconfigs.cilium.io\n displayName: Cilium Clusterwide + Envoy Config\n description: |\n Cilium Clusterwide Envoy Config specifies + Envoy resources and K8s service mappings\n to be provisioned into Cilium host + proxy instances in cluster context.\n- kind: CiliumEnvoyConfig\n version: v2\n + \ name: ciliumenvoyconfigs.cilium.io\n displayName: Cilium Envoy Config\n description: + |\n Cilium Envoy Config specifies Envoy resources and K8s service mappings\n + \ to be provisioned into Cilium host proxy instances in namespace context.\n- + kind: CiliumBGPPeeringPolicy\n version: v2alpha1\n name: ciliumbgppeeringpolicies.cilium.io\n + \ displayName: Cilium BGP Peering Policy\n description: |\n Cilium BGP Peering + Policy instructs Cilium to create specific BGP peering\n configurations.\n- + kind: CiliumBGPClusterConfig\n version: v2alpha1\n name: ciliumbgpclusterconfigs.cilium.io\n + \ displayName: Cilium BGP Cluster Config\n description: |\n Cilium BGP Cluster + Config instructs Cilium operator to create specific BGP cluster\n configurations.\n- + kind: CiliumBGPPeerConfig\n version: v2alpha1\n name: ciliumbgppeerconfigs.cilium.io\n + \ displayName: Cilium BGP Peer Config\n description: |\n CiliumBGPPeerConfig + is a common set of BGP peer configurations. It can be referenced \n by multiple + peers from CiliumBGPClusterConfig.\n- kind: CiliumBGPAdvertisement\n version: + v2alpha1\n name: ciliumbgpadvertisements.cilium.io\n displayName: Cilium BGP + Advertisement\n description: |\n CiliumBGPAdvertisement is used to define + source of BGP advertisement as well as BGP attributes \n to be advertised with + those prefixes.\n- kind: CiliumBGPNodeConfig\n version: v2alpha1\n name: ciliumbgpnodeconfigs.cilium.io\n + \ displayName: Cilium BGP Node Config\n description: |\n CiliumBGPNodeConfig + is read only node specific BGP configuration. It is constructed by Cilium operator.\n + \ It will also contain node local BGP state information.\n- kind: CiliumBGPNodeConfigOverride\n + \ version: v2alpha1\n name: ciliumbgpnodeconfigoverrides.cilium.io\n displayName: + Cilium BGP Node Config Override\n description: |\n CiliumBGPNodeConfigOverride + can be used to override node specific BGP configuration.\n- kind: CiliumLoadBalancerIPPool\n + \ version: v2alpha1\n name: ciliumloadbalancerippools.cilium.io\n displayName: + Cilium Load Balancer IP Pool\n description: |\n Defining a Cilium Load Balancer + IP Pool instructs Cilium to assign IPs to LoadBalancer Services.\n- kind: CiliumNodeConfig\n + \ version: v2alpha1\n name: ciliumnodeconfigs.cilium.io\n displayName: Cilium + Node Configuration\n description: |\n CiliumNodeConfig is a list of configuration + key-value pairs. It is applied to\n nodes indicated by a label selector.\n- + kind: CiliumCIDRGroup\n version: v2alpha1\n name: ciliumcidrgroups.cilium.io\n + \ displayName: Cilium CIDR Group\n description: |\n CiliumCIDRGroup is a list + of CIDRs that can be referenced as a single entity from CiliumNetworkPolicies.\n- + kind: CiliumL2AnnouncementPolicy\n version: v2alpha1\n name: ciliuml2announcementpolicies.cilium.io\n + \ displayName: Cilium L2 Announcement Policy\n description: |\n CiliumL2AnnouncementPolicy + is a policy which determines which service IPs will be announced to\n the local + area network, by which nodes, and via which interfaces.\n- kind: CiliumPodIPPool\n + \ version: v2alpha1\n name: ciliumpodippools.cilium.io\n displayName: Cilium + Pod IP Pool\n description: |\n CiliumPodIPPool defines an IP pool that can + be used for pooled IPAM (i.e. the multi-pool IPAM mode).\n" +apiVersion: v2 +appVersion: 1.15.3 +description: eBPF-based Networking, Security, and Observability +home: https://cilium.io/ +icon: https://cdn.jsdelivr.net/gh/cilium/cilium@v1.15/Documentation/images/logo-solo.svg +keywords: +- BPF +- eBPF +- Kubernetes +- Networking +- Security +- Observability +- Troubleshooting +kubeVersion: '>= 1.16.0-0' +name: cilium +sources: +- https://github.com/cilium/cilium +version: 1.15.3 diff --git a/magnum_cluster_api/charts/cilium/LICENSE b/magnum_cluster_api/charts/cilium/LICENSE new file mode 100644 index 00000000..a2e486a8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} Authors of Cilium + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/magnum_cluster_api/charts/cilium/README.md b/magnum_cluster_api/charts/cilium/README.md new file mode 100644 index 00000000..1e927580 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/README.md @@ -0,0 +1,801 @@ +# cilium + +![Version: 1.15.3](https://img.shields.io/badge/Version-1.15.3-informational?style=flat-square) ![AppVersion: 1.15.3](https://img.shields.io/badge/AppVersion-1.15.3-informational?style=flat-square) + +Cilium is open source software for providing and transparently securing +network connectivity and loadbalancing between application workloads such as +application containers or processes. Cilium operates at Layer 3/4 to provide +traditional networking and security services as well as Layer 7 to protect and +secure use of modern application protocols such as HTTP, gRPC and Kafka. + +A new Linux kernel technology called eBPF is at the foundation of Cilium. +It supports dynamic insertion of eBPF bytecode into the Linux kernel at various +integration points such as: network IO, application sockets, and tracepoints +to implement security, networking and visibility logic. eBPF is highly +efficient and flexible. + +![Cilium feature overview](https://raw.githubusercontent.com/cilium/cilium/main/Documentation/images/cilium_overview.png) + +## Prerequisites + +* Kubernetes: `>= 1.16.0-0` +* Helm: `>= 3.0` + +## Getting Started + +Try Cilium on any Kubernetes distribution in under 15 minutes: + +| Minikube | Self-Managed K8s | Amazon EKS | Google GKE | Microsoft AKS | +|:-:|:-:|:-:|:-:|:-:| +| [![Minikube](https://raw.githubusercontent.com/cilium/charts/master/images/minikube.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Self-Managed Kubernetes](https://raw.githubusercontent.com/cilium/charts/master/images/k8s.png)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Amazon EKS](https://raw.githubusercontent.com/cilium/charts/master/images/aws.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Google GKE](https://raw.githubusercontent.com/cilium/charts/master/images/google-cloud.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Microsoft AKS](https://raw.githubusercontent.com/cilium/charts/master/images/azure.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | + +Or, for a quick install with the default configuration: + +``` +$ helm repo add cilium https://helm.cilium.io/ +$ helm install cilium cilium/cilium --namespace=kube-system +``` + +After Cilium is installed, you can explore the features that Cilium has to +offer from the [Getting Started Guides page](https://docs.cilium.io/en/stable/gettingstarted/). + +## Source Code + +* + +## Getting Help + +The best way to get help if you get stuck is to ask a question on the +[Cilium Slack channel](https://cilium.herokuapp.com/). With Cilium +contributors across the globe, there is almost always someone available to help. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| MTU | int | `0` | Configure the underlying network MTU to overwrite auto-detected MTU. | +| affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-agent. | +| agent | bool | `true` | Install the cilium agent resources. | +| agentNotReadyTaintKey | string | `"node.cilium.io/agent-not-ready"` | Configure the key of the taint indicating that Cilium is not ready on the node. When set to a value starting with `ignore-taint.cluster-autoscaler.kubernetes.io/`, the Cluster Autoscaler will ignore the taint on its decisions, allowing the cluster to scale up. | +| aksbyocni.enabled | bool | `false` | Enable AKS BYOCNI integration. Note that this is incompatible with AKS clusters not created in BYOCNI mode: use Azure integration (`azure.enabled`) instead. | +| alibabacloud.enabled | bool | `false` | Enable AlibabaCloud ENI integration | +| annotateK8sNode | bool | `false` | Annotate k8s node upon initialization with Cilium's metadata. | +| annotations | object | `{}` | Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) | +| apiRateLimit | string | `nil` | The api-rate-limit option can be used to overwrite individual settings of the default configuration for rate limiting calls to the Cilium Agent API | +| authentication.enabled | bool | `true` | Enable authentication processing and garbage collection. Note that if disabled, policy enforcement will still block requests that require authentication. But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. | +| authentication.gcInterval | string | `"5m0s"` | Interval for garbage collection of auth map entries. | +| authentication.mutual.connectTimeout | string | `"5s"` | Timeout for connecting to the remote node TCP socket | +| authentication.mutual.port | int | `4250` | Port on the agent where mutual authentication handshakes between agents will be performed | +| authentication.mutual.spire.adminSocketPath | string | `"/run/spire/sockets/admin.sock"` | SPIRE socket path where the SPIRE delegated api agent is listening | +| authentication.mutual.spire.agentSocketPath | string | `"/run/spire/sockets/agent/agent.sock"` | SPIRE socket path where the SPIRE workload agent is listening. Applies to both the Cilium Agent and Operator | +| authentication.mutual.spire.annotations | object | `{}` | Annotations to be added to all top-level spire objects (resources under templates/spire) | +| authentication.mutual.spire.connectionTimeout | string | `"30s"` | SPIRE connection timeout | +| authentication.mutual.spire.enabled | bool | `false` | Enable SPIRE integration (beta) | +| authentication.mutual.spire.install.agent.affinity | object | `{}` | SPIRE agent affinity configuration | +| authentication.mutual.spire.install.agent.annotations | object | `{}` | SPIRE agent annotations | +| authentication.mutual.spire.install.agent.image | object | `{"digest":"sha256:99405637647968245ff9fe215f8bd2bd0ea9807be9725f8bf19fe1b21471e52b","override":null,"pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-agent","tag":"1.8.5","useDigest":true}` | SPIRE agent image | +| authentication.mutual.spire.install.agent.labels | object | `{}` | SPIRE agent labels | +| authentication.mutual.spire.install.agent.nodeSelector | object | `{}` | SPIRE agent nodeSelector configuration ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| authentication.mutual.spire.install.agent.podSecurityContext | object | `{}` | Security context to be added to spire agent pods. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| authentication.mutual.spire.install.agent.securityContext | object | `{}` | Security context to be added to spire agent containers. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container | +| authentication.mutual.spire.install.agent.serviceAccount | object | `{"create":true,"name":"spire-agent"}` | SPIRE agent service account | +| authentication.mutual.spire.install.agent.skipKubeletVerification | bool | `true` | SPIRE Workload Attestor kubelet verification. | +| authentication.mutual.spire.install.agent.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/not-ready"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"},{"key":"CriticalAddonsOnly","operator":"Exists"}]` | SPIRE agent tolerations configuration By default it follows the same tolerations as the agent itself to allow the Cilium agent on this node to connect to SPIRE. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| authentication.mutual.spire.install.enabled | bool | `true` | Enable SPIRE installation. This will only take effect only if authentication.mutual.spire.enabled is true | +| authentication.mutual.spire.install.existingNamespace | bool | `false` | SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. | +| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.36.1","useDigest":true}` | init container image of SPIRE agent and server | +| authentication.mutual.spire.install.namespace | string | `"cilium-spire"` | SPIRE namespace to install into | +| authentication.mutual.spire.install.server.affinity | object | `{}` | SPIRE server affinity configuration | +| authentication.mutual.spire.install.server.annotations | object | `{}` | SPIRE server annotations | +| authentication.mutual.spire.install.server.ca.keyType | string | `"rsa-4096"` | SPIRE CA key type AWS requires the use of RSA. EC cryptography is not supported | +| authentication.mutual.spire.install.server.ca.subject | object | `{"commonName":"Cilium SPIRE CA","country":"US","organization":"SPIRE"}` | SPIRE CA Subject | +| authentication.mutual.spire.install.server.dataStorage.accessMode | string | `"ReadWriteOnce"` | Access mode of the SPIRE server data storage | +| authentication.mutual.spire.install.server.dataStorage.enabled | bool | `true` | Enable SPIRE server data storage | +| authentication.mutual.spire.install.server.dataStorage.size | string | `"1Gi"` | Size of the SPIRE server data storage | +| authentication.mutual.spire.install.server.dataStorage.storageClass | string | `nil` | StorageClass of the SPIRE server data storage | +| authentication.mutual.spire.install.server.image | object | `{"digest":"sha256:28269265882048dcf0fed32fe47663cd98613727210b8d1a55618826f9bf5428","override":null,"pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-server","tag":"1.8.5","useDigest":true}` | SPIRE server image | +| authentication.mutual.spire.install.server.initContainers | list | `[]` | SPIRE server init containers | +| authentication.mutual.spire.install.server.labels | object | `{}` | SPIRE server labels | +| authentication.mutual.spire.install.server.nodeSelector | object | `{}` | SPIRE server nodeSelector configuration ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| authentication.mutual.spire.install.server.podSecurityContext | object | `{}` | Security context to be added to spire server pods. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| authentication.mutual.spire.install.server.securityContext | object | `{}` | Security context to be added to spire server containers. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container | +| authentication.mutual.spire.install.server.service.annotations | object | `{}` | Annotations to be added to the SPIRE server service | +| authentication.mutual.spire.install.server.service.labels | object | `{}` | Labels to be added to the SPIRE server service | +| authentication.mutual.spire.install.server.service.type | string | `"ClusterIP"` | Service type for the SPIRE server service | +| authentication.mutual.spire.install.server.serviceAccount | object | `{"create":true,"name":"spire-server"}` | SPIRE server service account | +| authentication.mutual.spire.install.server.tolerations | list | `[]` | SPIRE server tolerations configuration ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| authentication.mutual.spire.serverAddress | string | `nil` | SPIRE server address used by Cilium Operator If k8s Service DNS along with port number is used (e.g. ..svc(.*): format), Cilium Operator will resolve its address by looking up the clusterIP from Service resource. Example values: 10.0.0.1:8081, spire-server.cilium-spire.svc:8081 | +| authentication.mutual.spire.trustDomain | string | `"spiffe.cilium"` | SPIFFE trust domain to use for fetching certificates | +| authentication.queueSize | int | `1024` | Buffer size of the channel Cilium uses to receive authentication events from the signal map. | +| authentication.rotatedIdentitiesQueueSize | int | `1024` | Buffer size of the channel Cilium uses to receive certificate expiration events from auth handlers. | +| autoDirectNodeRoutes | bool | `false` | Enable installation of PodCIDR routes between worker nodes if worker nodes share a common L2 network segment. | +| azure.enabled | bool | `false` | Enable Azure integration. Note that this is incompatible with AKS clusters created in BYOCNI mode: use AKS BYOCNI integration (`aksbyocni.enabled`) instead. | +| bandwidthManager | object | `{"bbr":false,"enabled":false}` | Enable bandwidth manager to optimize TCP and UDP workloads and allow for rate-limiting traffic from individual Pods with EDT (Earliest Departure Time) through the "kubernetes.io/egress-bandwidth" Pod annotation. | +| bandwidthManager.bbr | bool | `false` | Activate BBR TCP congestion control for Pods | +| bandwidthManager.enabled | bool | `false` | Enable bandwidth manager infrastructure (also prerequirement for BBR) | +| bgp | object | `{"announce":{"loadbalancerIP":false,"podCIDR":false},"enabled":false}` | Configure BGP | +| bgp.announce.loadbalancerIP | bool | `false` | Enable allocation and announcement of service LoadBalancer IPs | +| bgp.announce.podCIDR | bool | `false` | Enable announcement of node pod CIDR | +| bgp.enabled | bool | `false` | Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside cilium-agent and cilium-operator | +| bgpControlPlane | object | `{"enabled":false,"secretsNamespace":{"create":false,"name":"kube-system"}}` | This feature set enables virtual BGP routers to be created via CiliumBGPPeeringPolicy CRDs. | +| bgpControlPlane.enabled | bool | `false` | Enables the BGP control plane. | +| bgpControlPlane.secretsNamespace | object | `{"create":false,"name":"kube-system"}` | SecretsNamespace is the namespace which BGP support will retrieve secrets from. | +| bgpControlPlane.secretsNamespace.create | bool | `false` | Create secrets namespace for BGP secrets. | +| bgpControlPlane.secretsNamespace.name | string | `"kube-system"` | The name of the secret namespace to which Cilium agents are given read access | +| bpf.authMapMax | int | `524288` | Configure the maximum number of entries in auth map. | +| bpf.autoMount.enabled | bool | `true` | Enable automatic mount of BPF filesystem When `autoMount` is enabled, the BPF filesystem is mounted at `bpf.root` path on the underlying host and inside the cilium agent pod. If users disable `autoMount`, it's expected that users have mounted bpffs filesystem at the specified `bpf.root` volume, and then the volume will be mounted inside the cilium agent pod at the same path. | +| bpf.ctAnyMax | int | `262144` | Configure the maximum number of entries for the non-TCP connection tracking table. | +| bpf.ctTcpMax | int | `524288` | Configure the maximum number of entries in the TCP connection tracking table. | +| bpf.hostLegacyRouting | bool | `false` | Configure whether direct routing mode should route traffic via host stack (true) or directly and more efficiently out of BPF (false) if the kernel supports it. The latter has the implication that it will also bypass netfilter in the host namespace. | +| bpf.lbExternalClusterIP | bool | `false` | Allow cluster external access to ClusterIP services. | +| bpf.lbMapMax | int | `65536` | Configure the maximum number of service entries in the load balancer maps. | +| bpf.mapDynamicSizeRatio | float64 | `0.0025` | Configure auto-sizing for all BPF maps based on available memory. ref: https://docs.cilium.io/en/stable/network/ebpf/maps/ | +| bpf.masquerade | bool | `false` | Enable native IP masquerade support in eBPF | +| bpf.monitorAggregation | string | `"medium"` | Configure the level of aggregation for monitor notifications. Valid options are none, low, medium, maximum. | +| bpf.monitorFlags | string | `"all"` | Configure which TCP flags trigger notifications when seen for the first time in a connection. | +| bpf.monitorInterval | string | `"5s"` | Configure the typical time between monitor notifications for active connections. | +| bpf.natMax | int | `524288` | Configure the maximum number of entries for the NAT table. | +| bpf.neighMax | int | `524288` | Configure the maximum number of entries for the neighbor table. | +| bpf.policyMapMax | int | `16384` | Configure the maximum number of entries in endpoint policy map (per endpoint). | +| bpf.preallocateMaps | bool | `false` | Enables pre-allocation of eBPF map values. This increases memory usage but can reduce latency. | +| bpf.root | string | `"/sys/fs/bpf"` | Configure the mount point for the BPF filesystem | +| bpf.tproxy | bool | `false` | Configure the eBPF-based TPROXY to reduce reliance on iptables rules for implementing Layer 7 policy. | +| bpf.vlanBypass | list | `[]` | Configure explicitly allowed VLAN id's for bpf logic bypass. [0] will allow all VLAN id's without any filtering. | +| bpfClockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | +| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"extraVolumeMounts":[],"extraVolumes":[],"image":{"digest":"sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.1.9","useDigest":true},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen.affinity | object | `{}` | Affinity for certgen | +| certgen.annotations | object | `{"cronJob":{},"job":{}}` | Annotations to be added to the hubble-certgen initial Job and CronJob | +| certgen.extraVolumeMounts | list | `[]` | Additional certgen volumeMounts. | +| certgen.extraVolumes | list | `[]` | Additional certgen volumes. | +| certgen.podLabels | object | `{}` | Labels to be added to hubble-certgen pods | +| certgen.tolerations | list | `[]` | Node tolerations for pod assignment on nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| certgen.ttlSecondsAfterFinished | int | `1800` | Seconds after which the completed job pod will be deleted | +| cgroup | object | `{"autoMount":{"enabled":true,"resources":{}},"hostRoot":"/run/cilium/cgroupv2"}` | Configure cgroup related configuration | +| cgroup.autoMount.enabled | bool | `true` | Enable auto mount of cgroup2 filesystem. When `autoMount` is enabled, cgroup2 filesystem is mounted at `cgroup.hostRoot` path on the underlying host and inside the cilium agent pod. If users disable `autoMount`, it's expected that users have mounted cgroup2 filesystem at the specified `cgroup.hostRoot` volume, and then the volume will be mounted inside the cilium agent pod at the same path. | +| cgroup.autoMount.resources | object | `{}` | Init Container Cgroup Automount resource limits & requests | +| cgroup.hostRoot | string | `"/run/cilium/cgroupv2"` | Configure cgroup root where cgroup2 filesystem is mounted on the host (see also: `cgroup.autoMount`) | +| cleanBpfState | bool | `false` | Clean all eBPF datapath state from the initContainer of the cilium-agent DaemonSet. WARNING: Use with care! | +| cleanState | bool | `false` | Clean all local Cilium state from the initContainer of the cilium-agent DaemonSet. Implies cleanBpfState: true. WARNING: Use with care! | +| cluster.id | int | `0` | Unique ID of the cluster. Must be unique across all connected clusters and in the range of 1 to 255. Only required for Cluster Mesh, may be 0 if Cluster Mesh is not used. | +| cluster.name | string | `"default"` | Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. | +| clustermesh.annotations | object | `{}` | Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) | +| clustermesh.apiserver.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"clustermesh-apiserver"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for clustermesh.apiserver | +| clustermesh.apiserver.etcd.init.extraArgs | list | `[]` | Additional arguments to `clustermesh-apiserver etcdinit`. | +| clustermesh.apiserver.etcd.init.extraEnv | list | `[]` | Additional environment variables to `clustermesh-apiserver etcdinit`. | +| clustermesh.apiserver.etcd.init.resources | object | `{}` | Specifies the resources for etcd init container in the apiserver | +| clustermesh.apiserver.etcd.lifecycle | object | `{}` | lifecycle setting for the etcd container | +| clustermesh.apiserver.etcd.resources | object | `{}` | Specifies the resources for etcd container in the apiserver | +| clustermesh.apiserver.etcd.securityContext | object | `{}` | Security context to be added to clustermesh-apiserver etcd containers | +| clustermesh.apiserver.extraArgs | list | `[]` | Additional clustermesh-apiserver arguments. | +| clustermesh.apiserver.extraEnv | list | `[]` | Additional clustermesh-apiserver environment variables. | +| clustermesh.apiserver.extraVolumeMounts | list | `[]` | Additional clustermesh-apiserver volumeMounts. | +| clustermesh.apiserver.extraVolumes | list | `[]` | Additional clustermesh-apiserver volumes. | +| clustermesh.apiserver.image | object | `{"digest":"sha256:da4573f8fe4415bdb786c4fdcbc3b518e5a485f930cd4292416eb80800cbd7fc","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.15.3","useDigest":true}` | Clustermesh API server image. | +| clustermesh.apiserver.kvstoremesh.enabled | bool | `false` | Enable KVStoreMesh. KVStoreMesh caches the information retrieved from the remote clusters in the local etcd instance. | +| clustermesh.apiserver.kvstoremesh.extraArgs | list | `[]` | Additional KVStoreMesh arguments. | +| clustermesh.apiserver.kvstoremesh.extraEnv | list | `[]` | Additional KVStoreMesh environment variables. | +| clustermesh.apiserver.kvstoremesh.extraVolumeMounts | list | `[]` | Additional KVStoreMesh volumeMounts. | +| clustermesh.apiserver.kvstoremesh.lifecycle | object | `{}` | lifecycle setting for the KVStoreMesh container | +| clustermesh.apiserver.kvstoremesh.resources | object | `{}` | Resource requests and limits for the KVStoreMesh container | +| clustermesh.apiserver.kvstoremesh.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | KVStoreMesh Security context | +| clustermesh.apiserver.lifecycle | object | `{}` | lifecycle setting for the apiserver container | +| clustermesh.apiserver.metrics.enabled | bool | `true` | Enables exporting apiserver metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.etcd.enabled | bool | `true` | Enables exporting etcd metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.etcd.mode | string | `"basic"` | Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. | +| clustermesh.apiserver.metrics.etcd.port | int | `9963` | Configure the port the etcd metric server listens on. | +| clustermesh.apiserver.metrics.kvstoremesh.enabled | bool | `true` | Enables exporting KVStoreMesh metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.kvstoremesh.port | int | `9964` | Configure the port the KVStoreMesh metric server listens on. | +| clustermesh.apiserver.metrics.port | int | `9962` | Configure the port the apiserver metric server listens on. | +| clustermesh.apiserver.metrics.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor clustermesh-apiserver | +| clustermesh.apiserver.metrics.serviceMonitor.enabled | bool | `false` | Enable service monitor. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| clustermesh.apiserver.metrics.serviceMonitor.etcd.interval | string | `"10s"` | Interval for scrape metrics (etcd metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.etcd.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.etcd.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics (apiserver metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.interval | string | `"10s"` | Interval for scrape metrics (KVStoreMesh metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor clustermesh-apiserver | +| clustermesh.apiserver.metrics.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) | +| clustermesh.apiserver.metrics.serviceMonitor.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) | +| clustermesh.apiserver.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| clustermesh.apiserver.podAnnotations | object | `{}` | Annotations to be added to clustermesh-apiserver pods | +| clustermesh.apiserver.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | +| clustermesh.apiserver.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | +| clustermesh.apiserver.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | +| clustermesh.apiserver.podLabels | object | `{}` | Labels to be added to clustermesh-apiserver pods | +| clustermesh.apiserver.podSecurityContext | object | `{}` | Security context to be added to clustermesh-apiserver pods | +| clustermesh.apiserver.priorityClassName | string | `""` | The priority class to use for clustermesh-apiserver | +| clustermesh.apiserver.replicas | int | `1` | Number of replicas run for the clustermesh-apiserver deployment. | +| clustermesh.apiserver.resources | object | `{}` | Resource requests and limits for the clustermesh-apiserver | +| clustermesh.apiserver.securityContext | object | `{}` | Security context to be added to clustermesh-apiserver containers | +| clustermesh.apiserver.service.annotations | object | `{}` | Annotations for the clustermesh-apiserver For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 | +| clustermesh.apiserver.service.externalTrafficPolicy | string | `nil` | The externalTrafficPolicy of service used for apiserver access. | +| clustermesh.apiserver.service.internalTrafficPolicy | string | `nil` | The internalTrafficPolicy of service used for apiserver access. | +| clustermesh.apiserver.service.nodePort | int | `32379` | Optional port to use as the node port for apiserver access. WARNING: make sure to configure a different NodePort in each cluster if kube-proxy replacement is enabled, as Cilium is currently affected by a known bug (#24692) when NodePorts are handled by the KPR implementation. If a service with the same NodePort exists both in the local and the remote cluster, all traffic originating from inside the cluster and targeting the corresponding NodePort will be redirected to a local backend, regardless of whether the destination node belongs to the local or the remote cluster. | +| clustermesh.apiserver.service.type | string | `"NodePort"` | The type of service used for apiserver access. | +| clustermesh.apiserver.terminationGracePeriodSeconds | int | `30` | terminationGracePeriodSeconds for the clustermesh-apiserver deployment | +| clustermesh.apiserver.tls.admin | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver admin certificate and private key. Used if 'auto' is not enabled. | +| clustermesh.apiserver.tls.authMode | string | `"legacy"` | Configure the clustermesh authentication mode. Supported values: - legacy: All clusters access remote clustermesh instances with the same username (i.e., remote). The "remote" certificate must be generated with CN=remote if provided manually. - migration: Intermediate mode required to upgrade from legacy to cluster (and vice versa) with no disruption. Specifically, it enables the creation of the per-cluster usernames, while still using the common one for authentication. The "remote" certificate must be generated with CN=remote if provided manually (same as legacy). - cluster: Each cluster accesses remote etcd instances with a username depending on the local cluster name (i.e., remote-). The "remote" certificate must be generated with CN=remote- if provided manually. Cluster mode is meaningful only when the same CA is shared across all clusters part of the mesh. | +| clustermesh.apiserver.tls.auto | object | `{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm"}` | Configure automatic TLS certificates generation. A Kubernetes CronJob is used the generate any certificates not provided by the user at installation time. | +| clustermesh.apiserver.tls.auto.certManagerIssuerRef | object | `{}` | certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. | +| clustermesh.apiserver.tls.auto.certValidityDuration | int | `1095` | Generated certificates validity duration in days. | +| clustermesh.apiserver.tls.auto.enabled | bool | `true` | When set to true, automatically generate a CA and certificates to enable mTLS between clustermesh-apiserver and external workload instances. If set to false, the certs to be provided by setting appropriate values below. | +| clustermesh.apiserver.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver client certificate and private key. Used if 'auto' is not enabled. | +| clustermesh.apiserver.tls.remote | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver remote cluster certificate and private key. Used if 'auto' is not enabled. | +| clustermesh.apiserver.tls.server | object | `{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}` | base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. Used if 'auto' is not enabled. | +| clustermesh.apiserver.tls.server.extraDnsNames | list | `[]` | Extra DNS names added to certificate when it's auto generated | +| clustermesh.apiserver.tls.server.extraIpAddresses | list | `[]` | Extra IP addresses added to certificate when it's auto generated | +| clustermesh.apiserver.tolerations | list | `[]` | Node tolerations for pod assignment on nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| clustermesh.apiserver.topologySpreadConstraints | list | `[]` | Pod topology spread constraints for clustermesh-apiserver | +| clustermesh.apiserver.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | clustermesh-apiserver update strategy | +| clustermesh.config | object | `{"clusters":[],"domain":"mesh.cilium.io","enabled":false}` | Clustermesh explicit configuration. | +| clustermesh.config.clusters | list | `[]` | List of clusters to be peered in the mesh. | +| clustermesh.config.domain | string | `"mesh.cilium.io"` | Default dns domain for the Clustermesh API servers This is used in the case cluster addresses are not provided and IPs are used. | +| clustermesh.config.enabled | bool | `false` | Enable the Clustermesh explicit configuration. | +| clustermesh.maxConnectedClusters | int | `255` | The maximum number of clusters to support in a ClusterMesh. This value cannot be changed on running clusters, and all clusters in a ClusterMesh must be configured with the same value. Values > 255 will decrease the maximum allocatable cluster-local identities. Supported values are 255 and 511. | +| clustermesh.useAPIServer | bool | `false` | Deploy clustermesh-apiserver for clustermesh | +| cni.binPath | string | `"/opt/cni/bin"` | Configure the path to the CNI binary directory on the host. | +| cni.chainingMode | string | `nil` | Configure chaining on top of other CNI plugins. Possible values: - none - aws-cni - flannel - generic-veth - portmap | +| cni.chainingTarget | string | `nil` | A CNI network name in to which the Cilium plugin should be added as a chained plugin. This will cause the agent to watch for a CNI network with this network name. When it is found, this will be used as the basis for Cilium's CNI configuration file. If this is set, it assumes a chaining mode of generic-veth. As a special case, a chaining mode of aws-cni implies a chainingTarget of aws-cni. | +| cni.confFileMountPath | string | `"/tmp/cni-configuration"` | Configure the path to where to mount the ConfigMap inside the agent pod. | +| cni.confPath | string | `"/etc/cni/net.d"` | Configure the path to the CNI configuration directory on the host. | +| cni.configMapKey | string | `"cni-config"` | Configure the key in the CNI ConfigMap to read the contents of the CNI configuration from. | +| cni.customConf | bool | `false` | Skip writing of the CNI configuration. This can be used if writing of the CNI configuration is performed by external automation. | +| cni.exclusive | bool | `true` | Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. | +| cni.hostConfDirMountPath | string | `"/host/etc/cni/net.d"` | Configure the path to where the CNI configuration directory is mounted inside the agent pod. | +| cni.install | bool | `true` | Install the CNI configuration and binary files into the filesystem. | +| cni.logFile | string | `"/var/run/cilium/cilium-cni.log"` | Configure the log file for CNI logging with retention policy of 7 days. Disable CNI file logging by setting this field to empty explicitly. | +| cni.resources | object | `{"requests":{"cpu":"100m","memory":"10Mi"}}` | Specifies the resources for the cni initContainer | +| cni.uninstall | bool | `false` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | +| conntrackGCInterval | string | `"0s"` | Configure how frequently garbage collection should occur for the datapath connection tracking table. | +| conntrackGCMaxInterval | string | `""` | Configure the maximum frequency for the garbage collection of the connection tracking table. Only affects the automatic computation for the frequency and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently clean up unused identities created from ToFQDN policies. | +| containerRuntime | object | `{"integration":"none"}` | Configure container runtime specific integration. Deprecated in favor of bpf.autoMount.enabled. To be removed in 1.15. | +| containerRuntime.integration | string | `"none"` | Enables specific integrations for container runtimes. Supported values: - crio - none | +| crdWaitTimeout | string | `"5m"` | Configure timeout in which Cilium will exit if CRDs are not available | +| customCalls | object | `{"enabled":false}` | Tail call hooks for custom eBPF programs. | +| customCalls.enabled | bool | `false` | Enable tail call hooks for custom eBPF programs. | +| daemon.allowedConfigOverrides | string | `nil` | allowedConfigOverrides is a list of config-map keys that can be overridden. That is to say, if this value is set, config sources (excepting the first one) can only override keys in this list. This takes precedence over blockedConfigOverrides. By default, all keys may be overridden. To disable overrides, set this to "none" or change the configSources variable. | +| daemon.blockedConfigOverrides | string | `nil` | blockedConfigOverrides is a list of config-map keys that may not be overridden. In other words, if any of these keys appear in a configuration source excepting the first one, they will be ignored This is ignored if allowedConfigOverrides is set. By default, all keys may be overridden. | +| daemon.configSources | string | `nil` | Configure a custom list of possible configuration override sources The default is "config-map:cilium-config,cilium-node-config". For supported values, see the help text for the build-config subcommand. Note that this value should be a comma-separated string. | +| daemon.runPath | string | `"/var/run/cilium"` | Configure where Cilium runtime state should be stored. | +| dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for cilium-agent grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | +| debug.enabled | bool | `false` | Enable debug logging | +| debug.verbose | string | `nil` | Configure verbosity levels for debug logging This option is used to enable debug messages for operations related to such sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is for enabling debug messages emitted per request, message and connection. Multiple values can be set via a space-separated string (e.g. "datapath envoy"). Applicable values: - flow - kvstore - envoy - datapath - policy | +| disableEndpointCRD | bool | `false` | Disable the usage of CiliumEndpoint CRD. | +| dnsPolicy | string | `""` | DNS policy for Cilium agent pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | +| dnsProxy.dnsRejectResponseCode | string | `"refused"` | DNS response code for rejecting DNS requests, available options are '[nameError refused]'. | +| dnsProxy.enableDnsCompression | bool | `true` | Allow the DNS proxy to compress responses to endpoints that are larger than 512 Bytes or the EDNS0 option, if present. | +| dnsProxy.endpointMaxIpPerHostname | int | `50` | Maximum number of IPs to maintain per FQDN name for each endpoint. | +| dnsProxy.idleConnectionGracePeriod | string | `"0s"` | Time during which idle but previously active connections with expired DNS lookups are still considered alive. | +| dnsProxy.maxDeferredConnectionDeletes | int | `10000` | Maximum number of IPs to retain for expired DNS lookups with still-active connections. | +| dnsProxy.minTtl | int | `0` | The minimum time, in seconds, to use DNS data for toFQDNs policies. If the upstream DNS server returns a DNS record with a shorter TTL, Cilium overwrites the TTL with this value. Setting this value to zero means that Cilium will honor the TTLs returned by the upstream DNS server. | +| dnsProxy.preCache | string | `""` | DNS cache data at this path is preloaded on agent startup. | +| dnsProxy.proxyPort | int | `0` | Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. | +| dnsProxy.proxyResponseMaxDelay | string | `"100ms"` | The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. | +| egressGateway.enabled | bool | `false` | Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. | +| egressGateway.installRoutes | bool | `false` | Deprecated without a replacement necessary. | +| egressGateway.reconciliationTriggerInterval | string | `"1s"` | Time between triggers of egress gateway state reconciliations | +| enableCiliumEndpointSlice | bool | `false` | Enable CiliumEndpointSlice feature. | +| enableCriticalPriorityClass | bool | `true` | Explicitly enable or disable priority class. .Capabilities.KubeVersion is unsettable in `helm template` calls, it depends on k8s libraries version that Helm was compiled against. This option allows to explicitly disable setting the priority class, which is useful for rendering charts for gke clusters in advance. | +| enableIPv4BIGTCP | bool | `false` | Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods | +| enableIPv4Masquerade | bool | `true` | Enables masquerading of IPv4 traffic leaving the node from endpoints. | +| enableIPv6BIGTCP | bool | `false` | Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods | +| enableIPv6Masquerade | bool | `true` | Enables masquerading of IPv6 traffic leaving the node from endpoints. | +| enableK8sTerminatingEndpoint | bool | `true` | Configure whether to enable auto detect of terminating state for endpoints in order to support graceful termination. | +| enableMasqueradeRouteSource | bool | `false` | Enables masquerading to the source of the route for traffic leaving the node from endpoints. | +| enableRuntimeDeviceDetection | bool | `false` | Enables experimental support for the detection of new and removed datapath devices. When devices change the eBPF datapath is reloaded and services updated. If "devices" is set then only those devices, or devices matching a wildcard will be considered. | +| enableXTSocketFallback | bool | `true` | Enables the fallback compatibility solution for when the xt_socket kernel module is missing and it is needed for the datapath L7 redirection to work properly. See documentation for details on when this can be disabled: https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. | +| encryption.enabled | bool | `false` | Enable transparent network encryption. | +| encryption.interface | string | `""` | Deprecated in favor of encryption.ipsec.interface. To be removed in 1.15. The interface to use for encrypted traffic. This option is only effective when encryption.type is set to ipsec. | +| encryption.ipsec.interface | string | `""` | The interface to use for encrypted traffic. | +| encryption.ipsec.keyFile | string | `""` | Name of the key file inside the Kubernetes secret configured via secretName. | +| encryption.ipsec.keyRotationDuration | string | `"5m"` | Maximum duration of the IPsec key rotation. The previous key will be removed after that delay. | +| encryption.ipsec.keyWatcher | bool | `true` | Enable the key watcher. If disabled, a restart of the agent will be necessary on key rotations. | +| encryption.ipsec.mountPath | string | `""` | Path to mount the secret inside the Cilium pod. | +| encryption.ipsec.secretName | string | `""` | Name of the Kubernetes secret containing the encryption keys. | +| encryption.keyFile | string | `"keys"` | Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. Name of the key file inside the Kubernetes secret configured via secretName. This option is only effective when encryption.type is set to ipsec. | +| encryption.mountPath | string | `"/etc/ipsec"` | Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. Path to mount the secret inside the Cilium pod. This option is only effective when encryption.type is set to ipsec. | +| encryption.nodeEncryption | bool | `false` | Enable encryption for pure node to node traffic. This option is only effective when encryption.type is set to "wireguard". | +| encryption.secretName | string | `"cilium-ipsec-keys"` | Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. Name of the Kubernetes secret containing the encryption keys. This option is only effective when encryption.type is set to ipsec. | +| encryption.strictMode | object | `{"allowRemoteNodeIdentities":false,"cidr":"","enabled":false}` | Configure the WireGuard Pod2Pod strict mode. | +| encryption.strictMode.allowRemoteNodeIdentities | bool | `false` | Allow dynamic lookup of remote node identities. This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. | +| encryption.strictMode.cidr | string | `""` | CIDR for the WireGuard Pod2Pod strict mode. | +| encryption.strictMode.enabled | bool | `false` | Enable WireGuard Pod2Pod strict mode. | +| encryption.type | string | `"ipsec"` | Encryption method. Can be either ipsec or wireguard. | +| encryption.wireguard.persistentKeepalive | string | `"0s"` | Controls Wireguard PersistentKeepalive option. Set 0s to disable. | +| encryption.wireguard.userspaceFallback | bool | `false` | Enables the fallback to the user-space implementation. | +| endpointHealthChecking.enabled | bool | `true` | Enable connectivity health checking between virtual endpoints. | +| endpointRoutes.enabled | bool | `false` | Enable use of per endpoint routes instead of routing via the cilium_host interface. | +| endpointStatus | object | `{"enabled":false,"status":""}` | Enable endpoint status. Status can be: policy, health, controllers, log and / or state. For 2 or more options use a space. | +| eni.awsEnablePrefixDelegation | bool | `false` | Enable ENI prefix delegation | +| eni.awsReleaseExcessIPs | bool | `false` | Release IPs not used from the ENI | +| eni.ec2APIEndpoint | string | `""` | EC2 API endpoint to use | +| eni.enabled | bool | `false` | Enable Elastic Network Interface (ENI) integration. | +| eni.eniTags | object | `{}` | Tags to apply to the newly created ENIs | +| eni.gcInterval | string | `"5m"` | Interval for garbage collection of unattached ENIs. Set to "0s" to disable. | +| eni.gcTags | object | `{"io.cilium/cilium-managed":"true,"io.cilium/cluster-name":""}` | Additional tags attached to ENIs created by Cilium. Dangling ENIs with this tag will be garbage collected | +| eni.iamRole | string | `""` | If using IAM role for Service Accounts will not try to inject identity values from cilium-aws kubernetes secret. Adds annotation to service account if managed by Helm. See https://github.com/aws/amazon-eks-pod-identity-webhook | +| eni.instanceTagsFilter | list | `[]` | Filter via AWS EC2 Instance tags (k=v) which will dictate which AWS EC2 Instances are going to be used to create new ENIs | +| eni.subnetIDsFilter | list | `[]` | Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | +| eni.subnetTagsFilter | list | `[]` | Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | +| eni.updateEC2AdapterLimitViaAPI | bool | `true` | Update ENI Adapter limits from the EC2 API | +| envoy.affinity | object | `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"cilium.io/no-schedule","operator":"NotIn","values":["true"]}]}]}},"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]},"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium-envoy"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-envoy. | +| envoy.annotations | object | `{}` | Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) | +| envoy.connectTimeoutSeconds | int | `2` | Time in seconds after which a TCP connection attempt times out | +| envoy.dnsPolicy | string | `nil` | DNS policy for Cilium envoy pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | +| envoy.enabled | bool | `false` | Enable Envoy Proxy in standalone DaemonSet. | +| envoy.extraArgs | list | `[]` | Additional envoy container arguments. | +| envoy.extraContainers | list | `[]` | Additional containers added to the cilium Envoy DaemonSet. | +| envoy.extraEnv | list | `[]` | Additional envoy container environment variables. | +| envoy.extraHostPathMounts | list | `[]` | Additional envoy hostPath mounts. | +| envoy.extraVolumeMounts | list | `[]` | Additional envoy volumeMounts. | +| envoy.extraVolumes | list | `[]` | Additional envoy volumes. | +| envoy.healthPort | int | `9878` | TCP port for the health API. | +| envoy.idleTimeoutDurationSeconds | int | `60` | Set Envoy upstream HTTP idle connection timeout seconds. Does not apply to connections with pending requests. Default 60s | +| envoy.image | object | `{"digest":"sha256:877ead12d08d4c04a9f67f86d3c6e542aeb7bf97e1e401aee74de456f496ac30","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.27.3-99c1c8f42c8de70fc8f6dd594f4a425cd38b6688","useDigest":true}` | Envoy container image. | +| envoy.livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | +| envoy.livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | +| envoy.log.format | string | `"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"` | The format string to use for laying out the log message metadata of Envoy. | +| envoy.log.path | string | `""` | Path to a separate Envoy log file, if any. Defaults to /dev/stdout. | +| envoy.maxConnectionDurationSeconds | int | `0` | Set Envoy HTTP option max_connection_duration seconds. Default 0 (disable) | +| envoy.maxRequestsPerConnection | int | `0` | ProxyMaxRequestsPerConnection specifies the max_requests_per_connection setting for Envoy | +| envoy.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for cilium-envoy. | +| envoy.podAnnotations | object | `{}` | Annotations to be added to envoy pods | +| envoy.podLabels | object | `{}` | Labels to be added to envoy pods | +| envoy.podSecurityContext | object | `{}` | Security Context for cilium-envoy pods. | +| envoy.priorityClassName | string | `nil` | The priority class to use for cilium-envoy. | +| envoy.prometheus | object | `{"enabled":true,"port":"9964","serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Configure Cilium Envoy Prometheus options. Note that some of these apply to either cilium-agent or cilium-envoy. | +| envoy.prometheus.enabled | bool | `true` | Enable prometheus metrics for cilium-envoy | +| envoy.prometheus.port | string | `"9964"` | Serve prometheus metrics for cilium-envoy on the configured port | +| envoy.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) Note that this setting applies to both cilium-envoy _and_ cilium-agent with Envoy enabled. | +| envoy.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| envoy.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-envoy or for cilium-agent with Envoy configured. | +| envoy.prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-envoy or for cilium-agent with Envoy configured. | +| envoy.readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | +| envoy.readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | +| envoy.resources | object | `{}` | Envoy resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| envoy.rollOutPods | bool | `false` | Roll out cilium envoy pods automatically when configmap is updated. | +| envoy.securityContext.capabilities.envoy | list | `["NET_ADMIN","SYS_ADMIN"]` | Capabilities for the `cilium-envoy` container | +| envoy.securityContext.privileged | bool | `false` | Run the pod with elevated privileges | +| envoy.securityContext.seLinuxOptions | object | `{"level":"s0","type":"spc_t"}` | SELinux options for the `cilium-envoy` container | +| envoy.startupProbe.failureThreshold | int | `105` | failure threshold of startup probe. 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) | +| envoy.startupProbe.periodSeconds | int | `2` | interval between checks of the startup probe | +| envoy.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for cilium-envoy DaemonSet. | +| envoy.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for envoy scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| envoy.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | cilium-envoy update strategy ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset | +| envoyConfig.enabled | bool | `false` | Enable CiliumEnvoyConfig CRD CiliumEnvoyConfig CRD can also be implicitly enabled by other options. | +| envoyConfig.secretsNamespace | object | `{"create":true,"name":"cilium-secrets"}` | SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. | +| envoyConfig.secretsNamespace.create | bool | `true` | Create secrets namespace for CiliumEnvoyConfig CRDs. | +| envoyConfig.secretsNamespace.name | string | `"cilium-secrets"` | The name of the secret namespace to which Cilium agents are given read access. | +| etcd.annotations | object | `{}` | Annotations to be added to all top-level etcd-operator objects (resources under templates/etcd-operator) | +| etcd.clusterDomain | string | `"cluster.local"` | Cluster domain for cilium-etcd-operator. | +| etcd.enabled | bool | `false` | Enable etcd mode for the agent. | +| etcd.endpoints | list | `["https://CHANGE-ME:2379"]` | List of etcd endpoints (not needed when using managed=true). | +| etcd.extraArgs | list | `[]` | Additional cilium-etcd-operator container arguments. | +| etcd.extraVolumeMounts | list | `[]` | Additional cilium-etcd-operator volumeMounts. | +| etcd.extraVolumes | list | `[]` | Additional cilium-etcd-operator volumes. | +| etcd.image | object | `{"digest":"sha256:04b8327f7f992693c2cb483b999041ed8f92efc8e14f2a5f3ab95574a65ea2dc","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-etcd-operator","tag":"v2.0.7","useDigest":true}` | cilium-etcd-operator image. | +| etcd.k8sService | bool | `false` | If etcd is behind a k8s service set this option to true so that Cilium does the service translation automatically without requiring a DNS to be running. | +| etcd.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-etcd-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| etcd.podAnnotations | object | `{}` | Annotations to be added to cilium-etcd-operator pods | +| etcd.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | +| etcd.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | +| etcd.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | +| etcd.podLabels | object | `{}` | Labels to be added to cilium-etcd-operator pods | +| etcd.podSecurityContext | object | `{}` | Security context to be added to cilium-etcd-operator pods | +| etcd.priorityClassName | string | `""` | The priority class to use for cilium-etcd-operator | +| etcd.resources | object | `{}` | cilium-etcd-operator resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| etcd.securityContext | object | `{}` | Security context to be added to cilium-etcd-operator pods | +| etcd.ssl | bool | `false` | Enable use of TLS/SSL for connectivity to etcd. (auto-enabled if managed=true) | +| etcd.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for cilium-etcd-operator scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| etcd.topologySpreadConstraints | list | `[]` | Pod topology spread constraints for cilium-etcd-operator | +| etcd.updateStrategy | object | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":1},"type":"RollingUpdate"}` | cilium-etcd-operator update strategy | +| externalIPs.enabled | bool | `false` | Enable ExternalIPs service support. | +| externalWorkloads | object | `{"enabled":false}` | Configure external workloads support | +| externalWorkloads.enabled | bool | `false` | Enable support for external workloads, such as VMs (false by default). | +| extraArgs | list | `[]` | Additional agent container arguments. | +| extraConfig | object | `{}` | extraConfig allows you to specify additional configuration parameters to be included in the cilium-config configmap. | +| extraContainers | list | `[]` | Additional containers added to the cilium DaemonSet. | +| extraEnv | list | `[]` | Additional agent container environment variables. | +| extraHostPathMounts | list | `[]` | Additional agent hostPath mounts. | +| extraVolumeMounts | list | `[]` | Additional agent volumeMounts. | +| extraVolumes | list | `[]` | Additional agent volumes. | +| gatewayAPI.enabled | bool | `false` | Enable support for Gateway API in cilium This will automatically set enable-envoy-config as well. | +| gatewayAPI.secretsNamespace | object | `{"create":true,"name":"cilium-secrets","sync":true}` | SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. | +| gatewayAPI.secretsNamespace.create | bool | `true` | Create secrets namespace for Gateway API. | +| gatewayAPI.secretsNamespace.name | string | `"cilium-secrets"` | Name of Gateway API secret namespace. | +| gatewayAPI.secretsNamespace.sync | bool | `true` | Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. If disabled, TLS secrets must be maintained externally. | +| gke.enabled | bool | `false` | Enable Google Kubernetes Engine integration | +| healthChecking | bool | `true` | Enable connectivity health checking. | +| healthPort | int | `9879` | TCP port for the agent health API. This is not the port for cilium-health. | +| highScaleIPcache | object | `{"enabled":false}` | EnableHighScaleIPcache enables the special ipcache mode for high scale clusters. The ipcache content will be reduced to the strict minimum and traffic will be encapsulated to carry security identities. | +| highScaleIPcache.enabled | bool | `false` | Enable the high scale mode for the ipcache. | +| hostFirewall | object | `{"enabled":false}` | Configure the host firewall. | +| hostFirewall.enabled | bool | `false` | Enables the enforcement of host policies in the eBPF datapath. | +| hostPort.enabled | bool | `false` | Enable hostPort service support. | +| hubble.annotations | object | `{}` | Annotations to be added to all top-level hubble objects (resources under templates/hubble) | +| hubble.enabled | bool | `true` | Enable Hubble (true by default). | +| hubble.export | object | `{"dynamic":{"config":{"configMapName":"cilium-flowlog-config","content":[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}],"createConfigMap":true},"enabled":false},"fileMaxBackups":5,"fileMaxSizeMb":10,"static":{"allowList":[],"denyList":[],"enabled":false,"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log"}}` | Hubble flows export. | +| hubble.export.dynamic | object | `{"config":{"configMapName":"cilium-flowlog-config","content":[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}],"createConfigMap":true},"enabled":false}` | - Dynamic exporters configuration. Dynamic exporters may be reconfigured without a need of agent restarts. | +| hubble.export.dynamic.config.configMapName | string | `"cilium-flowlog-config"` | -- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. | +| hubble.export.dynamic.config.content | list | `[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}]` | -- Exporters configuration in YAML format. | +| hubble.export.dynamic.config.createConfigMap | bool | `true` | -- True if helm installer should create config map. Switch to false if you want to self maintain the file content. | +| hubble.export.fileMaxBackups | int | `5` | - Defines max number of backup/rotated files. | +| hubble.export.fileMaxSizeMb | int | `10` | - Defines max file size of output file before it gets rotated. | +| hubble.export.static | object | `{"allowList":[],"denyList":[],"enabled":false,"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log"}` | - Static exporter configuration. Static exporter is bound to agent lifecycle. | +| hubble.listenAddress | string | `":4244"` | An additional address for Hubble to listen to. Set this field ":4244" if you are enabling Hubble Relay, as it assumes that Hubble is listening on port 4244. | +| hubble.metrics | object | `{"dashboards":{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null},"enableOpenMetrics":false,"enabled":null,"port":9965,"serviceAnnotations":{},"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Hubble metrics configuration. See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics for more comprehensive documentation about Hubble metrics. | +| hubble.metrics.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for hubble grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | +| hubble.metrics.enableOpenMetrics | bool | `false` | Enables exporting hubble metrics in OpenMetrics format. | +| hubble.metrics.enabled | string | `nil` | Configures the list of metrics to collect. If empty or null, metrics are disabled. Example: enabled: - dns:query;ignoreAAAA - drop - tcp - flow - icmp - http You can specify the list of metrics from the helm CLI: --set hubble.metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" | +| hubble.metrics.port | int | `9965` | Configure the port the hubble metric server listens on. | +| hubble.metrics.serviceAnnotations | object | `{}` | Annotations to be added to hubble-metrics service. | +| hubble.metrics.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor hubble | +| hubble.metrics.serviceMonitor.enabled | bool | `false` | Create ServiceMonitor resources for Prometheus Operator. This requires the prometheus CRDs to be available. ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| hubble.metrics.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| hubble.metrics.serviceMonitor.jobLabel | string | `""` | jobLabel to add for ServiceMonitor hubble | +| hubble.metrics.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor hubble | +| hubble.metrics.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor hubble | +| hubble.metrics.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor hubble | +| hubble.peerService.clusterDomain | string | `"cluster.local"` | The cluster domain to use to query the Hubble Peer service. It should be the local cluster. | +| hubble.peerService.targetPort | int | `4244` | Target Port for the Peer service, must match the hubble.listenAddress' port. | +| hubble.preferIpv6 | bool | `false` | Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. | +| hubble.redact | object | `{"enabled":false,"http":{"headers":{"allow":[],"deny":[]},"urlQuery":false,"userInfo":true},"kafka":{"apiKey":false}}` | Enables redacting sensitive information present in Layer 7 flows. | +| hubble.redact.http.headers.allow | list | `[]` | List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. Example: redact: enabled: true http: headers: allow: - traceparent - tracestate - Cache-Control You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" | +| hubble.redact.http.headers.deny | list | `[]` | List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. Example: redact: enabled: true http: headers: deny: - Authorization - Proxy-Authorization You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" | +| hubble.redact.http.urlQuery | bool | `false` | Enables redacting URL query (GET) parameters. Example: redact: enabled: true http: urlQuery: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.urlQuery="true" | +| hubble.redact.http.userInfo | bool | `true` | Enables redacting user info, e.g., password when basic auth is used. Example: redact: enabled: true http: userInfo: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.userInfo="true" | +| hubble.redact.kafka.apiKey | bool | `false` | Enables redacting Kafka's API key. Example: redact: enabled: true kafka: apiKey: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.kafka.apiKey="true" | +| hubble.relay.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for hubble-replay | +| hubble.relay.annotations | object | `{}` | Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) | +| hubble.relay.dialTimeout | string | `nil` | Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). | +| hubble.relay.enabled | bool | `false` | Enable Hubble Relay (requires hubble.enabled=true) | +| hubble.relay.extraEnv | list | `[]` | Additional hubble-relay environment variables. | +| hubble.relay.extraVolumeMounts | list | `[]` | Additional hubble-relay volumeMounts. | +| hubble.relay.extraVolumes | list | `[]` | Additional hubble-relay volumes. | +| hubble.relay.gops.enabled | bool | `true` | Enable gops for hubble-relay | +| hubble.relay.gops.port | int | `9893` | Configure gops listen port for hubble-relay | +| hubble.relay.image | object | `{"digest":"sha256:b9c6431aa4f22242a5d0d750c621d9d04bdc25549e4fb1116bfec98dd87958a2","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.15.3","useDigest":true}` | Hubble-relay container image. | +| hubble.relay.listenHost | string | `""` | Host to listen to. Specify an empty string to bind to all the interfaces. | +| hubble.relay.listenPort | string | `"4245"` | Port to listen to. | +| hubble.relay.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| hubble.relay.podAnnotations | object | `{}` | Annotations to be added to hubble-relay pods | +| hubble.relay.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | +| hubble.relay.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | +| hubble.relay.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | +| hubble.relay.podLabels | object | `{}` | Labels to be added to hubble-relay pods | +| hubble.relay.podSecurityContext | object | `{"fsGroup":65532}` | hubble-relay pod security context | +| hubble.relay.pprof.address | string | `"localhost"` | Configure pprof listen address for hubble-relay | +| hubble.relay.pprof.enabled | bool | `false` | Enable pprof for hubble-relay | +| hubble.relay.pprof.port | int | `6062` | Configure pprof listen port for hubble-relay | +| hubble.relay.priorityClassName | string | `""` | The priority class to use for hubble-relay | +| hubble.relay.prometheus | object | `{"enabled":false,"port":9966,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":null,"relabelings":null}}` | Enable prometheus metrics for hubble-relay on the configured port at /metrics | +| hubble.relay.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor hubble-relay | +| hubble.relay.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| hubble.relay.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| hubble.relay.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor hubble-relay | +| hubble.relay.prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor hubble-relay | +| hubble.relay.prometheus.serviceMonitor.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor hubble-relay | +| hubble.relay.replicas | int | `1` | Number of replicas run for the hubble-relay deployment. | +| hubble.relay.resources | object | `{}` | Specifies the resources for the hubble-relay pods | +| hubble.relay.retryTimeout | string | `nil` | Backoff duration to retry connecting to the local hubble instance in case of failure (e.g. "30s"). | +| hubble.relay.rollOutPods | bool | `false` | Roll out Hubble Relay pods automatically when configmap is updated. | +| hubble.relay.securityContext | object | `{"capabilities":{"drop":["ALL"]},"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | hubble-relay container security context | +| hubble.relay.service | object | `{"nodePort":31234,"type":"ClusterIP"}` | hubble-relay service configuration. | +| hubble.relay.service.nodePort | int | `31234` | - The port to use when the service type is set to NodePort. | +| hubble.relay.service.type | string | `"ClusterIP"` | - The type of service used for Hubble Relay access, either ClusterIP or NodePort. | +| hubble.relay.sortBufferDrainTimeout | string | `nil` | When the per-request flows sort buffer is not full, a flow is drained every time this timeout is reached (only affects requests in follow-mode) (e.g. "1s"). | +| hubble.relay.sortBufferLenMax | string | `nil` | Max number of flows that can be buffered for sorting before being sent to the client (per request) (e.g. 100). | +| hubble.relay.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for hubble relay Deployment. | +| hubble.relay.tls | object | `{"client":{"cert":"","key":""},"server":{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false,"relayName":"ui.hubble-relay.cilium.io"}}` | TLS configuration for Hubble Relay | +| hubble.relay.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values for the hubble-relay client certificate and private key This keypair is presented to Hubble server instances for mTLS authentication and is required when hubble.tls.enabled is true. These values need to be set manually if hubble.tls.auto.enabled is false. | +| hubble.relay.tls.server | object | `{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false,"relayName":"ui.hubble-relay.cilium.io"}` | base64 encoded PEM values for the hubble-relay server certificate and private key | +| hubble.relay.tls.server.extraDnsNames | list | `[]` | extra DNS names added to certificate when its auto gen | +| hubble.relay.tls.server.extraIpAddresses | list | `[]` | extra IP addresses added to certificate when its auto gen | +| hubble.relay.tolerations | list | `[]` | Node tolerations for pod assignment on nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| hubble.relay.topologySpreadConstraints | list | `[]` | Pod topology spread constraints for hubble-relay | +| hubble.relay.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | hubble-relay update strategy | +| hubble.skipUnknownCGroupIDs | bool | `true` | Skip Hubble events with unknown cgroup ids | +| hubble.socketPath | string | `"/var/run/cilium/hubble.sock"` | Unix domain socket path to listen to when Hubble is enabled. | +| hubble.tls | object | `{"auto":{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm","schedule":"0 0 1 */4 *"},"enabled":true,"server":{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}}` | TLS configuration for Hubble | +| hubble.tls.auto | object | `{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm","schedule":"0 0 1 */4 *"}` | Configure automatic TLS certificates generation. | +| hubble.tls.auto.certManagerIssuerRef | object | `{}` | certmanager issuer used when hubble.tls.auto.method=certmanager. | +| hubble.tls.auto.certValidityDuration | int | `1095` | Generated certificates validity duration in days. | +| hubble.tls.auto.enabled | bool | `true` | Auto-generate certificates. When set to true, automatically generate a CA and certificates to enable mTLS between Hubble server and Hubble Relay instances. If set to false, the certs for Hubble server need to be provided by setting appropriate values below. | +| hubble.tls.auto.method | string | `"helm"` | Set the method to auto-generate certificates. Supported values: - helm: This method uses Helm to generate all certificates. - cronJob: This method uses a Kubernetes CronJob the generate any certificates not provided by the user at installation time. - certmanager: This method use cert-manager to generate & rotate certificates. | +| hubble.tls.auto.schedule | string | `"0 0 1 */4 *"` | Schedule for certificates regeneration (regardless of their expiration date). Only used if method is "cronJob". If nil, then no recurring job will be created. Instead, only the one-shot job is deployed to generate the certificates at installation time. Defaults to midnight of the first day of every fourth month. For syntax, see https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax | +| hubble.tls.enabled | bool | `true` | Enable mutual TLS for listenAddress. Setting this value to false is highly discouraged as the Hubble API provides access to potentially sensitive network flow metadata and is exposed on the host network. | +| hubble.tls.server | object | `{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}` | base64 encoded PEM values for the Hubble server certificate and private key | +| hubble.tls.server.extraDnsNames | list | `[]` | Extra DNS names added to certificate when it's auto generated | +| hubble.tls.server.extraIpAddresses | list | `[]` | Extra IP addresses added to certificate when it's auto generated | +| hubble.ui.affinity | object | `{}` | Affinity for hubble-ui | +| hubble.ui.annotations | object | `{}` | Annotations to be added to all top-level hubble-ui objects (resources under templates/hubble-ui) | +| hubble.ui.backend.extraEnv | list | `[]` | Additional hubble-ui backend environment variables. | +| hubble.ui.backend.extraVolumeMounts | list | `[]` | Additional hubble-ui backend volumeMounts. | +| hubble.ui.backend.extraVolumes | list | `[]` | Additional hubble-ui backend volumes. | +| hubble.ui.backend.image | object | `{"digest":"sha256:1e7657d997c5a48253bb8dc91ecee75b63018d16ff5e5797e5af367336bc8803","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui-backend","tag":"v0.13.0","useDigest":true}` | Hubble-ui backend image. | +| hubble.ui.backend.livenessProbe.enabled | bool | `false` | Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) | +| hubble.ui.backend.readinessProbe.enabled | bool | `false` | Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) | +| hubble.ui.backend.resources | object | `{}` | Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. | +| hubble.ui.backend.securityContext | object | `{}` | Hubble-ui backend security context. | +| hubble.ui.baseUrl | string | `"/"` | Defines base url prefix for all hubble-ui http requests. It needs to be changed in case if ingress for hubble-ui is configured under some sub-path. Trailing `/` is required for custom path, ex. `/service-map/` | +| hubble.ui.enabled | bool | `false` | Whether to enable the Hubble UI. | +| hubble.ui.frontend.extraEnv | list | `[]` | Additional hubble-ui frontend environment variables. | +| hubble.ui.frontend.extraVolumeMounts | list | `[]` | Additional hubble-ui frontend volumeMounts. | +| hubble.ui.frontend.extraVolumes | list | `[]` | Additional hubble-ui frontend volumes. | +| hubble.ui.frontend.image | object | `{"digest":"sha256:7d663dc16538dd6e29061abd1047013a645e6e69c115e008bee9ea9fef9a6666","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui","tag":"v0.13.0","useDigest":true}` | Hubble-ui frontend image. | +| hubble.ui.frontend.resources | object | `{}` | Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. | +| hubble.ui.frontend.securityContext | object | `{}` | Hubble-ui frontend security context. | +| hubble.ui.frontend.server.ipv6 | object | `{"enabled":true}` | Controls server listener for ipv6 | +| hubble.ui.ingress | object | `{"annotations":{},"className":"","enabled":false,"hosts":["chart-example.local"],"labels":{},"tls":[]}` | hubble-ui ingress configuration. | +| hubble.ui.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| hubble.ui.podAnnotations | object | `{}` | Annotations to be added to hubble-ui pods | +| hubble.ui.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | +| hubble.ui.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | +| hubble.ui.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | +| hubble.ui.podLabels | object | `{}` | Labels to be added to hubble-ui pods | +| hubble.ui.priorityClassName | string | `""` | The priority class to use for hubble-ui | +| hubble.ui.replicas | int | `1` | The number of replicas of Hubble UI to deploy. | +| hubble.ui.rollOutPods | bool | `false` | Roll out Hubble-ui pods automatically when configmap is updated. | +| hubble.ui.securityContext | object | `{"fsGroup":1001,"runAsGroup":1001,"runAsUser":1001}` | Security context to be added to Hubble UI pods | +| hubble.ui.service | object | `{"annotations":{},"nodePort":31235,"type":"ClusterIP"}` | hubble-ui service configuration. | +| hubble.ui.service.annotations | object | `{}` | Annotations to be added for the Hubble UI service | +| hubble.ui.service.nodePort | int | `31235` | - The port to use when the service type is set to NodePort. | +| hubble.ui.service.type | string | `"ClusterIP"` | - The type of service used for Hubble UI access, either ClusterIP or NodePort. | +| hubble.ui.standalone.enabled | bool | `false` | When true, it will allow installing the Hubble UI only, without checking dependencies. It is useful if a cluster already has cilium and Hubble relay installed and you just want Hubble UI to be deployed. When installed via helm, installing UI should be done via `helm upgrade` and when installed via the cilium cli, then `cilium hubble enable --ui` | +| hubble.ui.standalone.tls.certsVolume | object | `{}` | When deploying Hubble UI in standalone, with tls enabled for Hubble relay, it is required to provide a volume for mounting the client certificates. | +| hubble.ui.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values used to connect to hubble-relay This keypair is presented to Hubble Relay instances for mTLS authentication and is required when hubble.relay.tls.server.enabled is true. These values need to be set manually if hubble.tls.auto.enabled is false. | +| hubble.ui.tolerations | list | `[]` | Node tolerations for pod assignment on nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| hubble.ui.topologySpreadConstraints | list | `[]` | Pod topology spread constraints for hubble-ui | +| hubble.ui.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | hubble-ui update strategy. | +| identityAllocationMode | string | `"crd"` | Method to use for identity allocation (`crd` or `kvstore`). | +| identityChangeGracePeriod | string | `"5s"` | Time to wait before using new identity on endpoint identity change. | +| image | object | `{"digest":"sha256:da74ab61d1bc665c1c088dff41d5be388d252ca5800f30c7d88844e6b5e440b0","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.15.3","useDigest":true}` | Agent container image. | +| imagePullSecrets | string | `nil` | Configure image pull secrets for pulling container images | +| ingressController.default | bool | `false` | Set cilium ingress controller to be the default ingress controller This will let cilium ingress controller route entries without ingress class set | +| ingressController.defaultSecretName | string | `nil` | Default secret name for ingresses without .spec.tls[].secretName set. | +| ingressController.defaultSecretNamespace | string | `nil` | Default secret namespace for ingresses without .spec.tls[].secretName set. | +| ingressController.enableProxyProtocol | bool | `false` | Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. | +| ingressController.enabled | bool | `false` | Enable cilium ingress controller This will automatically set enable-envoy-config as well. | +| ingressController.enforceHttps | bool | `true` | Enforce https for host having matching TLS host in Ingress. Incoming traffic to http listener will return 308 http error code with respective location in header. | +| ingressController.ingressLBAnnotationPrefixes | list | `["service.beta.kubernetes.io","service.kubernetes.io","cloud.google.com"]` | IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service | +| ingressController.loadbalancerMode | string | `"dedicated"` | Default ingress load balancer mode Supported values: shared, dedicated For granular control, use the following annotations on the ingress resource ingress.cilium.io/loadbalancer-mode: shared|dedicated, | +| ingressController.secretsNamespace | object | `{"create":true,"name":"cilium-secrets","sync":true}` | SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. | +| ingressController.secretsNamespace.create | bool | `true` | Create secrets namespace for Ingress. | +| ingressController.secretsNamespace.name | string | `"cilium-secrets"` | Name of Ingress secret namespace. | +| ingressController.secretsNamespace.sync | bool | `true` | Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. If disabled, TLS secrets must be maintained externally. | +| ingressController.service | object | `{"allocateLoadBalancerNodePorts":null,"annotations":{},"insecureNodePort":null,"labels":{},"loadBalancerClass":null,"loadBalancerIP":null,"name":"cilium-ingress","secureNodePort":null,"type":"LoadBalancer"}` | Load-balancer service in shared mode. This is a single load-balancer service for all Ingress resources. | +| ingressController.service.allocateLoadBalancerNodePorts | string | `nil` | Configure if node port allocation is required for LB service ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation | +| ingressController.service.annotations | object | `{}` | Annotations to be added for the shared LB service | +| ingressController.service.insecureNodePort | string | `nil` | Configure a specific nodePort for insecure HTTP traffic on the shared LB service | +| ingressController.service.labels | object | `{}` | Labels to be added for the shared LB service | +| ingressController.service.loadBalancerClass | string | `nil` | Configure a specific loadBalancerClass on the shared LB service (requires Kubernetes 1.24+) | +| ingressController.service.loadBalancerIP | string | `nil` | Configure a specific loadBalancerIP on the shared LB service | +| ingressController.service.name | string | `"cilium-ingress"` | Service name | +| ingressController.service.secureNodePort | string | `nil` | Configure a specific nodePort for secure HTTPS traffic on the shared LB service | +| ingressController.service.type | string | `"LoadBalancer"` | Service type for the shared LB service | +| initResources | object | `{}` | resources & limits for the agent init containers | +| installNoConntrackIptablesRules | bool | `false` | Install Iptables rules to skip netfilter connection tracking on all pod traffic. This option is only effective when Cilium is running in direct routing and full KPR mode. Moreover, this option cannot be enabled when Cilium is running in a managed Kubernetes environment or in a chained CNI setup. | +| ipMasqAgent | object | `{"enabled":false}` | Configure the eBPF-based ip-masq-agent | +| ipam.ciliumNodeUpdateRate | string | `"15s"` | Maximum rate at which the CiliumNode custom resource is updated. | +| ipam.mode | string | `"cluster-pool"` | Configure IP Address Management mode. ref: https://docs.cilium.io/en/stable/network/concepts/ipam/ | +| ipam.operator.autoCreateCiliumPodIPPools | object | `{}` | IP pools to auto-create in multi-pool IPAM mode. | +| ipam.operator.clusterPoolIPv4MaskSize | int | `24` | IPv4 CIDR mask size to delegate to individual nodes for IPAM. | +| ipam.operator.clusterPoolIPv4PodCIDRList | list | `["10.0.0.0/8"]` | IPv4 CIDR list range to delegate to individual nodes for IPAM. | +| ipam.operator.clusterPoolIPv6MaskSize | int | `120` | IPv6 CIDR mask size to delegate to individual nodes for IPAM. | +| ipam.operator.clusterPoolIPv6PodCIDRList | list | `["fd00::/104"]` | IPv6 CIDR list range to delegate to individual nodes for IPAM. | +| ipam.operator.externalAPILimitBurstSize | string | `20` | The maximum burst size when rate limiting access to external APIs. Also known as the token bucket capacity. | +| ipam.operator.externalAPILimitQPS | string | `4.0` | The maximum queries per second when rate limiting access to external APIs. Also known as the bucket refill rate, which is used to refill the bucket up to the burst size capacity. | +| ipv4.enabled | bool | `true` | Enable IPv4 support. | +| ipv4NativeRoutingCIDR | string | `""` | Allows to explicitly specify the IPv4 CIDR for native routing. When specified, Cilium assumes networking for this CIDR is preconfigured and hands traffic destined for that range to the Linux network stack without applying any SNAT. Generally speaking, specifying a native routing CIDR implies that Cilium can depend on the underlying networking stack to route packets to their destination. To offer a concrete example, if Cilium is configured to use direct routing and the Kubernetes CIDR is included in the native routing CIDR, the user must configure the routes to reach pods, either manually or by setting the auto-direct-node-routes flag. | +| ipv6.enabled | bool | `false` | Enable IPv6 support. | +| ipv6NativeRoutingCIDR | string | `""` | Allows to explicitly specify the IPv6 CIDR for native routing. When specified, Cilium assumes networking for this CIDR is preconfigured and hands traffic destined for that range to the Linux network stack without applying any SNAT. Generally speaking, specifying a native routing CIDR implies that Cilium can depend on the underlying networking stack to route packets to their destination. To offer a concrete example, if Cilium is configured to use direct routing and the Kubernetes CIDR is included in the native routing CIDR, the user must configure the routes to reach pods, either manually or by setting the auto-direct-node-routes flag. | +| k8s | object | `{}` | Configure Kubernetes specific configuration | +| k8sClientRateLimit | object | `{"burst":null,"qps":null}` | Configure the client side rate limit for the agent and operator If the amount of requests to the Kubernetes API server exceeds the configured rate limit, the agent and operator will start to throttle requests by delaying them until there is budget or the request times out. | +| k8sClientRateLimit.burst | int | 10 for k8s up to 1.26. 20 for k8s version 1.27+ | The burst request rate in requests per second. The rate limiter will allow short bursts with a higher rate. | +| k8sClientRateLimit.qps | int | 5 for k8s up to 1.26. 10 for k8s version 1.27+ | The sustained request rate in requests per second. | +| k8sNetworkPolicy.enabled | bool | `true` | Enable support for K8s NetworkPolicy | +| k8sServiceHost | string | `""` | Kubernetes service host | +| k8sServicePort | string | `""` | Kubernetes service port | +| keepDeprecatedLabels | bool | `false` | Keep the deprecated selector labels when deploying Cilium DaemonSet. | +| keepDeprecatedProbes | bool | `false` | Keep the deprecated probes when deploying Cilium DaemonSet | +| kubeConfigPath | string | `"~/.kube/config"` | Kubernetes config path | +| kubeProxyReplacementHealthzBindAddr | string | `""` | healthz server bind address for the kube-proxy replacement. To enable set the value to '0.0.0.0:10256' for all ipv4 addresses and this '[::]:10256' for all ipv6 addresses. By default it is disabled. | +| l2NeighDiscovery.enabled | bool | `true` | Enable L2 neighbor discovery in the agent | +| l2NeighDiscovery.refreshPeriod | string | `"30s"` | Override the agent's default neighbor resolution refresh period. | +| l2announcements | object | `{"enabled":false}` | Configure L2 announcements | +| l2announcements.enabled | bool | `false` | Enable L2 announcements | +| l2podAnnouncements | object | `{"enabled":false,"interface":"eth0"}` | Configure L2 pod announcements | +| l2podAnnouncements.enabled | bool | `false` | Enable L2 pod announcements | +| l2podAnnouncements.interface | string | `"eth0"` | Interface used for sending Gratuitous ARP pod announcements | +| l7Proxy | bool | `true` | Enable Layer 7 network policy. | +| livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | +| livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | +| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | +| loadBalancer.acceleration | string | `"disabled"` | acceleration is the option to accelerate service handling via XDP Applicable values can be: disabled (do not use XDP), native (XDP BPF program is run directly out of the networking driver's early receive path), or best-effort (use native mode XDP acceleration on devices that support it). | +| loadBalancer.l7 | object | `{"algorithm":"round_robin","backend":"disabled","ports":[]}` | L7 LoadBalancer | +| loadBalancer.l7.algorithm | string | `"round_robin"` | Default LB algorithm The default LB algorithm to be used for services, which can be overridden by the service annotation (e.g. service.cilium.io/lb-l7-algorithm) Applicable values: round_robin, least_request, random | +| loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing by way of service annotation. | +| loadBalancer.l7.ports | list | `[]` | List of ports from service to be automatically redirected to above backend. Any service exposing one of these ports will be automatically redirected. Fine-grained control can be achieved by using the service annotation. | +| localRedirectPolicy | bool | `false` | Enable Local Redirect Policy. | +| logSystemLoad | bool | `false` | Enables periodic logging of system load | +| maglev | object | `{}` | Configure maglev consistent hashing | +| monitor | object | `{"enabled":false}` | cilium-monitor sidecar. | +| monitor.enabled | bool | `false` | Enable the cilium-monitor sidecar. | +| name | string | `"cilium"` | Agent container name. | +| nat46x64Gateway | object | `{"enabled":false}` | Configure standalone NAT46/NAT64 gateway | +| nat46x64Gateway.enabled | bool | `false` | Enable RFC8215-prefixed translation | +| nodePort | object | `{"autoProtectPortRange":true,"bindProtection":true,"enableHealthCheck":true,"enableHealthCheckLoadBalancerIP":false,"enabled":false}` | Configure N-S k8s service loadbalancing | +| nodePort.autoProtectPortRange | bool | `true` | Append NodePort range to ip_local_reserved_ports if clash with ephemeral ports is detected. | +| nodePort.bindProtection | bool | `true` | Set to true to prevent applications binding to service ports. | +| nodePort.enableHealthCheck | bool | `true` | Enable healthcheck nodePort server for NodePort services | +| nodePort.enableHealthCheckLoadBalancerIP | bool | `false` | Enable access of the healthcheck nodePort on the LoadBalancerIP. Needs EnableHealthCheck to be enabled | +| nodePort.enabled | bool | `false` | Enable the Cilium NodePort service implementation. | +| nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for cilium-agent. | +| nodeinit.affinity | object | `{}` | Affinity for cilium-nodeinit | +| nodeinit.annotations | object | `{}` | Annotations to be added to all top-level nodeinit objects (resources under templates/cilium-nodeinit) | +| nodeinit.bootstrapFile | string | `"/tmp/cilium-bootstrap.d/cilium-bootstrap-time"` | bootstrapFile is the location of the file where the bootstrap timestamp is written by the node-init DaemonSet | +| nodeinit.enabled | bool | `false` | Enable the node initialization DaemonSet | +| nodeinit.extraEnv | list | `[]` | Additional nodeinit environment variables. | +| nodeinit.extraVolumeMounts | list | `[]` | Additional nodeinit volumeMounts. | +| nodeinit.extraVolumes | list | `[]` | Additional nodeinit volumes. | +| nodeinit.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/startup-script","tag":"62093c5c233ea914bfa26a10ba41f8780d9b737f"}` | node-init image. | +| nodeinit.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for nodeinit pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| nodeinit.podAnnotations | object | `{}` | Annotations to be added to node-init pods. | +| nodeinit.podLabels | object | `{}` | Labels to be added to node-init pods. | +| nodeinit.prestop | object | `{"postScript":"","preScript":""}` | prestop offers way to customize prestop nodeinit script (pre and post position) | +| nodeinit.priorityClassName | string | `""` | The priority class to use for the nodeinit pod. | +| nodeinit.resources | object | `{"requests":{"cpu":"100m","memory":"100Mi"}}` | nodeinit resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| nodeinit.securityContext | object | `{"capabilities":{"add":["SYS_MODULE","NET_ADMIN","SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"]},"privileged":false,"seLinuxOptions":{"level":"s0","type":"spc_t"}}` | Security context to be added to nodeinit pods. | +| nodeinit.startup | object | `{"postScript":"","preScript":""}` | startup offers way to customize startup nodeinit script (pre and post position) | +| nodeinit.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for nodeinit scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| nodeinit.updateStrategy | object | `{"type":"RollingUpdate"}` | node-init update strategy | +| operator.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"io.cilium/app":"operator"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-operator | +| operator.annotations | object | `{}` | Annotations to be added to all top-level cilium-operator objects (resources under templates/cilium-operator) | +| operator.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for cilium-operator grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | +| operator.dnsPolicy | string | `""` | DNS policy for Cilium operator pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | +| operator.enabled | bool | `true` | Enable the cilium-operator component (required). | +| operator.endpointGCInterval | string | `"5m0s"` | Interval for endpoint garbage collection. | +| operator.extraArgs | list | `[]` | Additional cilium-operator container arguments. | +| operator.extraEnv | list | `[]` | Additional cilium-operator environment variables. | +| operator.extraHostPathMounts | list | `[]` | Additional cilium-operator hostPath mounts. | +| operator.extraVolumeMounts | list | `[]` | Additional cilium-operator volumeMounts. | +| operator.extraVolumes | list | `[]` | Additional cilium-operator volumes. | +| operator.identityGCInterval | string | `"15m0s"` | Interval for identity garbage collection. | +| operator.identityHeartbeatTimeout | string | `"30m0s"` | Timeout for identity heartbeats. | +| operator.image | object | `{"alibabacloudDigest":"sha256:59d5c0c5782163d38151dd06bae0118144f6c080598901a632c628b1143ccd10","awsDigest":"sha256:2b05dc6b88037a5ce05e4030ef616b1f7be9e65083e35abd36a1b66953fd0b6a","azureDigest":"sha256:b85a2671a74903c6e9a45e884654bb970b5b8d6a6e20371811a6cc0ad92b2f87","genericDigest":"sha256:c97f23161906b82f5c81a2d825b0646a5aa1dfb4adf1d49cbb87815079e69d61","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.15.3","useDigest":true}` | cilium-operator image. | +| operator.nodeGCInterval | string | `"5m0s"` | Interval for cilium node garbage collection. | +| operator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| operator.podAnnotations | object | `{}` | Annotations to be added to cilium-operator pods | +| operator.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | +| operator.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | +| operator.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | +| operator.podLabels | object | `{}` | Labels to be added to cilium-operator pods | +| operator.podSecurityContext | object | `{}` | Security context to be added to cilium-operator pods | +| operator.pprof.address | string | `"localhost"` | Configure pprof listen address for cilium-operator | +| operator.pprof.enabled | bool | `false` | Enable pprof for cilium-operator | +| operator.pprof.port | int | `6061` | Configure pprof listen port for cilium-operator | +| operator.priorityClassName | string | `""` | The priority class to use for cilium-operator | +| operator.prometheus | object | `{"enabled":true,"port":9963,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":null,"relabelings":null}}` | Enable prometheus metrics for cilium-operator on the configured port at /metrics | +| operator.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-operator | +| operator.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| operator.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| operator.prometheus.serviceMonitor.jobLabel | string | `""` | jobLabel to add for ServiceMonitor cilium-operator | +| operator.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-operator | +| operator.prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-operator | +| operator.prometheus.serviceMonitor.relabelings | string | `nil` | Relabeling configs for the ServiceMonitor cilium-operator | +| operator.removeNodeTaints | bool | `true` | Remove Cilium node taint from Kubernetes nodes that have a healthy Cilium pod running. | +| operator.replicas | int | `2` | Number of replicas to run for the cilium-operator deployment | +| operator.resources | object | `{}` | cilium-operator resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| operator.rollOutPods | bool | `false` | Roll out cilium-operator pods automatically when configmap is updated. | +| operator.securityContext | object | `{}` | Security context to be added to cilium-operator pods | +| operator.setNodeNetworkStatus | bool | `true` | Set Node condition NetworkUnavailable to 'false' with the reason 'CiliumIsUp' for nodes that have a healthy Cilium pod. | +| operator.setNodeTaints | string | same as removeNodeTaints | Taint nodes where Cilium is scheduled but not running. This prevents pods from being scheduled to nodes where Cilium is not the default CNI provider. | +| operator.skipCNPStatusStartupClean | bool | `false` | Skip CNP node status clean up at operator startup. | +| operator.skipCRDCreation | bool | `false` | Skip CRDs creation for cilium-operator | +| operator.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for cilium-operator scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| operator.topologySpreadConstraints | list | `[]` | Pod topology spread constraints for cilium-operator | +| operator.unmanagedPodWatcher.intervalSeconds | int | `15` | Interval, in seconds, to check if there are any pods that are not managed by Cilium. | +| operator.unmanagedPodWatcher.restart | bool | `true` | Restart any pod that are not managed by Cilium. | +| operator.updateStrategy | object | `{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"50%"},"type":"RollingUpdate"}` | cilium-operator update strategy | +| pmtuDiscovery.enabled | bool | `false` | Enable path MTU discovery to send ICMP fragmentation-needed replies to the client. | +| podAnnotations | object | `{}` | Annotations to be added to agent pods | +| podLabels | object | `{}` | Labels to be added to agent pods | +| podSecurityContext | object | `{}` | Security Context for cilium-agent pods. | +| policyCIDRMatchMode | string | `nil` | policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. The possible value is "nodes". | +| policyEnforcementMode | string | `"default"` | The agent can be put into one of the three policy enforcement modes: default, always and never. ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes | +| pprof.address | string | `"localhost"` | Configure pprof listen address for cilium-agent | +| pprof.enabled | bool | `false` | Enable pprof for cilium-agent | +| pprof.port | int | `6060` | Configure pprof listen port for cilium-agent | +| preflight.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-preflight | +| preflight.annotations | object | `{}` | Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) | +| preflight.enabled | bool | `false` | Enable Cilium pre-flight resources (required for upgrade) | +| preflight.extraEnv | list | `[]` | Additional preflight environment variables. | +| preflight.extraVolumeMounts | list | `[]` | Additional preflight volumeMounts. | +| preflight.extraVolumes | list | `[]` | Additional preflight volumes. | +| preflight.image | object | `{"digest":"sha256:da74ab61d1bc665c1c088dff41d5be388d252ca5800f30c7d88844e6b5e440b0","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.15.3","useDigest":true}` | Cilium pre-flight image. | +| preflight.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for preflight pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| preflight.podAnnotations | object | `{}` | Annotations to be added to preflight pods | +| preflight.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | +| preflight.podDisruptionBudget.maxUnavailable | int | `1` | Maximum number/percentage of pods that may be made unavailable | +| preflight.podDisruptionBudget.minAvailable | string | `nil` | Minimum number/percentage of pods that should remain scheduled. When it's set, maxUnavailable must be disabled by `maxUnavailable: null` | +| preflight.podLabels | object | `{}` | Labels to be added to the preflight pod. | +| preflight.podSecurityContext | object | `{}` | Security context to be added to preflight pods. | +| preflight.priorityClassName | string | `""` | The priority class to use for the preflight pod. | +| preflight.resources | object | `{}` | preflight resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| preflight.securityContext | object | `{}` | Security context to be added to preflight pods | +| preflight.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for preflight Deployment and DaemonSet. | +| preflight.tofqdnsPreCache | string | `""` | Path to write the `--tofqdns-pre-cache` file to. | +| preflight.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/not-ready"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"},{"key":"CriticalAddonsOnly","operator":"Exists"}]` | Node tolerations for preflight scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| preflight.updateStrategy | object | `{"type":"RollingUpdate"}` | preflight update strategy | +| preflight.validateCNPs | bool | `true` | By default we should always validate the installed CNPs before upgrading Cilium. This will make sure the user will have the policies deployed in the cluster with the right schema. | +| priorityClassName | string | `""` | The priority class to use for cilium-agent. | +| prometheus | object | `{"controllerGroupMetrics":["write-cni-file","sync-host-ips","sync-lb-maps-with-k8s-services"],"enabled":false,"metrics":null,"port":9962,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}],"trustCRDsExist":false}}` | Configure prometheus metrics on the configured port at /metrics | +| prometheus.controllerGroupMetrics | list | `["write-cni-file","sync-host-ips","sync-lb-maps-with-k8s-services"]` | - Enable controller group metrics for monitoring specific Cilium subsystems. The list is a list of controller group names. The special values of "all" and "none" are supported. The set of controller group names is not guaranteed to be stable between Cilium versions. | +| prometheus.metrics | string | `nil` | Metrics that should be enabled or disabled from the default metric list. The list is expected to be separated by a space. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar). ref: https://docs.cilium.io/en/stable/observability/metrics/ | +| prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-agent | +| prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| prometheus.serviceMonitor.jobLabel | string | `""` | jobLabel to add for ServiceMonitor cilium-agent | +| prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-agent | +| prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-agent | +| prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-agent | +| prometheus.serviceMonitor.trustCRDsExist | bool | `false` | Set to `true` and helm will not check for monitoring.coreos.com/v1 CRDs before deploying | +| proxy | object | `{"prometheus":{"enabled":true,"port":null},"sidecarImageRegex":"cilium/istio_proxy"}` | Configure Istio proxy options. | +| proxy.prometheus.enabled | bool | `true` | Deprecated in favor of envoy.prometheus.enabled | +| proxy.prometheus.port | string | `nil` | Deprecated in favor of envoy.prometheus.port | +| proxy.sidecarImageRegex | string | `"cilium/istio_proxy"` | Regular expression matching compatible Istio sidecar istio-proxy container image names | +| rbac.create | bool | `true` | Enable creation of Resource-Based Access Control configuration. | +| readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | +| readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | +| remoteNodeIdentity | bool | `true` | Enable use of the remote node identity. ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity Deprecated without replacement in 1.15. To be removed in 1.16. | +| resourceQuotas | object | `{"cilium":{"hard":{"pods":"10k"}},"enabled":false,"operator":{"hard":{"pods":"15"}}}` | Enable resource quotas for priority classes used in the cluster. | +| resources | object | `{}` | Agent resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| rollOutCiliumPods | bool | `false` | Roll out cilium agent pods automatically when configmap is updated. | +| routingMode | string | `"tunnel"` | Enable native-routing mode or tunneling mode. Possible values: - "" - native - tunnel | +| sctp | object | `{"enabled":false}` | SCTP Configuration Values | +| sctp.enabled | bool | `false` | Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. | +| securityContext.capabilities.applySysctlOverwrites | list | `["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"]` | capabilities for the `apply-sysctl-overwrites` init container | +| securityContext.capabilities.ciliumAgent | list | `["CHOWN","KILL","NET_ADMIN","NET_RAW","IPC_LOCK","SYS_MODULE","SYS_ADMIN","SYS_RESOURCE","DAC_OVERRIDE","FOWNER","SETGID","SETUID"]` | Capabilities for the `cilium-agent` container | +| securityContext.capabilities.cleanCiliumState | list | `["NET_ADMIN","SYS_MODULE","SYS_ADMIN","SYS_RESOURCE"]` | Capabilities for the `clean-cilium-state` init container | +| securityContext.capabilities.mountCgroup | list | `["SYS_ADMIN","SYS_CHROOT","SYS_PTRACE"]` | Capabilities for the `mount-cgroup` init container | +| securityContext.privileged | bool | `false` | Run the pod with elevated privileges | +| securityContext.seLinuxOptions | object | `{"level":"s0","type":"spc_t"}` | SELinux options for the `cilium-agent` and init containers | +| serviceAccounts | object | Component's fully qualified name. | Define serviceAccount names for components. | +| serviceAccounts.clustermeshcertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"clustermesh-apiserver-generate-certs"}` | Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob | +| serviceAccounts.hubblecertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"hubble-generate-certs"}` | Hubblecertgen is used if hubble.tls.auto.method=cronJob | +| serviceAccounts.nodeinit.enabled | bool | `false` | Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by this issue. Name and automount can be configured, if enabled is set to true. Otherwise, they are ignored. Enabled can be removed once the issue is fixed. Cilium-nodeinit DS must also be fixed. | +| serviceNoBackendResponse | string | `"reject"` | Configure what the response should be to traffic for a service without backends. "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". Possible values: - reject (default) - drop | +| sleepAfterInit | bool | `false` | Do not run Cilium agent when running with clean mode. Useful to completely uninstall Cilium as it will stop Cilium from starting and create artifacts in the node. | +| socketLB | object | `{"enabled":false}` | Configure socket LB | +| socketLB.enabled | bool | `false` | Enable socket LB | +| startupProbe.failureThreshold | int | `105` | failure threshold of startup probe. 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) | +| startupProbe.periodSeconds | int | `2` | interval between checks of the startup probe | +| svcSourceRangeCheck | bool | `true` | Enable check of service source ranges (currently, only for LoadBalancer). | +| synchronizeK8sNodes | bool | `true` | Synchronize Kubernetes nodes to kvstore and perform CNP GC. | +| terminationGracePeriodSeconds | int | `1` | Configure termination grace period for cilium-agent DaemonSet. | +| tls | object | `{"ca":{"cert":"","certValidityDuration":1095,"key":""},"caBundle":{"enabled":false,"key":"ca.crt","name":"cilium-root-ca.crt","useSecret":false},"secretsBackend":"local"}` | Configure TLS configuration in the agent. | +| tls.ca | object | `{"cert":"","certValidityDuration":1095,"key":""}` | Base64 encoded PEM values for the CA certificate and private key. This can be used as common CA to generate certificates used by hubble and clustermesh components. It is neither required nor used when cert-manager is used to generate the certificates. | +| tls.ca.cert | string | `""` | Optional CA cert. If it is provided, it will be used by cilium to generate all other certificates. Otherwise, an ephemeral CA is generated. | +| tls.ca.certValidityDuration | int | `1095` | Generated certificates validity duration in days. This will be used for auto generated CA. | +| tls.ca.key | string | `""` | Optional CA private key. If it is provided, it will be used by cilium to generate all other certificates. Otherwise, an ephemeral CA is generated. | +| tls.caBundle | object | `{"enabled":false,"key":"ca.crt","name":"cilium-root-ca.crt","useSecret":false}` | Configure the CA trust bundle used for the validation of the certificates leveraged by hubble and clustermesh. When enabled, it overrides the content of the 'ca.crt' field of the respective certificates, allowing for CA rotation with no down-time. | +| tls.caBundle.enabled | bool | `false` | Enable the use of the CA trust bundle. | +| tls.caBundle.key | string | `"ca.crt"` | Entry of the ConfigMap containing the CA trust bundle. | +| tls.caBundle.name | string | `"cilium-root-ca.crt"` | Name of the ConfigMap containing the CA trust bundle. | +| tls.caBundle.useSecret | bool | `false` | Use a Secret instead of a ConfigMap. | +| tls.secretsBackend | string | `"local"` | This configures how the Cilium agent loads the secrets used TLS-aware CiliumNetworkPolicies (namely the secrets referenced by terminatingTLS and originatingTLS). Possible values: - local - k8s | +| tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for agent scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| tunnelPort | int | Port 8472 for VXLAN, Port 6081 for Geneve | Configure VXLAN and Geneve tunnel port. | +| tunnelProtocol | string | `"vxlan"` | Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. Possible values: - "" - vxlan - geneve | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | Cilium agent update strategy | +| vtep.cidr | string | `""` | A space separated list of VTEP device CIDRs, for example "1.1.1.0/24 1.1.2.0/24" | +| vtep.enabled | bool | `false` | Enables VXLAN Tunnel Endpoint (VTEP) Integration (beta) to allow Cilium-managed pods to talk to third party VTEP devices over Cilium tunnel. | +| vtep.endpoint | string | `""` | A space separated list of VTEP device endpoint IPs, for example "1.1.1.1 1.1.2.1" | +| vtep.mac | string | `""` | A space separated list of VTEP device MAC addresses (VTEP MAC), for example "x:x:x:x:x:x y:y:y:y:y:y:y" | +| vtep.mask | string | `""` | VTEP CIDRs Mask that applies to all VTEP CIDRs, for example "255.255.255.0" | +| waitForKubeProxy | bool | `false` | Wait for KUBE-PROXY-CANARY iptables rule to appear in "wait-for-kube-proxy" init container before launching cilium-agent. More context can be found in the commit message of below PR https://github.com/cilium/cilium/pull/20123 | +| wellKnownIdentities.enabled | bool | `false` | Enable the use of well-known identities. | diff --git a/magnum_cluster_api/charts/cilium/README.md.gotmpl b/magnum_cluster_api/charts/cilium/README.md.gotmpl new file mode 100644 index 00000000..db2d81b7 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/README.md.gotmpl @@ -0,0 +1,54 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +Cilium is open source software for providing and transparently securing +network connectivity and loadbalancing between application workloads such as +application containers or processes. Cilium operates at Layer 3/4 to provide +traditional networking and security services as well as Layer 7 to protect and +secure use of modern application protocols such as HTTP, gRPC and Kafka. + +A new Linux kernel technology called eBPF is at the foundation of Cilium. +It supports dynamic insertion of eBPF bytecode into the Linux kernel at various +integration points such as: network IO, application sockets, and tracepoints +to implement security, networking and visibility logic. eBPF is highly +efficient and flexible. + +![Cilium feature overview](https://raw.githubusercontent.com/cilium/cilium/main/Documentation/images/cilium_overview.png) + +## Prerequisites + +* Kubernetes: `{{ template "chart.kubeVersion" . }}` +* Helm: `>= 3.0` + +## Getting Started + +Try Cilium on any Kubernetes distribution in under 15 minutes: + +| Minikube | Self-Managed K8s | Amazon EKS | Google GKE | Microsoft AKS | +|:-:|:-:|:-:|:-:|:-:| +| [![Minikube](https://raw.githubusercontent.com/cilium/charts/master/images/minikube.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Self-Managed Kubernetes](https://raw.githubusercontent.com/cilium/charts/master/images/k8s.png)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Amazon EKS](https://raw.githubusercontent.com/cilium/charts/master/images/aws.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Google GKE](https://raw.githubusercontent.com/cilium/charts/master/images/google-cloud.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | [![Microsoft AKS](https://raw.githubusercontent.com/cilium/charts/master/images/azure.svg)](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) | + +Or, for a quick install with the default configuration: + +``` +$ helm repo add cilium https://helm.cilium.io/ +$ helm install cilium cilium/cilium --namespace=kube-system +``` + +After Cilium is installed, you can explore the features that Cilium has to +offer from the [Getting Started Guides page](https://docs.cilium.io/en/stable/gettingstarted/). + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +## Getting Help + +The best way to get help if you get stuck is to ask a question on the +[Cilium Slack channel](https://cilium.herokuapp.com/). With Cilium +contributors across the globe, there is almost always someone available to help. + +{{ template "chart.valuesSection" . }} diff --git a/magnum_cluster_api/charts/cilium/files/agent/poststart-eni.bash b/magnum_cluster_api/charts/cilium/files/agent/poststart-eni.bash new file mode 100644 index 00000000..a57d8968 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/agent/poststart-eni.bash @@ -0,0 +1,19 @@ +set -o errexit +set -o pipefail +set -o nounset + +# When running in AWS ENI mode, it's likely that 'aws-node' has +# had a chance to install SNAT iptables rules. These can result +# in dropped traffic, so we should attempt to remove them. +# We do it using a 'postStart' hook since this may need to run +# for nodes which might have already been init'ed but may still +# have dangling rules. This is safe because there are no +# dependencies on anything that is part of the startup script +# itself, and can be safely run multiple times per node (e.g. in +# case of a restart). +if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; +then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore +fi +echo 'Done!' diff --git a/magnum_cluster_api/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json b/magnum_cluster_api/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json new file mode 100644 index 00000000..94af2eac --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json @@ -0,0 +1,8479 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Cilium (https://cilium.io/) metrics", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "iteration": 1606309591568, + "links": [], + "panels": [ + { + "aliasColors": { + "error": "#890f02", + "warning": "#c15c17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "error", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_errors_warnings_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, level) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{level}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors & Warnings", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#cffaff" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(irate(cilium_process_cpu_seconds_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(irate(cilium_process_cpu_seconds_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(irate(cilium_process_cpu_seconds_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 161, + "panels": [], + "title": "Generic", + "type": "row" + }, + { + "aliasColors": { + "AVG_virtual_memory_bytes": "#508642", + "Average Virtual Memory": "#f9d9f9", + "MAX_virtual_memory_bytes": "#e5ac0e", + "Max Virtual Memory": "#584477" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max Virtual Memory", + "fillBelowTo": "Min Virtual Memory", + "lines": false + }, + { + "alias": "Min Virtual Memory", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_process_virtual_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Min Virtual Memory", + "refId": "A" + }, + { + "expr": "avg(cilium_process_virtual_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Average Virtual Memory", + "refId": "B" + }, + { + "expr": "max(cilium_process_virtual_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max Virtual Memory", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Memory Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "MAX_resident_memory_bytes_max": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 6 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_process_resident_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "AVG_resident_memory_bytes", + "refId": "C" + }, + { + "expr": "max(cilium_process_resident_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "MAX_resident_memory_bytes_max", + "refId": "D" + }, + { + "expr": "min(cilium_process_resident_memory_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MIN_resident_memory_bytes_min", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Resident memory status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "all nodes": "#e5a8e2" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 6 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "all nodes", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "all nodes", + "refId": "A" + }, + { + "expr": "min(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min/node", + "refId": "B" + }, + { + "expr": "avg(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg/node", + "refId": "C" + }, + { + "expr": "max(cilium_process_open_fds{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max/node", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Open file descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "MAX_resident_memory_bytes_max": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "BPF memory usage in the entire system including components not managed by Cilium.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 178, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_bpf_maps_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"} + cilium_bpf_progs_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "AVG_bpf_memory_bytes_avg", + "refId": "C" + }, + { + "expr": "max(cilium_bpf_maps_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"} + cilium_bpf_progs_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MAX_bpf_memory_bytes_max", + "refId": "D" + }, + { + "expr": "min(cilium_bpf_maps_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"} + cilium_bpf_progs_virtual_memory_max_bytes{k8s_app=\"cilium\", pod=~\"$pod\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MIN_bpf_memory_bytes_min", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System-wide BPF memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:136", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:137", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Fill percentage of BPF maps, tagged by map name", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 194, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cilium_bpf_map_pressure{k8s_app=\"cilium\", pod=~\"$pod\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "BPF map pressure", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:230", + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.0", + "min": null, + "show": true + }, + { + "$$hashKey": "object:231", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 155, + "panels": [], + "title": "API", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 152, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_agent_api_process_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API call latency (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 153, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_agent_api_process_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API call latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 156, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# API calls (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# API calls (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 159, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path, return_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{return_code}} ({{method}} {{path}} )", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API return codes (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_agent_api_process_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path, return_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{return_code}} ({{method}} {{path}} )", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API return codes (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 72, + "panels": [], + "title": "Cilium", + "type": "row" + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 144, + "links": [], + "mode": "markdown", + "title": "BPF", + "type": "text" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# system calls (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "hiddenSeries": false, + "id": 145, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# system calls (max node)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 46 + }, + "hiddenSeries": false, + "id": 140, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_bpf_syscall_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/ rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "system call latency (avg node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 46 + }, + "hiddenSeries": false, + "id": 148, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_bpf_syscall_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/ rate(cilium_bpf_syscall_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "system call latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 142, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app=\"cilium\", pod=~\"$pod\"}[5m])) by (pod, map_name, operation))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{map_name}} {{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "map ops (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 52 + }, + "hiddenSeries": false, + "id": 147, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, max(rate(cilium_bpf_map_ops_total{k8s_app=\"cilium\", pod=~\"$pod\"}[5m])) by (pod, map_name, operation))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{map_name}} {{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "map ops (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 52 + }, + "hiddenSeries": false, + "id": 143, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_bpf_map_ops_total{k8s_app=\"cilium\",outcome=\"fail\", pod=~\"$pod\"}[5m])) by (pod, map_name, operation)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{map_name}} {{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "map ops (sum failures)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 182, + "links": [], + "mode": "markdown", + "title": "kvstore", + "type": "text" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 59 + }, + "hiddenSeries": false, + "id": 184, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kvstore_operations_total{pod=~\"$pod\"}[1m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}} {{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# operations (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 59 + }, + "hiddenSeries": false, + "id": 186, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(kvstore_operations_total{pod=~\"$pod\"}[1m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}} {{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# operations (max node)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 188, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, avg(rate(cilium_kvstore_operations_duration_seconds_sum{pod=~\"$pod\"}[1m])) by (pod, action, scope) / avg(rate(cilium_kvstore_operations_duration_seconds_count{pod=~\"$pod\"}[1m])) by (pod, action, scope))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "latency (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 64 + }, + "hiddenSeries": false, + "id": 190, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, max(rate(cilium_kvstore_operations_duration_seconds_sum{pod=~\"$pod\"}[1m])) by (pod, action, scope) / avg(rate(cilium_kvstore_operations_duration_seconds_count{pod=~\"$pod\"}[1m])) by (pod, action, scope))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 192, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kvstore_events_queue_seconds_count{pod=~\"$pod\"}[1m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events received (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 47, + "links": [], + "mode": "markdown", + "title": "Cilium network information", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_forward_count_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, direction)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Forwarded Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 76 + }, + "hiddenSeries": false, + "id": 111, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "EGRESS", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_forward_bytes_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, direction) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Forwarded Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 82 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv4 Conntrack TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 82 + }, + "hiddenSeries": false, + "id": 128, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv6 Conntrack TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 88 + }, + "hiddenSeries": false, + "id": 129, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv4\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv4 Conntrack Non-TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Alive ipv4": "#0a50a1", + "Alive ipv4 non-TCP": "#f9d9f9", + "Alive ipv6": "#614d93", + "Alive ipv6 TCP": "#806eb7", + "Alive ipv6 non-TCP": "#614d93", + "Alive CT entries ipv6": "#badff4", + "Deleted CT entries ipv4": "#bf1b00", + "Deleted ipv4": "#890f02", + "Deleted ipv4 non-TCP": "#890f02", + "Deleted ipv6": "#bf1b00", + "L7 denied request": "#890f02", + "L7 forwarded request": "#7eb26d", + "avg": "#e0f9d7", + "deleted": "#6ed0e0", + "deleted max": "#447ebc", + "max": "#629e51", + "min": "#629e51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 88 + }, + "hiddenSeries": false, + "id": 130, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "deleted", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "deleted max", + "yaxis": 2 + }, + { + "alias": "deleted min", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"alive\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "avg(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted", + "refId": "D" + }, + { + "expr": "max(cilium_datapath_conntrack_gc_entries{k8s_app=\"cilium\", status=\"deleted\", family=\"ipv6\", protocol=\"non-TCP\", pod=~\"$pod\"}) by (family,status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deleted max", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IPv6 Conntrack Non-TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "ipv4": "#5195ce", + "ipv6": "#6d1f62" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 87, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_ip_addresses{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod, family)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{family}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocated Addresses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "dump_interrupts conntrack ipv4": "#ea6460", + "dump_interrupts conntrack ipv6": "#58140c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 94 + }, + "hiddenSeries": false, + "id": 79, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_datapath_conntrack_dump_resets_total{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod, area, family, name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} {{area}} {{family}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Datapath Conntrack Dump Resets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 99 + }, + "hiddenSeries": false, + "id": 106, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_services_events_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Updates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 99 + }, + "hiddenSeries": false, + "id": 89, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "avg(cilium_unreachable_health_endpoints) by (pod)", + "yaxis": 2 + }, + { + "alias": "average unreachable health endpoints", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_unreachable_nodes{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "unreachable nodes", + "refId": "A" + }, + { + "expr": "sum(cilium_unreachable_health_endpoints{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "unreachable health endpoints", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connectivity Health", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 104 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_count_total{direction=\"EGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dropped Egress Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Avg": "#cca300", + "Max": "rgb(167, 150, 111)" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 104 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max", + "fillBelowTo": "Min", + "lines": false + }, + { + "alias": "Min", + "lines": false + }, + { + "alias": "add k8s", + "yaxis": 2 + }, + { + "alias": "delete k8s", + "yaxis": 2 + }, + { + "alias": "update k8s", + "yaxis": 2 + }, + { + "alias": "add local-node", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_nodes_all_events_received_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, event_type, source) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{eventType}} {{source}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 109 + }, + "hiddenSeries": false, + "id": 113, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_bytes_total{direction=\"EGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (reason) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dropped Egress Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Average Nodes": "#eab839", + "Max Nodes": "#c15c17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 109 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max Nodes", + "fillBelowTo": "Min Nodes", + "lines": false + }, + { + "alias": "Min Nodes", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_nodes_all_num{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Average Nodes", + "refId": "A" + }, + { + "expr": "min(cilium_nodes_all_num{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Min Nodes", + "refId": "B" + }, + { + "expr": "max(cilium_nodes_all_num{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max Nodes", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Nodes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 114 + }, + "id": 28, + "links": [], + "mode": "markdown", + "title": "Policy", + "type": "text" + }, + { + "aliasColors": { + "L7 denied request": "#ea6460", + "L7 forwarded request": "#7eb26d", + "denied": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 115 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "L7 denied request", + "yaxis": 2 + }, + { + "alias": "denied", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_policy_l7_denied_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "denied", + "refId": "A" + }, + { + "expr": "sum(rate(cilium_policy_l7_forwarded_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "forwarded", + "refId": "B" + }, + { + "expr": "sum(rate(cilium_policy_l7_received_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "L7 forwarded request", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 115 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_count_total{direction=\"INGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[5m])) by (reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cilium drops Ingress", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max per node processingTime": "#e24d42", + "Max per node upstreamTime": "#58140c", + "avg(cilium_policy_l7_parse_errors_total{pod=~\"cilium.*\"})": "#bf1b00", + "parse errors": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 120 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max per node processingTime", + "yaxis": 2 + }, + { + "alias": "Max per node upstreamTime", + "yaxis": 2 + }, + { + "alias": "avg(cilium_policy_l7_parse_errors_total{pod=~\"cilium.*\"})", + "yaxis": 2 + }, + { + "alias": "parse errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_proxy_upstream_reply_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_proxy_upstream_reply_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "A" + }, + { + "expr": "avg(cilium_policy_l7_parse_errors_total{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "parse errors", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy response time (Avg)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 120 + }, + "hiddenSeries": false, + "id": 114, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_drop_bytes_total{direction=\"INGRESS\", k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (reason) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dropped Ingress Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#64b0c8", + "count": "#9ac48a", + "max": "#5195ce", + "min": "#6ed0e0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 125 + }, + "hiddenSeries": false, + "id": 104, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "avg count", + "yaxis": 2 + }, + { + "alias": "max count", + "yaxis": 2 + }, + { + "alias": "avg count" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(rate(cilium_triggers_policy_update_call_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_triggers_policy_update_call_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(rate(cilium_triggers_policy_update_call_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_triggers_policy_update_call_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(rate(cilium_triggers_policy_update_call_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_triggers_policy_update_call_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policy Trigger Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max per node processingTime": "#e24d42", + "Max per node upstreamTime": "#58140c", + "parse errors": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 125 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "parse errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_proxy_upstream_reply_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope) / sum(rate(cilium_proxy_upstream_reply_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max {{scope}}", + "refId": "B" + }, + { + "expr": "max(rate(cilium_policy_l7_parse_errors_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "parse errors", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy response time (Max)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "both": "#7eb26d", + "egress": "#e5ac0e", + "ingress": "#e0752d", + "none": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 130 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_policy_endpoint_enforcement_status{k8s_app=\"cilium\", pod=~\"$pod\"}) by (enforcement)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1s", + "intervalFactor": 1, + "legendFormat": "{{enforcement}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoints policy enforcement status", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#b7dbab", + "max": "rgba(89, 132, 76, 0.54)", + "min": "#2f575e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 130 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_proxy_redirects{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_proxy_redirects{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_proxy_redirects{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy Redirects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "average duration": "#d683ce", + "folds": "#614d93", + "max duration": "#614d93", + "max trigger": "#967302", + "min duration": "#584477", + "min trigger": "#fceaca" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 130 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min trigger", + "lines": false + }, + { + "alias": "min trigger", + "lines": false + }, + { + "alias": "folds", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(rate(cilium_triggers_policy_update_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min trigger", + "refId": "A" + }, + { + "expr": "avg(rate(cilium_triggers_policy_update_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "average trigger", + "refId": "B" + }, + { + "expr": "max(rate(cilium_triggers_policy_update_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max trigger", + "refId": "C" + }, + { + "expr": "max(rate(cilium_triggers_policy_update_folds{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "folds", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policy Trigger Runs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#f2c96d", + "policy errors": "#bf1b00", + "policy change errors": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 135 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "policy errors", + "yaxis": 2 + }, + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + }, + { + "alias": "policy change errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_policy{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_policy{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_policy{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + }, + { + "expr": "sum(cilium_policy_change_total{k8s_app=\"cilium\", pod=~\"$pod\", outcome=\"fail\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "policy change errors", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policies Per Node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max per node processingTime": "#e24d42", + "Max per node upstreamTime": "#58140c", + "parse errors": "#bf1b00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 135 + }, + "hiddenSeries": false, + "id": 123, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "parse errors", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_proxy_upstream_reply_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS proxy requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg": "#f9d9f9", + "max": "#806eb7", + "min": "#806eb7" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 140 + }, + "hiddenSeries": false, + "id": 117, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(cilium_policy_max_revision{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(cilium_policy_max_revision{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(cilium_policy_max_revision{k8s_app=\"cilium\", pod=~\"$pod\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Policy Revision", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 145 + }, + "id": 73, + "links": [], + "mode": "markdown", + "title": "Endpoints", + "type": "text" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 146 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(histogram_quantile(0.90, rate(cilium_endpoint_regeneration_time_stats_seconds_bucket{k8s_app=\"cilium\", scope!=\"total\", pod=~\"$pod\"}[5m]))) by (scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoint regeneration time (90th percentile)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 146 + }, + "hiddenSeries": false, + "id": 115, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(histogram_quantile(0.99, rate(cilium_endpoint_regeneration_time_stats_seconds_bucket{k8s_app=\"cilium\", scope!=\"total\", pod=~\"$pod\"}[5m]))) by (scope)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoint regeneration time (99th percentile)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fail": "#bf1b00", + "fail/min": "#890f02", + "success": "#447ebc", + "success/min": "#3f6833" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 155 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fail", + "yaxis": 2 + }, + { + "alias": "success" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_endpoint_regenerations_total{k8s_app=\"cilium\", pod=~\"$pod\"}[30s])) by(outcome)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{outcome}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoint regenerations", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "disconnecting": "#614d93", + "ready": "rgba(81, 220, 95, 0.52)", + "waiting-to-regenerate": "#0a50a1" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 155 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cilium_endpoint_state{k8s_app=\"cilium\", pod=~\"$pod\"}) by (endpoint_state)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{endpoint_state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cilium endpoint state", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 160 + }, + "id": 74, + "links": [], + "mode": "markdown", + "title": "Controllers", + "type": "text" + }, + { + "aliasColors": { + "Failed": "#bf1b00", + "Failing": "#890f02", + "Runs": "#5195ce" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 161 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Failing", + "yaxis": 1 + }, + { + "alias": "Failed", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_controllers_runs_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Runs", + "refId": "A" + }, + { + "expr": "sum(cilium_controllers_failing{k8s_app=\"cilium\", pod=~\"$pod\"}) by(pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controllers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "duration failure": "#890f02", + "duration success": "#508642", + "failure": "#890f02", + "runs failure": "#890f02", + "runs success": "#7eb26d", + "success": "#508642" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 161 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "success", + "yaxis": 1 + }, + { + "alias": "failure", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_controllers_runs_duration_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, status) / sum(rate(cilium_controllers_runs_duration_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 166 + }, + "id": 60, + "links": [], + "mode": "markdown", + "title": "Kubernetes integration", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 167 + }, + "hiddenSeries": false, + "id": 163, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_k8s_client_api_latency_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_k8s_client_api_latency_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver latency (average node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 167 + }, + "hiddenSeries": false, + "id": 165, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(cilium_k8s_client_api_latency_time_seconds_sum{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])/rate(cilium_k8s_client_api_latency_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver latency (max node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 174 + }, + "hiddenSeries": false, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_k8s_client_api_latency_time_seconds_count{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, path)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver #calls (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 174 + }, + "hiddenSeries": false, + "id": 166, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_k8s_client_api_calls_total{k8s_app=\"cilium\", pod=~\"$pod\"}[1m])) by (pod, method, return_code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{return_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "apiserver calls (sum all nodes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 182 + }, + "hiddenSeries": false, + "id": 172, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"true\", valid=\"true\", pod=~\"$pod\"}[5m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Valid, Unnecessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 182 + }, + "hiddenSeries": false, + "id": 174, + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"true\", valid=\"false\", pod=~\"$pod\"}[5m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Invalid, Unnecessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 188 + }, + "hiddenSeries": false, + "id": 175, + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"false\", valid=\"true\", pod=~\"$pod\"}[5m])) by (pod, scope, action, valid)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Valid, Necessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 188 + }, + "hiddenSeries": false, + "id": 173, + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cilium_kubernetes_events_received_total{k8s_app=\"cilium\", equal=\"false\", valid=\"false\", pod=~\"$pod\"}[5m])) by (pod, scope, action)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} {{scope}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Invalid, Necessary K8s Events Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 196 + }, + "hiddenSeries": false, + "id": 108, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"CiliumNetworkPolicy\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CiliumNetworkPolicy Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "create avg": "#70dbed", + "delete avg": "#e24d42", + "update avg": "#e0f9d7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 196 + }, + "hiddenSeries": false, + "id": 119, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"NetworkPolicy\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "NetworkPolicy Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "create avg": "#70dbed", + "delete avg": "#e24d42", + "update avg": "#e0f9d7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 203 + }, + "hiddenSeries": false, + "id": 109, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Pod\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "create avg": "#70dbed", + "delete avg": "#e24d42", + "update avg": "#e0f9d7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 203 + }, + "hiddenSeries": false, + "id": 122, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Node\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} avg", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Node Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 210 + }, + "hiddenSeries": false, + "id": 118, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Service\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 210 + }, + "hiddenSeries": false, + "id": 120, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Endpoint\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Endpoints Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 217 + }, + "hiddenSeries": false, + "id": 121, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_kubernetes_events_total{k8s_app=\"cilium\", scope=\"Namespace\", pod=~\"$pod\"}[1m])) by (pod, action) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Namespace Events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 25, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "cilium.*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(cilium_version, pod)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(cilium_version, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Cilium Metrics", + "uid": "vtuWtdumz", + "version": 1 +} diff --git a/magnum_cluster_api/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json b/magnum_cluster_api/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json new file mode 100644 index 00000000..87939f69 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json @@ -0,0 +1,331 @@ +{ + "node": { + "id": "host~127.0.0.1~no-id~localdomain", + "cluster": "ingress-cluster" + }, + "staticResources": { + "listeners": [ + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} + { + "name": "envoy-prometheus-metrics-listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-prometheus-metrics-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "prometheus_metrics_route", + "domains": [ + "*" + ], + "routes": [ + { + "name": "prometheus_metrics_route", + "match": { + "prefix": "/metrics" + }, + "route": { + "cluster": "/envoy-admin", + "prefix_rewrite": "/stats/prometheus" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "stream_idle_timeout": "0s" + } + } + ] + } + ] + }, + {{- end }} + { + "name": "envoy-health-listener", + "address": { + "socket_address": { + "address": {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }}, + "port_value": {{ .Values.envoy.healthPort }} + } + }, + {{- if and .Values.ipv4.enabled .Values.ipv6.enabled }} + "additional_addresses": [ + { + "address": { + "socket_address": { + "address": "::1", + "port_value": {{ .Values.envoy.healthPort }} + } + } + } + ], + {{- end }} + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-health-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "health", + "domains": [ + "*" + ], + "routes": [ + { + "name": "health", + "match": { + "prefix": "/healthz" + }, + "route": { + "cluster": "/envoy-admin", + "prefix_rewrite": "/ready" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "stream_idle_timeout": "0s" + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "ingress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + }, + { + "name": "egress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "egress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + }, + { + "name": "ingress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "{{ .Values.envoy.idleTimeoutDurationSeconds }}s", + "maxConnectionDuration": "{{ .Values.envoy.maxConnectionDurationSeconds }}s", + "maxRequestsPerConnection": {{ .Values.envoy.maxRequestsPerConnection }} + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "{{ .Values.envoy.connectTimeoutSeconds }}.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "xds-grpc-cilium", + "type": "STATIC", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "loadAssignment": { + "clusterName": "xds-grpc-cilium", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/xds.sock" + } + } + } + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + } + }, + { + "name": "/envoy-admin", + "type": "STATIC", + "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", + "loadAssignment": { + "clusterName": "/envoy-admin", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } + } + ] + } + ] + } + } + ] + }, + "dynamicResources": { + "ldsConfig": { + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + }, + "cdsConfig": { + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + } + }, + "bootstrapExtensions": [ + { + "name": "envoy.bootstrap.internal_listener", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + } + } + ], + "layeredRuntime": { + "layers": [ + { + "name": "static_layer_0", + "staticLayer": { + "overload": { + "global_downstream_max_connections": 50000 + } + } + } + ] + }, + "admin": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } +} diff --git a/magnum_cluster_api/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json b/magnum_cluster_api/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json new file mode 100644 index 00000000..e677deae --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json @@ -0,0 +1,998 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 8, + "links": [], + "panels": [ + { + "aliasColors": { + "avg": "#cffaff" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(irate(cilium_operator_process_cpu_seconds_total{io_cilium_app=\"operator\"}[1m])) by (pod) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "avg(irate(cilium_operator_process_cpu_seconds_total{io_cilium_app=\"operator\"}[1m])) by (pod) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "B" + }, + { + "expr": "max(irate(cilium_operator_process_cpu_seconds_total{io_cilium_app=\"operator\"}[1m])) by (pod) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "MAX_resident_memory_bytes_max": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_operator_process_resident_memory_bytes{io_cilium_app=\"operator\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "AVG_resident_memory_bytes", + "refId": "C" + }, + { + "expr": "max(cilium_operator_process_resident_memory_bytes{io_cilium_app=\"operator\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "MAX_resident_memory_bytes_max", + "refId": "D" + }, + { + "expr": "min(cilium_operator_process_resident_memory_bytes{io_cilium_app=\"operator\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "MIN_resident_memory_bytes_min", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Resident memory status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 6, + "panels": [], + "title": "IPAM", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(cilium_operator_ipam_ips) by (type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IP Addresses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cilium_operator_ec2_api_duration_seconds_sum[1m])/rate(cilium_operator_ec2_api_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}} {{response_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "EC2 API Interactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cilium_operator_ipam_nodes", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{category}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of nodes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cilium_operator_ipam_available", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "interfaces", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# interfaces with addresses available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cilium_operator_ipam_resync_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "operations", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Metadata Resync Operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cilium_operator_ec2_api_rate_limit_duration_seconds_sum[1m])/rate(cilium_operator_ec2_api_rate_limit_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "EC2 client side rate limiting", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(cilium_operator_ipam_interface_creation_ops[1m])) by (subnetId, status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{status}} ({{subnetId}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Interface Creation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 25, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Cilium Operator", + "uid": "1GC0TT4Wz", + "version": 2 +} diff --git a/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dashboard.json b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dashboard.json new file mode 100644 index 00000000..8de5ec1d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dashboard.json @@ -0,0 +1,3232 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "title": "General Processing", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "avg", + "lines": false + }, + { + "alias": "avg", + "fill": 0, + "fillBelowTo": "min" + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(sum(rate(hubble_flows_processed_total[1m])) by (pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "min(sum(rate(hubble_flows_processed_total[1m])) by (pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "B" + }, + { + "expr": "max(sum(rate(hubble_flows_processed_total[1m])) by (pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Flows processed Per Node", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Flows Types", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total{type=\"L7\"}[1m])) by (pod, subtype)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{subtype}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "L7 Flow Distribution", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total{type=\"Trace\"}[1m])) by (pod, subtype)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{subtype}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Trace Flow Distribution", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 16, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_flows_processed_total[1m])) by (pod, verdict)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{verdict}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Forwarded vs Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_drop_total[1m])) by (pod, reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Drop Reason", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": true, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(hubble_port_distribution_total[1m])) by (pod, protocol)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Protocol Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum (rate(hubble_port_distribution_total{port!=\"0\"}[1m])) by (pod, port, protocol))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{port}}/{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 Port Distribution", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + }, + { + "alias": "FIN", + "yaxis": 2 + }, + { + "alias": "RST", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv4\"}[1m])) by (pod, flag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{flag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TCPv4", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.2 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Missing TCP SYN-ACK", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + }, + { + "alias": "FIN", + "yaxis": 2 + }, + { + "alias": "RST", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv4\", flag=\"SYN\"}[1m])) by (pod) - sum(rate(hubble_tcp_flags_total{family=\"IPv4\", flag=\"SYN-ACK\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing SYN-ACK", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.2 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing TCPv4 SYN-ACKs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv6\"}[1m])) by (pod, flag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{flag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TCPv6", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.2 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Missing TCPv6 SYN-ACKs alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "fin", + "yaxis": 1 + }, + { + "alias": "FIN", + "yaxis": 2 + }, + { + "alias": "RST", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_tcp_flags_total{family=\"IPv6\", flag=\"SYN\"}[1m])) by (pod) - sum(rate(hubble_tcp_flags_total{family=\"IPv6\", flag=\"SYN-ACK\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing SYN-ACK", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.2 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing TCPv6 SYN-ACKs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv4\"}[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ICMPv4", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Missing ICMPv4 Echo-Reply alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 64, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv4\", type=\"EchoRequest\"}[1m])) by (pod) - sum(rate(hubble_icmp_total{family=\"IPv4\", type=\"EchoReply\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing ICMP Echo-Reply", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.1 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing ICMPv4 Echo-Reply", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv6\"}[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ICMPv6", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_icmp_total{family=\"IPv6\", type=\"EchoRequest\"}[1m])) by (pod) - sum(rate(hubble_icmp_total{family=\"IPv6\", type=\"EchoReply\"}[1m])) by (pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Missing ICMP Echo-Reply", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing ICMPv6 Echo-Reply", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 42, + "panels": [], + "title": "Network Policy", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Denies by Reason", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 61, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, protocol)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Denied Packets by Protocol", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, source))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 Source Pods with Denied Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 54, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(hubble_drop_total{reason=\"POLICY_DENIED\"}[1m])) by (pod, destination))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{destination}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 Destination Pods with Denied Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 47, + "panels": [], + "title": "HTTP", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_http_requests_total[1m])) by (pod, method)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 49, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_http_responses_total[1m])) by (pod, status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP responses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(hubble_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Request/Response Latency (p50)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(hubble_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Request/Response Latency (p99)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": true, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_http_requests_total[5m])) by (pod, protocol)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Protocol Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 6, + "panels": [], + "title": "DNS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 70 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_queries_total[1m])) by (pod, qtypes)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{qtypes}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 70 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_responses_total{rcode=\"No Error\"}[1m])) by (pod, qtypes)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{qtypes}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS responses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.5 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "DNS Request/Response Symmetry alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 70 + }, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_queries_total[1m])) by (pod, qtypes) - sum(rate(hubble_dns_responses_total[1m])) by (pod, qtypes)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{qtypes}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Missing DNS Responses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_response_types_total[1m])) by (pod, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Response Record Type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 57, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_responses_total{rcode=\"No Error\"}[1m])) by (pod,ips_returned)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ips_returned}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Response IPs Returned", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(hubble_dns_responses_total{rcode!=\"No Error\"}[1m])) by (pod, qtypes, rcode)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{rcode}} ({{qtypes}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DNS Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 56, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,sum(rate(hubble_dns_responses_total{rcode!=\"No Error\"}[1m])) by (pod, destination))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{destination}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pods with DNS errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, sum(rate(hubble_dns_queries_total[10m])*60) by (query, qtypes))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{query}} ({{qtypes}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 DNS Queries per minute", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Hubble Metrics and Monitoring", + "uid": "5HftnJAWz", + "version": 24 +} diff --git a/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json new file mode 100644 index 00000000..d286fdb3 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json @@ -0,0 +1,602 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.7" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 16612, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cilium-overview" + ], + "targetBlank": false, + "title": "Cilium Overviews", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "hubble" + ], + "targetBlank": false, + "title": "Hubble", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "DNS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_dns_queries_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source) > 0", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "DNS queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(hubble_dns_queries_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])*60) by (query))", + "legendFormat": "{{query}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 DNS queries", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "round(sum(rate(hubble_dns_queries_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source) - sum(label_replace(sum(rate(hubble_dns_responses_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\"}[$__rate_interval])) by (destination), \"source\", \"$1\", \"destination\", \"(.*)\")) without (destination), 0.001) > 0", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing DNS responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_dns_responses_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\", rcode!=\"No Error\"}[$__rate_interval])) by (destination, rcode) > 0", + "legendFormat": "{{destination}}: {{rcode}}", + "range": true, + "refId": "A" + } + ], + "title": "DNS errors", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "kubecon-demo" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "prometheus_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(cilium_version, cluster)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cilium_version, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(source_namespace)", + "hide": 0, + "includeAll": true, + "label": "Source Namespace", + "multi": true, + "name": "source_namespace", + "options": [], + "query": { + "query": "label_values(source_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(destination_namespace)", + "hide": 0, + "includeAll": true, + "label": "Destination Namespace", + "multi": true, + "name": "destination_namespace", + "options": [], + "query": { + "query": "label_values(destination_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Hubble / DNS Overview (Namespace)", + "uid": "_f0DUpY4k", + "version": 26, + "weekStart": "" + } + \ No newline at end of file diff --git a/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json new file mode 100644 index 00000000..428c58f4 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json @@ -0,0 +1,1385 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.7" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "round(sum(rate(hubble_http_requests_total{reporter=~\"${reporter}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\"}[$__rate_interval])), 0.001)", + "refId": "A" + } + ], + "title": "Incoming Request Volume", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", status!~\"5.*\"}[$__rate_interval]))\n/\nsum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{ cluster }} {{ method }} {{ source_namespace }}/{{ source_workload }}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Request Success Rate (non-5xx responses)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.50, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": "P50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.95, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\"}[$__rate_interval])) by (le))", + "hide": false, + "interval": "", + "legendFormat": "P95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\"}[$__rate_interval])) by (le))", + "hide": false, + "interval": "", + "legendFormat": "P99", + "range": true, + "refId": "C" + } + ], + "title": "Request Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 6, + "panels": [], + "title": "Requests by Source", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "sum", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, source_namespace, source_workload, status), 0.001)", + "interval": "", + "legendFormat": "{{ cluster }} {{ method }} {{ source_namespace }}/{{ source_workload }}: {{ status }}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Requests by Source and Response Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\",status!~\"5.*\"}[$__rate_interval])) by (cluster, source_namespace, source_workload)\n/\nsum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, source_namespace, source_workload)", + "interval": "", + "legendFormat": "{{ cluster }} {{ method }} {{ source_namespace }}/{{ source_workload }}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Request Success Rate (non-5xx responses) By Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, source_namespace, source_workload, le))", + "interval": "", + "legendFormat": "{{ cluster }} {{ source_namespace }}/{{ source_workload }} P50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, source_namespace, source_workload, le))", + "hide": false, + "interval": "", + "legendFormat": "{{ cluster }} {{ source_namespace }}/{{ source_workload }} P95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, source_namespace, source_workload, le))", + "hide": false, + "interval": "", + "legendFormat": "{{ cluster }} {{ source_namespace }}/{{ source_workload }} P99", + "range": true, + "refId": "C" + } + ], + "title": "HTTP Request Duration by Source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\", workload=~\"${source_workload}\"}\n) by (namespace, workload)", + "interval": "", + "legendFormat": "{{ cluster }} {{ namespace }}/{{ workload }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage by Source", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 9, + "panels": [], + "title": "Requests by Destination", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "sum", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, destination_namespace, destination_workload, status), 0.001)", + "interval": "", + "legendFormat": "{{ cluster }} {{ method }} {{ destination_namespace }}/{{ destination_workload }}: {{ status }}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Requests by Destination and Response Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "min", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\",status!~\"5.*\"}[$__rate_interval])) by (cluster, destination_namespace, destination_workload)\n/\nsum(rate(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, destination_namespace, destination_workload)", + "interval": "", + "legendFormat": "{{ cluster }} {{ method }} {{ destination_namespace }}/{{ destination_workload }}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Request Success Rate (non-5xx responses) By Destination", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, destination_namespace, destination_workload, le))", + "interval": "", + "legendFormat": "{{ cluster }} {{ destination_namespace }}/{{ destination_workload }} P50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, destination_namespace, destination_workload, le))", + "hide": false, + "interval": "", + "legendFormat": "{{ cluster }} {{ destination_namespace }}/{{ destination_workload }} P95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(hubble_http_request_duration_seconds_bucket{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", reporter=\"${reporter}\", source_namespace=~\"${source_namespace}\", source_workload=~\"${source_workload}\"}[$__rate_interval])) by (cluster, destination_namespace, destination_workload, le))", + "hide": false, + "interval": "", + "legendFormat": "{{ cluster }} {{ destination_namespace }}/{{ destination_workload }} P99", + "range": true, + "refId": "C" + } + ], + "title": "HTTP Request Duration by Destination", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"${cluster}\", namespace=~\"${destination_namespace}\", workload=\"${destination_workload}\"}\n) by (namespace, workload)", + "interval": "", + "legendFormat": "{{ cluster }} {{ namespace }}/{{ workload }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage by Destination", + "type": "timeseries" + } + ], + "refresh": "30s", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(hubble_http_requests_total, cluster)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(hubble_http_requests_total, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\"}, destination_namespace)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Destination Namespace", + "multi": false, + "name": "destination_namespace", + "options": [], + "query": { + "query": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\"}, destination_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\"}, destination_workload)", + "hide": 0, + "includeAll": false, + "label": "Destination Workload", + "multi": false, + "name": "destination_workload", + "options": [], + "query": { + "query": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\"}, destination_workload)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(hubble_http_requests_total, reporter)", + "hide": 0, + "includeAll": false, + "label": "Reporter", + "multi": false, + "name": "reporter", + "options": [], + "query": { + "query": "label_values(hubble_http_requests_total, reporter)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\"}, source_namespace)", + "hide": 0, + "includeAll": true, + "label": "Source Namespace", + "multi": true, + "name": "source_namespace", + "options": [], + "query": { + "query": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\"}, source_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", source_namespace=~\"${source_namespace}\"}, source_workload)", + "hide": 0, + "includeAll": true, + "label": "Source Workload", + "multi": true, + "name": "source_workload", + "options": [], + "query": { + "query": "label_values(hubble_http_requests_total{cluster=~\"${cluster}\", destination_namespace=~\"${destination_namespace}\", destination_workload=~\"${destination_workload}\", source_namespace=~\"${source_namespace}\"}, source_workload)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Hubble L7 HTTP Metrics by Workload", + "uid": "3g264CZVz", + "version": 3, + "weekStart": "" +} diff --git a/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json new file mode 100644 index 00000000..d0cf9d3b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json @@ -0,0 +1,1001 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.7" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 16612, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cilium-overview" + ], + "targetBlank": false, + "title": "Cilium Overviews", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "hubble" + ], + "targetBlank": false, + "title": "Hubble", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "Flows processed", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (type, subtype)", + "legendFormat": "{{type}}/{{subtype}}", + "range": true, + "refId": "A" + } + ], + "title": "Flows processed by type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (verdict)", + "legendFormat": "{{verdict}}", + "range": true, + "refId": "A" + } + ], + "title": "Flows processed by verdict", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 36, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source))", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 sources", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 37, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (destination))", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 destinations", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 10, + "panels": [], + "title": "Connection drops", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_tcp_flags_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\", flag=\"SYN\"}[$__rate_interval])) by (source) - sum(label_replace(sum(rate(hubble_tcp_flags_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\", flag=\"SYN-ACK\"}[$__rate_interval])) by (destination), \"source\", \"$1\", \"destination\", \"(.*)\")) without (destination) > 0", + "hide": false, + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing TCP SYN-ACKs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_icmp_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\", type=\"EchoRequest\"}[$__rate_interval])) by (source) - sum(label_replace(sum(rate(hubble_icmp_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\", type=\"EchoReply\"}[$__rate_interval])) by (destination), \"source\", \"$1\", \"destination\", \"(.*)\")) without (destination) > 0", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing ICMP Echo Replys", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 6, + "panels": [], + "title": "Network Policy drops", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_drop_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source, reason) > 0", + "legendFormat": "{{source}}: {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Network Policy drops by source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "kube-dns-7d44cdb5d5-g85vg: UNSUPPORTED_PROTOCOL_FOR_NAT_MASQUERADE" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_drop_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (destination, reason) > 0", + "legendFormat": "{{destination}}: {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Network Policy drops by destination", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "kubecon-demo" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "prometheus_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(cilium_version, cluster)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cilium_version, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(source_namespace)", + "hide": 0, + "includeAll": true, + "label": "Source Namespace", + "multi": true, + "name": "source_namespace", + "options": [], + "query": { + "query": "label_values(source_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(destination_namespace)", + "hide": 0, + "includeAll": true, + "label": "Destination Namespace", + "multi": true, + "name": "destination_namespace", + "options": [], + "query": { + "query": "label_values(destination_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Hubble / Network Overview (Namespace)", + "uid": "nlsO8tYVz", + "version": 18, + "weekStart": "" + } + \ No newline at end of file diff --git a/magnum_cluster_api/charts/cilium/files/nodeinit/prestop.bash b/magnum_cluster_api/charts/cilium/files/nodeinit/prestop.bash new file mode 100644 index 00000000..e78f178a --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/nodeinit/prestop.bash @@ -0,0 +1,60 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +{{ .Values.nodeinit.prestop.preScript }} + +if stat /tmp/node-deinit.cilium.io > /dev/null 2>&1; then + exit 0 +fi + +echo "Waiting on pods to stop..." +if [ ! -f /etc/crictl.yaml ] || grep -q 'docker' /etc/crictl.yaml; then + # Works for COS, ubuntu + while docker ps | grep -v "node-init" | grep -q "POD_cilium"; do sleep 1; done +else + # COS-beta (with containerd). Some versions of COS have crictl in /home/kubernetes/bin. + while PATH="${PATH}:/home/kubernetes/bin" crictl ps | grep -v "node-init" | grep -q "POD_cilium"; do sleep 1; done +fi + +if ip link show cilium_host; then + echo "Deleting cilium_host interface..." + ip link del cilium_host +fi + +{{- if not (eq .Values.nodeinit.bootstrapFile "") }} +rm -f {{ .Values.nodeinit.bootstrapFile | quote }} +{{- end }} + +rm -f /tmp/node-init.cilium.io +touch /tmp/node-deinit.cilium.io + +{{- if .Values.nodeinit.reconfigureKubelet }} +# Check if we're running on a GKE containerd flavor. +GKE_KUBERNETES_BIN_DIR="/home/kubernetes/bin" +if [[ -f "${GKE_KUBERNETES_BIN_DIR}/gke" ]] && command -v containerd &>/dev/null; then + CONTAINERD_CONFIG="/etc/containerd/config.toml" + echo "Reverting changes to the containerd configuration" + sed -Ei "s/^\#(\s+conf_template)/\1/g" "${CONTAINERD_CONFIG}" + echo "Removing the kubelet wrapper" + [[ -f "${GKE_KUBERNETES_BIN_DIR}/the-kubelet" ]] && mv "${GKE_KUBERNETES_BIN_DIR}/the-kubelet" "${GKE_KUBERNETES_BIN_DIR}/kubelet" +else + echo "Changing kubelet configuration to --network-plugin=kubenet" + sed -i "s:--network-plugin=cni\ --cni-bin-dir={{ .Values.cni.binPath }}:--network-plugin=kubenet:g" /etc/default/kubelet +fi +echo "Restarting the kubelet" +systemctl restart kubelet +{{- end }} + +{{- if (and .Values.gke.enabled (or .Values.enableIPv4Masquerade .Values.gke.disableDefaultSnat))}} +# If the IP-MASQ chain exists, add back default jump rule from the GKE instance configure script +if iptables -w -t nat -L IP-MASQ > /dev/null; then + iptables -w -t nat -A POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ +fi +{{- end }} + +{{ .Values.nodeinit.prestop.postScript }} + +echo "Node de-initialization complete" diff --git a/magnum_cluster_api/charts/cilium/files/nodeinit/startup.bash b/magnum_cluster_api/charts/cilium/files/nodeinit/startup.bash new file mode 100644 index 00000000..aaaba092 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/nodeinit/startup.bash @@ -0,0 +1,206 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +echo "Link information:" +ip link + +echo "Routing table:" +ip route + +echo "Addressing:" +ip -4 a +ip -6 a + +{{ .Values.nodeinit.startup.preScript }} + +{{- if .Values.nodeinit.removeCbrBridge }} +if ip link show cbr0; then + echo "Detected cbr0 bridge. Deleting interface..." + ip link del cbr0 +fi +{{- end }} + +{{- if .Values.nodeinit.reconfigureKubelet }} +# Check if we're running on a GKE containerd flavor as indicated by the presence +# of the '--container-runtime-endpoint' flag in '/etc/default/kubelet'. +GKE_KUBERNETES_BIN_DIR="/home/kubernetes/bin" +KUBELET_DEFAULTS_FILE="/etc/default/kubelet" +if [[ -f "${GKE_KUBERNETES_BIN_DIR}/gke" ]] && [[ $(grep -cF -- '--container-runtime-endpoint' "${KUBELET_DEFAULTS_FILE}") == "1" ]]; then + echo "GKE *_containerd flavor detected..." + + # (GKE *_containerd) Upon node restarts, GKE's containerd images seem to reset + # the /etc directory and our changes to the kubelet and Cilium's CNI + # configuration are removed. This leaves room for containerd and its CNI to + # take over pods previously managed by Cilium, causing Cilium to lose + # ownership over these pods. We rely on the empirical observation that + # /home/kubernetes/bin/kubelet is not changed across node reboots, and replace + # it with a wrapper script that performs some initialization steps when + # required and then hands over control to the real kubelet. + + # Only create the kubelet wrapper if we haven't previously done so. + if [[ ! -f "${GKE_KUBERNETES_BIN_DIR}/the-kubelet" ]]; + then + echo "Installing the kubelet wrapper..." + + # Rename the real kubelet. + mv "${GKE_KUBERNETES_BIN_DIR}/kubelet" "${GKE_KUBERNETES_BIN_DIR}/the-kubelet" + + # Initialize the kubelet wrapper which lives in the place of the real kubelet. + touch "${GKE_KUBERNETES_BIN_DIR}/kubelet" + chmod a+x "${GKE_KUBERNETES_BIN_DIR}/kubelet" + + # Populate the kubelet wrapper. It will perform the initialization steps we + # need and then become the kubelet. + cat <<'EOF' | tee "${GKE_KUBERNETES_BIN_DIR}/kubelet" +#!/bin/bash + +set -euo pipefail + +CNI_CONF_DIR="/etc/cni/net.d" +CONTAINERD_CONFIG="/etc/containerd/config.toml" + +# kubelet version string format is "Kubernetes v1.24-gke.900" +K8S_VERSION=$(/home/kubernetes/bin/the-kubelet --version) + +# Helper to check if a version string, passed as first parameter, is greater than or +# equal the one passed as second parameter. +function version_gte() { + [[ "$(printf '%s\n' "${2}" "${1}" | sort -V | head -n1)" = "${2}" ]] && return +} + +# Only stop and start containerd if the Cilium CNI configuration does not exist, +# or if the 'conf_template' property is present in the containerd config file, +# in order to avoid unnecessarily restarting containerd. +if [[ -z "$(find "${CNI_CONF_DIR}" -type f -name '*cilium*')" || \ + "$(grep -cE '^\s+conf_template' "${CONTAINERD_CONFIG}")" != "0" ]]; +then + # Stop containerd as it starts by creating a CNI configuration from a template + # causing pods to start with IPs assigned by GKE's CNI. + # 'disable --now' is used instead of stop as this script runs concurrently + # with containerd on node startup, and hence containerd might not have been + # started yet, in which case 'disable' prevents it from starting. + echo "Disabling and stopping containerd" + systemctl disable --now containerd + + # Remove any pre-existing files in the CNI configuration directory. We skip + # any possibly existing Cilium configuration file for the obvious reasons. + echo "Removing undesired CNI configuration files" + find "${CNI_CONF_DIR}" -type f -not -name '*cilium*' -exec rm {} \; + + # As mentioned above, the containerd configuration needs a little tweak in + # order not to create the default CNI configuration, so we update its config. + echo "Fixing containerd configuration" + sed -Ei 's/^(\s+conf_template)/\#\1/g' "${CONTAINERD_CONFIG}" + + if version_gte "${K8S_VERSION#"Kubernetes "}" "v1.24"; then + # Starting from GKE node version 1.24, containerd version used is 1.6. + # Since that version containerd no longer allows missing configuration for the CNI, + # not even for pods with hostNetwork set to true. Thus, we add a temporary one. + # This will be replaced with the real config by the agent pod. + echo -e '{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}' > /etc/cni/net.d/05-cilium.conf + fi + + # Start containerd. It won't create it's CNI configuration file anymore. + echo "Enabling and starting containerd" + systemctl enable --now containerd +fi + +# Become the real kubelet and, for k8s < 1.24, pass it additional dockershim +# flags (and place these last so they have precedence). +if version_gte "${K8S_VERSION#"Kubernetes "}" "v1.24"; then + exec /home/kubernetes/bin/the-kubelet "${@}" +else + exec /home/kubernetes/bin/the-kubelet "${@}" --network-plugin=cni --cni-bin-dir={{ .Values.cni.binPath }} +fi +EOF + else + echo "Kubelet wrapper already exists, skipping..." + fi +else + # kubelet version string format is "Kubernetes v1.24-gke.900" + K8S_VERSION=$(kubelet --version) + + # Helper to check if a version string, passed as first parameter, is greater than or + # equal the one passed as second parameter. + function version_gte() { + [[ "$(printf '%s\n' "${2}" "${1}" | sort -V | head -n1)" = "${2}" ]] && return + } + + # Dockershim flags have been removed since k8s 1.24. + if ! version_gte "${K8S_VERSION#"Kubernetes "}" "v1.24"; then + # (Generic) Alter the kubelet configuration to run in CNI mode + echo "Changing kubelet configuration to --network-plugin=cni --cni-bin-dir={{ .Values.cni.binPath }}" + mkdir -p {{ .Values.cni.binPath }} + sed -i "s:--network-plugin=kubenet:--network-plugin=cni\ --cni-bin-dir={{ .Values.cni.binPath }}:g" "${KUBELET_DEFAULTS_FILE}" + fi +fi +echo "Restarting the kubelet..." +systemctl restart kubelet +{{- end }} + +{{- if (and .Values.gke.enabled (or .Values.enableIPv4Masquerade .Values.gke.disableDefaultSnat))}} +# If Cilium is configured to manage masquerading of traffic leaving the node, +# we need to disable the IP-MASQ chain because even if ip-masq-agent +# is not installed, the node init script installs some default rules into +# the IP-MASQ chain. +# If we remove the jump to that ip-masq chain, then we ensure the ip masquerade +# configuration is solely managed by Cilium. +# Also, if Cilium is installed, it may be expected that it would be solely responsible +# for the networking configuration on that node. So provide the same functionality +# as the --disable-snat-flag for existing GKE clusters. +iptables -w -t nat -D POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ || true +{{- end }} + +{{- if not (eq .Values.nodeinit.bootstrapFile "") }} +mkdir -p {{ .Values.nodeinit.bootstrapFile | dir | quote }} +date > {{ .Values.nodeinit.bootstrapFile | quote }} +{{- end }} + +{{- if .Values.azure.enabled }} +# AKS: If azure-vnet is installed on the node, and (still) configured in bridge mode, +# configure it as 'transparent' to be consistent with Cilium's CNI chaining config. +# If the azure-vnet CNI config is not removed, kubelet will execute CNI CHECK commands +# against it every 5 seconds and write 'bridge' to its state file, causing inconsistent +# behaviour when Pods are removed. +if [ -f /etc/cni/net.d/10-azure.conflist ]; then + echo "Ensuring azure-vnet is configured in 'transparent' mode..." + sed -i 's/"mode":\s*"bridge"/"mode":"transparent"/g' /etc/cni/net.d/10-azure.conflist +fi + +# The azure0 interface being present means the node was booted with azure-vnet configured +# in bridge mode. This means there might be ebtables rules and neight entries interfering +# with pod connectivity if we deploy with Azure IPAM. +if ip l show dev azure0 >/dev/null 2>&1; then + + # In Azure IPAM mode, also remove the azure-vnet state file, otherwise ebtables rules get + # restored by the azure-vnet CNI plugin on every CNI CHECK, which can cause connectivity + # issues in Cilium-managed Pods. Since azure-vnet is no longer called on scheduling events, + # this file can be removed. + rm -f /var/run/azure-vnet.json + + # This breaks connectivity for existing workload Pods when Cilium is scheduled, but we need + # to flush these to prevent Cilium-managed Pod IPs conflicting with Pod IPs previously allocated + # by azure-vnet. These ebtables DNAT rules contain fixed MACs that are no longer bound on the node, + # causing packets for these Pods to be redirected back out to the gateway, where they are dropped. + echo 'Flushing ebtables pre/postrouting rules in nat table.. (disconnecting non-Cilium Pods!)' + ebtables -t nat -F PREROUTING || true + ebtables -t nat -F POSTROUTING || true + + # ip-masq-agent periodically injects PERM neigh entries towards the gateway + # for all other k8s nodes in the cluster. These are safe to flush, as ARP can + # resolve these nodes as usual. PERM entries will be automatically restored later. + echo 'Deleting all permanent neighbour entries on azure0...' + ip neigh show dev azure0 nud permanent | cut -d' ' -f1 | xargs -r -n1 ip neigh del dev azure0 to || true +fi +{{- end }} + +{{- if .Values.nodeinit.revertReconfigureKubelet }} +rm -f /tmp/node-deinit.cilium.io +{{- end }} + +{{ .Values.nodeinit.startup.postScript }} + +echo "Node initialization complete" diff --git a/magnum_cluster_api/charts/cilium/files/spire/init.bash b/magnum_cluster_api/charts/cilium/files/spire/init.bash new file mode 100644 index 00000000..4b6f57bd --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/spire/init.bash @@ -0,0 +1,48 @@ +# shellcheck disable=SC2086 +# shellcheck disable=SC2139 +set -e + +echo "Waiting for spire process to start" +while ! pgrep spire-server > /dev/null; do sleep 5; done + +SPIRE_SERVER_ROOT_PATH="/proc/$(pgrep spire-server)/root" + +alias spire_server="${SPIRE_SERVER_ROOT_PATH}/opt/spire/bin/spire-server" +SOCKET_PATH="${SPIRE_SERVER_ROOT_PATH}/tmp/spire-server/private/api.sock" +SOCKET_FLAG="-socketPath ${SOCKET_PATH}" + +echo "Checking spire-server status" +while ! spire_server entry show ${SOCKET_FLAG} &> /dev/null; do + echo "Waiting for spire-server to start..." + sleep 5 +done + +echo "Spire Server is up, initializing cilium spire entries..." + +AGENT_SPIFFE_ID="spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/ns/{{ .Values.authentication.mutual.spire.install.namespace }}/sa/spire-agent" +AGENT_SELECTORS="-selector k8s_psat:agent_ns:{{ .Values.authentication.mutual.spire.install.namespace }} -selector k8s_psat:agent_sa:spire-agent" +CILIUM_AGENT_SPIFFE_ID="spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-agent" +CILIUM_AGENT_SELECTORS="-selector k8s:ns:{{ .Release.Namespace }} -selector k8s:sa:{{ .Values.serviceAccounts.cilium.name }}" +CILIUM_OPERATOR_SPIFFE_ID="spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-operator" +CILIUM_OPERATOR_SELECTORS="-selector k8s:ns:{{ .Release.Namespace }} -selector k8s:sa:{{ .Values.serviceAccounts.operator.name }}" + +while pgrep spire-server > /dev/null; +do + echo "Ensuring agent entry" + if spire_server entry show ${SOCKET_FLAG} -spiffeID $AGENT_SPIFFE_ID $AGENT_SELECTORS | grep -q "Found 0 entries" &> /dev/null; then + spire_server entry create ${SOCKET_FLAG} -spiffeID $AGENT_SPIFFE_ID $AGENT_SELECTORS -node + fi + + echo "Ensuring cilium-agent entry (required for the delegated identity to work)" + if spire_server entry show ${SOCKET_FLAG} -spiffeID $CILIUM_AGENT_SPIFFE_ID $CILIUM_AGENT_SELECTORS | grep -q "Found 0 entries" &> /dev/null; then + spire_server entry create ${SOCKET_FLAG} -spiffeID $CILIUM_AGENT_SPIFFE_ID -parentID $AGENT_SPIFFE_ID $CILIUM_AGENT_SELECTORS + fi + + echo "Ensuring cilium-operator entry (required for creating SPIFFE identities)" + if spire_server entry show ${SOCKET_FLAG} -spiffeID $CILIUM_OPERATOR_SPIFFE_ID $CILIUM_OPERATOR_SELECTORS | grep -q "Found 0 entries" &> /dev/null; then + spire_server entry create ${SOCKET_FLAG} -spiffeID $CILIUM_OPERATOR_SPIFFE_ID -parentID $AGENT_SPIFFE_ID $CILIUM_OPERATOR_SELECTORS + fi + + echo "Cilium Spire entries are initialized successfully or already in-sync" + sleep 30; +done diff --git a/magnum_cluster_api/charts/cilium/files/spire/wait-for-spire.bash b/magnum_cluster_api/charts/cilium/files/spire/wait-for-spire.bash new file mode 100644 index 00000000..3507af1c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/files/spire/wait-for-spire.bash @@ -0,0 +1,35 @@ +set -e + +echo "Waiting for spire server to be reachable to start" + +{{- if .Values.authentication.mutual.spire.serverAddress }} +ADDR="{{ .Values.authentication.mutual.spire.serverAddress }}" +{{- else }} +ADDR="spire-server.{{ .Values.authentication.mutual.spire.install.namespace}}.svc:8081" +{{- end }} +CONN_TIMEOUT="3" +TIMEOUT="60" + +call_tcp_endpoint_with_timeout() { + local addr="$1" + local timeout="$2" + + nc -z "$addr" -w "$timeout" &> /dev/null +} + +# wait for SPIRE server to be reachable till $TIMEOUT is reached +start_time=$(date +%s) +while true; do + if call_tcp_endpoint_with_timeout "$ADDR" "$CONN_TIMEOUT"; then + echo "SPIRE server is reachable" + break + fi + + if [ $(( $(date +%s) - start_time )) -gt "$TIMEOUT" ]; then + echo "Timed out waiting for spire server to be reachable" + exit 1 + fi + + echo "Waiting for spire server to be reachable" + sleep 1 +done diff --git a/magnum_cluster_api/charts/cilium/templates/NOTES.txt b/magnum_cluster_api/charts/cilium/templates/NOTES.txt new file mode 100644 index 00000000..f5405074 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/NOTES.txt @@ -0,0 +1,22 @@ +{{- if (and (.Values.preflight.enabled) (not (.Values.agent)) (not (.Values.operator.enabled))) }} + You have successfully ran the preflight check. + Now make sure to check the number of READY pods is the same as the number of running cilium pods. + Then make sure the cilium preflight deployment is also marked READY 1/1. + If you have an issues please refer to the CNP Validation section in the upgrade guide. +{{- else if (and (.Values.hubble.enabled) (.Values.hubble.relay.enabled)) }} + {{- if (.Values.hubble.ui.enabled) }} + You have successfully installed {{ title .Chart.Name }} with Hubble Relay and Hubble UI. + {{- else }} + You have successfully installed {{ title .Chart.Name }} with Hubble Relay. + {{- end }} +{{- else if .Values.hubble.enabled }} + You have successfully installed {{ title .Chart.Name }} with Hubble. +{{- else if (and (.Values.hubble.ui.enabled) (.Values.hubble.ui.standalone.enabled)) }} + You have successfully installed {{ title .Chart.Name }} with standalone Hubble UI. +{{- else }} + You have successfully installed {{ title .Chart.Name }}. +{{- end }} + +Your release version is {{ .Chart.Version }}. + +For any further help, visit https://docs.cilium.io/en/v{{ (semver .Chart.Version).Major }}.{{ (semver .Chart.Version).Minor }}/gettinghelp diff --git a/magnum_cluster_api/charts/cilium/templates/_helpers.tpl b/magnum_cluster_api/charts/cilium/templates/_helpers.tpl new file mode 100644 index 00000000..3e5429e2 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/_helpers.tpl @@ -0,0 +1,156 @@ +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cilium.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Render full image name from given values, e.g: +``` +image: + repository: quay.io/cilium/cilium + tag: v1.10.1 + useDigest: true + digest: abcdefgh +``` +then `include "cilium.image" .Values.image` +will return `quay.io/cilium/cilium:v1.10.1@abcdefgh` +*/}} +{{- define "cilium.image" -}} +{{- $digest := (.useDigest | default false) | ternary (printf "@%s" .digest) "" -}} +{{- if .override -}} +{{- printf "%s" .override -}} +{{- else -}} +{{- printf "%s:%s%s" .repository .tag $digest -}} +{{- end -}} +{{- end -}} + +{{/* +Return user specify priorityClass or default criticalPriorityClass +Usage: + include "cilium.priorityClass" (list $ ) +where: +* `priorityClass`: is user specify priorityClass e.g `.Values.operator.priorityClassName` +* `criticalPriorityClass`: default criticalPriorityClass, e.g `"system-cluster-critical"` + This value is used when `priorityClass` is `nil` and + `.Values.enableCriticalPriorityClass=true` and kubernetes supported it. +*/}} +{{- define "cilium.priorityClass" -}} +{{- $root := index . 0 -}} +{{- $priorityClass := index . 1 -}} +{{- $criticalPriorityClass := index . 2 -}} +{{- if $priorityClass }} + {{- $priorityClass }} +{{- else if and $root.Values.enableCriticalPriorityClass $criticalPriorityClass -}} + {{- if and (eq $root.Release.Namespace "kube-system") (semverCompare ">=1.10-0" $root.Capabilities.KubeVersion.Version) -}} + {{- $criticalPriorityClass }} + {{- else if semverCompare ">=1.17-0" $root.Capabilities.KubeVersion.Version -}} + {{- $criticalPriorityClass }} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "ingress.apiVersion" -}} +{{- if semverCompare ">=1.16-0, <1.19-0" .Capabilities.KubeVersion.Version -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else if semverCompare "^1.19-0" .Capabilities.KubeVersion.Version -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate backend for Hubble UI ingress. +*/}} +{{- define "ingress.paths" -}} +{{ if semverCompare ">=1.4-0, <1.19-0" .Capabilities.KubeVersion.Version -}} +backend: + serviceName: hubble-ui + servicePort: http +{{- else if semverCompare "^1.19-0" .Capabilities.KubeVersion.Version -}} +pathType: Prefix +backend: + service: + name: hubble-ui + port: + name: http +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "cronjob.apiVersion" -}} +{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version -}} +{{- print "batch/v1" -}} +{{- else -}} +{{- print "batch/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "podDisruptionBudget.apiVersion" -}} +{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version -}} +{{- print "policy/v1" -}} +{{- else -}} +{{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Generate TLS CA for Cilium +Note: Always use this template as follows: + {{- $_ := include "cilium.ca.setup" . -}} + +The assignment to `$_` is required because we store the generated CI in a global `commonCA` +and `commonCASecretName` variables. + +*/}} +{{- define "cilium.ca.setup" }} + {{- if not .commonCA -}} + {{- $ca := "" -}} + {{- $secretName := "cilium-ca" -}} + {{- $crt := .Values.tls.ca.cert -}} + {{- $key := .Values.tls.ca.key -}} + {{- if and $crt $key }} + {{- $ca = buildCustomCert $crt $key -}} + {{- else }} + {{- with lookup "v1" "Secret" .Release.Namespace $secretName }} + {{- $crt := index .data "ca.crt" }} + {{- $key := index .data "ca.key" }} + {{- $ca = buildCustomCert $crt $key -}} + {{- else }} + {{- $validity := ( .Values.tls.ca.certValidityDuration | int) -}} + {{- $ca = genCA "Cilium CA" $validity -}} + {{- end }} + {{- end -}} + {{- $_ := set (set . "commonCA" $ca) "commonCASecretName" $secretName -}} + {{- end -}} +{{- end -}} + +{{/* +Check if duration is non zero value, return duration, empty when zero. +*/}} +{{- define "hasDuration" }} +{{- $now := now }} +{{- if ne $now ($now | dateModify (toString .)) }} +{{- . }} +{{- end }} +{{- end }} + +{{/* +Validate duration field, return validated duration, 0s when provided duration is empty. +*/}} +{{- define "validateDuration" }} +{{- if . }} +{{- $_ := now | mustDateModify (toString .) }} +{{- . }} +{{- else -}} +0s +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrole.yaml new file mode 100644 index 00000000..9ccf2192 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrole.yaml @@ -0,0 +1,150 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) }} +{{- /* +Keep file in sync with cilium-preflight/clusterrole.yaml +*/ -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +{{- if .Values.annotateK8sNode }} +- apiGroups: + - "" + resources: + - nodes/status + verbs: + # To annotate the k8s node with Cilium's metadata + - patch +{{- end }} +{{- if .Values.l2announcements.enabled }} +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + - list + - delete +{{- end }} +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +{{- if eq "k8s" .Values.tls.secretsBackend }} +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +{{- end }} +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrolebinding.yaml new file mode 100644 index 00000000..34b0a58b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/daemonset.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/daemonset.yaml new file mode 100644 index 00000000..f602af67 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/daemonset.yaml @@ -0,0 +1,981 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) }} + +{{- /* Default values with backwards compatibility */ -}} +{{- $defaultKeepDeprecatedProbes := true -}} + +{{- /* Default values when 1.8 was initially deployed */ -}} +{{- if semverCompare ">=1.8" (default "1.8" .Values.upgradeCompatibility) -}} + {{- $defaultKeepDeprecatedProbes = false -}} +{{- end -}} + +{{- $kubeProxyReplacement := (coalesce .Values.kubeProxyReplacement "false") -}} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent + {{- if .Values.keepDeprecatedLabels }} + kubernetes.io/cluster-service: "true" + {{- if and .Values.gke.enabled (eq .Release.Namespace "kube-system" ) }} + {{- fail "Invalid configuration: Installing Cilium on GKE with 'kubernetes.io/cluster-service' labels on 'kube-system' namespace causes Cilium DaemonSet to be removed by GKE. Either install Cilium on a different Namespace or install with '--set keepDeprecatedLabels=false'" }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: cilium + {{- if .Values.keepDeprecatedLabels }} + kubernetes.io/cluster-service: "true" + {{- end }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- if and .Values.prometheus.enabled (not .Values.prometheus.serviceMonitor.enabled) }} + prometheus.io/port: "{{ .Values.prometheus.port }}" + prometheus.io/scrape: "true" + {{- end }} + {{- if .Values.rollOutCiliumPods }} + # ensure pods roll when configmap updates + cilium.io/cilium-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-configmap.yaml") . | sha256sum | quote }} + {{- end }} + {{- if not .Values.securityContext.privileged }} + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + {{- if .Values.cgroup.autoMount.enabled }} + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + {{- end }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + {{- if .Values.keepDeprecatedLabels }} + kubernetes.io/cluster-service: "true" + {{- end }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cilium-agent + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.sleepAfterInit }} + command: + - /bin/bash + - -c + - -- + args: + - | + while true; do + sleep 30; + done + livenessProbe: + exec: + command: + - "true" + readinessProbe: + exec: + command: + - "true" + {{- else }} + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + {{- with .Values.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} + startupProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: {{ .Values.healthPort }} + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: {{ .Values.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.startupProbe.periodSeconds }} + successThreshold: 1 + initialDelaySeconds: 5 + {{- end }} + livenessProbe: + {{- if or .Values.keepDeprecatedProbes $defaultKeepDeprecatedProbes }} + exec: + command: + - cilium + - status + - --brief + {{- else }} + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: {{ .Values.healthPort }} + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + {{- end }} + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 120 + {{- end }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: 1 + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + timeoutSeconds: 5 + readinessProbe: + {{- if or .Values.keepDeprecatedProbes $defaultKeepDeprecatedProbes }} + exec: + command: + - cilium + - status + - --brief + {{- else }} + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: {{ .Values.healthPort }} + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + {{- end }} + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + initialDelaySeconds: 5 + {{- end }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: 1 + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + timeoutSeconds: 5 + {{- end }} + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- with .Values.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if .Values.cni.install }} + lifecycle: + {{- if ne .Values.cni.chainingMode "aws-cni" }} + postStart: + exec: + command: + - "bash" + - "-c" + - | + {{- tpl (.Files.Get "files/agent/poststart-eni.bash") . | nindent 20 }} + {{- end }} + preStop: + exec: + command: + - /cni-uninstall.sh + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- if or .Values.prometheus.enabled .Values.hubble.metrics.enabled }} + ports: + - name: peer-service + containerPort: {{ .Values.hubble.peerService.targetPort }} + hostPort: {{ .Values.hubble.peerService.targetPort }} + protocol: TCP + {{- if .Values.prometheus.enabled }} + - name: prometheus + containerPort: {{ .Values.prometheus.port }} + hostPort: {{ .Values.prometheus.port }} + protocol: TCP + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.enabled) }} + - name: envoy-metrics + containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + {{- end }} + {{- end }} + {{- if .Values.hubble.metrics.enabled }} + - name: hubble-metrics + containerPort: {{ .Values.hubble.metrics.port }} + hostPort: {{ .Values.hubble.metrics.port }} + protocol: TCP + {{- end }} + {{- end }} + securityContext: + {{- if .Values.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + capabilities: + add: + {{- with .Values.securityContext.capabilities.ciliumAgent }} + {{- toYaml . | nindent 14 }} + {{- end }} + drop: + - ALL + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + mountPath: {{ dir .Values.authentication.mutual.spire.adminSocketPath }} + readOnly: false + {{- end }} + {{- if .Values.envoy.enabled }} + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + {{- end }} + {{- if not .Values.securityContext.privileged }} + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + {{- end}} + {{- /* CRI-O already mounts the BPF filesystem */ -}} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} + - name: bpf-maps + mountPath: /sys/fs/bpf + {{- if .Values.securityContext.privileged }} + mountPropagation: Bidirectional + {{- else }} + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + {{- end}} + {{- end }} + {{- if not (contains "/run/cilium/cgroupv2" .Values.cgroup.hostRoot) }} + # Check for duplicate mounts before mounting + - name: cilium-cgroup + mountPath: {{ .Values.cgroup.hostRoot }} + {{- end}} + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: {{ .Values.cni.hostConfDirMountPath }} + {{- if .Values.etcd.enabled }} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + - name: etcd-secrets + mountPath: /var/lib/etcd-secrets + readOnly: true + {{- end }} + {{- end }} + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + {{- if .Values.ipMasqAgent.enabled }} + - name: ip-masq-agent + mountPath: /etc/config + readOnly: true + {{- end }} + {{- if .Values.cni.configMap }} + - name: cni-configuration + mountPath: {{ .Values.cni.confFileMountPath }} + readOnly: true + {{- end }} + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + {{- if and .Values.encryption.enabled (eq .Values.encryption.type "ipsec") }} + - name: cilium-ipsec-secrets + mountPath: {{ .Values.encryption.ipsec.mountPath | default .Values.encryption.mountPath }} + {{- end }} + {{- if .Values.kubeConfigPath }} + - name: kube-config + mountPath: {{ .Values.kubeConfigPath }} + readOnly: true + {{- end }} + {{- if .Values.bgp.enabled }} + - name: bgp-config-path + mountPath: /var/lib/cilium/bgp + readOnly: true + {{- end }} + {{- if and .Values.hubble.enabled .Values.hubble.tls.enabled (hasKey .Values.hubble "listenAddress") }} + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + {{- end }} + - name: tmp + mountPath: /tmp + {{- range .Values.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- if .mountPropagation }} + mountPropagation: {{ .mountPropagation }} + {{- end }} + {{- end }} + {{- if .Values.hubble.export.dynamic.enabled }} + - name: hubble-flowlog-config + mountPath: /flowlog-config + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.monitor.enabled }} + - name: cilium-monitor + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - -c + - -- + args: + - |- + for i in {1..5}; do \ + [ -S /var/run/cilium/monitor1_2.sock ] && break || sleep 10;\ + done; \ + cilium-dbg monitor + {{- range $type := .Values.monitor.eventTypes -}} + {{ " " }}--type={{ $type }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cilium-run + mountPath: /var/run/cilium + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.monitor.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.extraContainers }} + {{- toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + initContainers: + - name: config + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - cilium-dbg + - build-config + {{- if (not (kindIs "invalid" .Values.daemon.configSources)) }} + - "--source={{.Values.daemon.configSources}}" + {{- end }} + {{- if (not (kindIs "invalid" .Values.daemon.allowedConfigOverrides)) }} + - "--allow-config-keys={{.Values.daemon.allowedConfigOverrides}}" + {{- end }} + {{- if (not (kindIs "invalid" .Values.daemon.blockedConfigOverrides)) }} + - "--deny-config-keys={{.Values.daemon.blockedConfigOverrides}}" + {{- end }} + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- if .Values.cgroup.autoMount.enabled }} + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: CGROUP_ROOT + value: {{ .Values.cgroup.hostRoot }} + - name: BIN_PATH + value: {{ .Values.cni.binPath }} + {{- with .Values.cgroup.autoMount.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + {{- if .Values.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + capabilities: + add: + {{- with .Values.securityContext.capabilities.mountCgroup }} + {{- toYaml . | nindent 14 }} + {{- end }} + drop: + - ALL + {{- end}} + - name: apply-sysctl-overwrites + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + env: + - name: BIN_PATH + value: {{ .Values.cni.binPath }} + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + {{- if .Values.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + capabilities: + add: + {{- with .Values.securityContext.capabilities.applySysctlOverwrites }} + {{- toYaml . | nindent 14 }} + {{- end }} + drop: + - ALL + {{- end}} + {{- end }} + {{- if and .Values.bpf.autoMount.enabled (not .Values.securityContext.privileged) }} + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + {{- /* CRI-O already mounts the BPF filesystem */ -}} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + {{- end }} + {{- end }} + {{- if and .Values.nodeinit.enabled .Values.nodeinit.bootstrapFile }} + - name: wait-for-node-init + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + command: + - sh + - -c + - | + until test -s {{ (print "/tmp/cilium-bootstrap.d/" (.Values.nodeinit.bootstrapFile | base)) | quote }}; do + echo "Waiting on node-init to run..."; + sleep 1; + done + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cilium-bootstrap-file-dir + mountPath: "/tmp/cilium-bootstrap.d" + {{- end }} + - name: clean-cilium-state + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + {{- if .Values.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + capabilities: + add: + {{- with .Values.securityContext.capabilities.cleanCiliumState }} + {{- toYaml . | nindent 14 }} + {{- end }} + drop: + - ALL + {{- end}} + volumeMounts: + {{- /* CRI-O already mounts the BPF filesystem */ -}} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} + - name: bpf-maps + mountPath: /sys/fs/bpf + {{- end }} + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: {{ .Values.cgroup.hostRoot }} + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- if and .Values.waitForKubeProxy (and (ne $kubeProxyReplacement "strict") (ne $kubeProxyReplacement "true")) }} + - name: wait-for-kube-proxy + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + securityContext: + privileged: true + command: + - bash + - -c + - | + while true + do + if iptables-nft-save -t mangle | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-PROXY-CANARY)'; then + echo "Found KUBE-IPTABLES-HINT or KUBE-PROXY-CANARY iptables rule in 'iptables-nft-save -t mangle'" + exit 0 + fi + if ip6tables-nft-save -t mangle | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-PROXY-CANARY)'; then + echo "Found KUBE-IPTABLES-HINT or KUBE-PROXY-CANARY iptables rule in 'ip6tables-nft-save -t mangle'" + exit 0 + fi + if iptables-legacy-save | grep -E '^:KUBE-PROXY-CANARY'; then + echo "Found KUBE-PROXY-CANARY iptables rule in 'iptables-legacy-save" + exit 0 + fi + if ip6tables-legacy-save | grep -E '^:KUBE-PROXY-CANARY'; then + echo "KUBE-PROXY-CANARY iptables rule in 'ip6tables-legacy-save'" + exit 0 + fi + echo "Waiting for kube-proxy to create iptables rules..."; + sleep 1; + done + terminationMessagePolicy: FallbackToLogsOnError + {{- end }} # wait-for-kube-proxy + {{- if .Values.cni.install }} + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "/install-plugin.sh" + {{- with .Values.cni.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + securityContext: + {{- if .Values.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin + {{- end }} # .Values.cni.install + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.priorityClassName "system-node-critical") }} + serviceAccount: {{ .Values.serviceAccounts.cilium.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.cilium.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.cilium.automount }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + hostNetwork: true + {{- if and .Values.etcd.managed (not .Values.etcd.k8sService) }} + # In managed etcd mode, Cilium must be able to resolve the DNS name of + # the etcd service + dnsPolicy: ClusterFirstWithHostNet + {{- else if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if and .Values.clustermesh.config.enabled (not (and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.kvstoremesh.enabled )) }} + hostAliases: + {{- range $cluster := .Values.clustermesh.config.clusters }} + {{- range $ip := $cluster.ips }} + - ip: {{ $ip }} + hostnames: [ "{{ $cluster.name }}.{{ $.Values.clustermesh.config.domain }}" ] + {{- end }} + {{- end }} + {{- end }} + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: {{ .Values.daemon.runPath }} + type: DirectoryOrCreate + {{- /* CRI-O already mounts the BPF filesystem */ -}} + {{- if and .Values.bpf.autoMount.enabled (not (eq .Values.containerRuntime.integration "crio")) }} + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + {{- end }} + {{- if .Values.cgroup.autoMount.enabled }} + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + {{- end }} + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: {{ .Values.cgroup.hostRoot}} + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: {{ .Values.cni.binPath }} + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: {{ .Values.cni.confPath }} + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + hostPath: + path: {{ dir .Values.authentication.mutual.spire.adminSocketPath }} + type: DirectoryOrCreate + {{- end }} + {{- if .Values.envoy.enabled }} + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "{{ .Values.daemon.runPath }}/envoy/sockets" + type: DirectoryOrCreate + {{- end }} + {{- if .Values.kubeConfigPath }} + - name: kube-config + hostPath: + path: {{ .Values.kubeConfigPath }} + type: FileOrCreate + {{- end }} + {{- if and .Values.nodeinit.enabled .Values.nodeinit.bootstrapFile }} + - name: cilium-bootstrap-file-dir + hostPath: + path: {{ .Values.nodeinit.bootstrapFile | dir | quote }} + type: DirectoryOrCreate + {{- end }} + {{- if .Values.etcd.enabled }} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: etcd-config + path: etcd.config + # To read the k8s etcd secrets in case the user might want to use TLS + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + - name: etcd-secrets + secret: + secretName: cilium-etcd-secrets + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + {{- end }} + {{- end }} + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: common-etcd-client-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + optional: true + items: + - key: {{ .Values.tls.caBundle.key }} + path: common-etcd-client-ca.crt + {{- end }} + {{- if and .Values.ipMasqAgent .Values.ipMasqAgent.enabled }} + - name: ip-masq-agent + configMap: + name: ip-masq-agent + optional: true + items: + - key: config + path: ip-masq-agent + {{- end }} + {{- if and .Values.encryption.enabled (eq .Values.encryption.type "ipsec") }} + - name: cilium-ipsec-secrets + secret: + secretName: {{ .Values.encryption.ipsec.secretName | default .Values.encryption.secretName }} + {{- end }} + {{- if .Values.cni.configMap }} + - name: cni-configuration + configMap: + name: {{ .Values.cni.configMap }} + {{- end }} + {{- if .Values.bgp.enabled }} + - name: bgp-config-path + configMap: + name: bgp-config + {{- end }} + {{- if not .Values.securityContext.privileged }} + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + {{- end }} + {{- if and .Values.hubble.enabled .Values.hubble.tls.enabled (hasKey .Values.hubble "listenAddress") }} + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: client-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + optional: true + items: + - key: {{ .Values.tls.caBundle.key }} + path: client-ca.crt + {{- end }} + {{- end }} + {{- if .Values.hubble.export.dynamic.enabled }} + - name: hubble-flowlog-config + configMap: + name: {{ .Values.hubble.export.dynamic.config.configMapName }} + optional: true + {{- end }} + {{- range .Values.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- if .hostPathType }} + type: {{ .hostPathType }} + {{- end }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml new file mode 100644 index 00000000..6f19135d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/dashboards-configmap.yaml @@ -0,0 +1,30 @@ +{{- if .Values.dashboards.enabled }} +{{- $files := .Files.Glob "files/cilium-agent/dashboards/*.json" }} +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Values.dashboards.namespace | default $.Release.Namespace }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + {{- if $.Values.dashboards.label }} + {{ $.Values.dashboards.label }}: {{ ternary $.Values.dashboards.labelValue "1" (not (empty $.Values.dashboards.labelValue)) | quote }} + {{- end }} + {{- if or $.Values.dashboards.annotations $.Values.annotations }} + annotations: + {{- with $.Values.dashboards.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/role.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/role.yaml new file mode 100644 index 00000000..e4ad86d1 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/role.yaml @@ -0,0 +1,116 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + +{{- end }} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.ingressController.enabled .Values.ingressController.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-ingress-secrets + namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +{{- end }} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-gateway-secrets + namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +{{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.envoyConfig.enabled .Values.envoyConfig.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-envoy-config-secrets + namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +{{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.bgpControlPlane.enabled .Values.bgpControlPlane.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-bgp-control-plane-secrets + namespace: {{ .Values.bgpControlPlane.secretsNamespace.name | quote }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +{{- end}} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/rolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/rolebinding.yaml new file mode 100644 index 00000000..cfa8697b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/rolebinding.yaml @@ -0,0 +1,110 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.ingressController.enabled .Values.ingressController.secretsNamespace.name}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-secrets + namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-ingress-secrets +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.name}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-gateway-secrets + namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-gateway-secrets +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.envoyConfig.enabled .Values.envoyConfig.secretsNamespace.name}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-envoy-config-secrets + namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-envoy-config-secrets +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.bgpControlPlane.enabled .Values.bgpControlPlane.secretsNamespace.name}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-bgp-control-plane-secrets + namespace: {{ .Values.bgpControlPlane.secretsNamespace.name | quote }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-bgp-control-plane-secrets +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end}} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/service.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/service.yaml new file mode 100644 index 00000000..f6ec4951 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/service.yaml @@ -0,0 +1,56 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.prometheus.enabled }} +{{- if .Values.prometheus.serviceMonitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: metrics + port: {{ .Values.prometheus.port }} + protocol: TCP + targetPort: prometheus + {{- if not .Values.envoy.enabled }} + - name: envoy-metrics + port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + targetPort: envoy-metrics + {{- end }} +{{- else if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: {{ .Release.Namespace }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: envoy-metrics + port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + targetPort: envoy-metrics +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/serviceaccount.yaml new file mode 100644 index 00000000..2c2cf399 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.cilium.annotations .Values.annotations }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.cilium.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-agent/servicemonitor.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-agent/servicemonitor.yaml new file mode 100644 index 00000000..c2ffa66c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-agent/servicemonitor.yaml @@ -0,0 +1,61 @@ +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.prometheus.enabled .Values.prometheus.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cilium-agent + namespace: {{ .Values.prometheus.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.prometheus.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.prometheus.serviceMonitor.annotations .Values.annotations }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: cilium + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: metrics + interval: {{ .Values.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.envoy.prometheus.serviceMonitor.enabled }} + - port: envoy-metrics + interval: {{ .Values.envoy.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.envoy.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.envoy.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + targetLabels: + - k8s-app +{{- if .Values.prometheus.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.prometheus.serviceMonitor.jobLabel | quote }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-ca-bundle-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-ca-bundle-configmap.yaml new file mode 100644 index 00000000..38522458 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-ca-bundle-configmap.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.tls.caBundle.enabled .Values.tls.caBundle.content -}} +--- +apiVersion: v1 +kind: {{ .Values.tls.caBundle.useSecret | ternary "Secret" "ConfigMap" }} +metadata: + name: {{ .Values.tls.caBundle.name }} + namespace: {{ .Release.Namespace }} +{{ .Values.tls.caBundle.useSecret | ternary "stringData" "data" }}: + {{ .Values.tls.caBundle.key }}: | + {{- .Values.tls.caBundle.content | nindent 4 }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-ca-secret.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-ca-secret.yaml new file mode 100644 index 00000000..f2225b93 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-ca-secret.yaml @@ -0,0 +1,17 @@ +{{- if or + (and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm")) + (and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm")) + (and .Values.tls.ca.key .Values.tls.ca.cert) +-}} + +{{- $_ := include "cilium.ca.setup" . -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .commonCASecretName }} + namespace: {{ .Release.Namespace }} +data: + ca.crt: {{ .commonCA.Cert | b64enc }} + ca.key: {{ .commonCA.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-configmap.yaml new file mode 100644 index 00000000..eba5bc4d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-configmap.yaml @@ -0,0 +1,1253 @@ +{{- if and (.Values.agent) (not .Values.preflight.enabled) }} +{{- /* Default values with backwards compatibility */ -}} +{{- $defaultBpfMapDynamicSizeRatio := 0.0 -}} +{{- $defaultBpfMasquerade := "false" -}} +{{- $defaultBpfClockProbe := "false" -}} +{{- $defaultBpfTProxy := "false" -}} +{{- $defaultIPAM := "cluster-pool" -}} +{{- $defaultOperatorApiServeAddr := "localhost:9234" -}} +{{- $defaultBpfCtTcpMax := 524288 -}} +{{- $defaultBpfCtAnyMax := 262144 -}} +{{- $enableIdentityMark := "true" -}} +{{- $fragmentTracking := "true" -}} +{{- $defaultKubeProxyReplacement := "false" -}} +{{- $azureUsePrimaryAddress := "true" -}} +{{- $defaultK8sClientQPS := 5 -}} +{{- $defaultK8sClientBurst := 10 -}} +{{- $defaultDNSProxyEnableTransparentMode := "false" -}} + +{{- /* Default values when 1.8 was initially deployed */ -}} +{{- if semverCompare ">=1.8" (default "1.8" .Values.upgradeCompatibility) -}} + {{- $defaultBpfMapDynamicSizeRatio = 0.0025 -}} + {{- $defaultBpfMasquerade = "true" -}} + {{- $defaultBpfClockProbe = "true" -}} + {{- $defaultIPAM = "cluster-pool" -}} + {{- if .Values.ipv4.enabled }} + {{- $defaultOperatorApiServeAddr = "127.0.0.1:9234" -}} + {{- else -}} + {{- $defaultOperatorApiServeAddr = "[::1]:9234" -}} + {{- end }} + {{- $defaultBpfCtTcpMax = 0 -}} + {{- $defaultBpfCtAnyMax = 0 -}} + {{- $defaultKubeProxyReplacement = "probe" -}} +{{- end -}} + +{{- /* Default values when 1.9 was initially deployed */ -}} +{{- if semverCompare ">=1.9" (default "1.9" .Values.upgradeCompatibility) -}} + {{- $defaultKubeProxyReplacement = "probe" -}} +{{- end -}} + +{{- /* Default values when 1.10 was initially deployed */ -}} +{{- if semverCompare ">=1.10" (default "1.10" .Values.upgradeCompatibility) -}} + {{- /* Needs to be explicitly disabled because it was enabled on all versions >=v1.8 above. */ -}} + {{- $defaultBpfMasquerade = "false" -}} +{{- end -}} + +{{- /* Default values when 1.12 was initially deployed */ -}} +{{- if semverCompare ">=1.12" (default "1.12" .Values.upgradeCompatibility) -}} + {{- if .Values.azure.enabled }} + {{- $azureUsePrimaryAddress = "false" -}} + {{- end }} + {{- $defaultKubeProxyReplacement = "disabled" -}} + {{- $defaultDNSProxyEnableTransparentMode = "true" -}} +{{- end -}} + +{{- /* Default values when 1.14 was initially deployed */ -}} +{{- if semverCompare ">=1.14" (default "1.14" .Values.upgradeCompatibility) -}} + {{- /* KPR default for 1.14 needed to override earlier version defaults set above when upgradeCompatibility is not specified */ -}} + {{- $defaultKubeProxyReplacement = "false" -}} +{{- end -}} + +{{- $ipam := (coalesce .Values.ipam.mode $defaultIPAM) -}} +{{- $bpfCtTcpMax := (coalesce .Values.bpf.ctTcpMax $defaultBpfCtTcpMax) -}} +{{- $bpfCtAnyMax := (coalesce .Values.bpf.ctAnyMax $defaultBpfCtAnyMax) -}} +{{- $stringValueKPR := (toString .Values.kubeProxyReplacement) -}} +{{- if (eq $stringValueKPR "") }} + {{- $stringValueKPR = "" -}} +{{- end}} +{{- $kubeProxyReplacement := (coalesce $stringValueKPR $defaultKubeProxyReplacement) -}} +{{- if and (ne $kubeProxyReplacement "disabled") (ne $kubeProxyReplacement "partial") (ne $kubeProxyReplacement "strict") (ne $kubeProxyReplacement "true") (ne $kubeProxyReplacement "false") }} + {{ fail "kubeProxyReplacement must be explicitly set to a valid value (true, false, disabled (deprecated), partial (deprecated), or strict (deprecated)) to continue." }} +{{- end }} +{{- $azureUsePrimaryAddress = (coalesce .Values.azure.usePrimaryAddress $azureUsePrimaryAddress) -}} +{{- $socketLB := (coalesce .Values.socketLB .Values.hostServices) -}} + +{{- $cniChainingMode := "none" -}} +{{- if (not (kindIs "invalid" .Values.cni.chainingMode)) -}} + {{- $cniChainingMode = .Values.cni.chainingMode -}} +{{- else if (not (kindIs "invalid" .Values.cni.chainingTarget)) -}} + {{- $cniChainingMode = "generic-veth" -}} +{{- end -}} + +{{- if semverCompare ">=1.27-0" .Capabilities.KubeVersion.Version -}} + {{- $defaultK8sClientQPS = 10 -}} + {{- $defaultK8sClientBurst = 20 -}} +{{- end -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: {{ .Release.Namespace }} +data: +{{- if .Values.etcd.enabled }} + # The kvstore configuration is used to enable use of a kvstore for state + # storage. This can either be provided with an external kvstore or with the + # help of cilium-etcd-operator which operates an etcd cluster automatically. + kvstore: etcd + {{- if .Values.etcd.k8sService }} + kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config", "etcd.operator": "true"}' + {{- else }} + kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}' + {{- end }} + + # This etcd-config contains the etcd endpoints of your cluster. If you use + # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config + etcd-config: |- + --- + endpoints: + {{- if .Values.etcd.managed }} + - https://cilium-etcd-client.{{ .Release.Namespace }}.svc:2379 + {{- else }} + {{- range .Values.etcd.endpoints }} + - {{ . }} + {{- end }} + {{- end }} + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + trusted-ca-file: '/var/lib/etcd-secrets/etcd-client-ca.crt' + key-file: '/var/lib/etcd-secrets/etcd-client.key' + cert-file: '/var/lib/etcd-secrets/etcd-client.crt' + {{- end }} +{{- end }} + +{{- if .Values.conntrackGCInterval }} + conntrack-gc-interval: {{ include "validateDuration" .Values.conntrackGCInterval | quote }} +{{- end }} + +{{- if .Values.conntrackGCMaxInterval }} + conntrack-gc-max-interval: {{ include "validateDuration" .Values.conntrackGCMaxInterval | quote }} +{{- end }} + +{{- if hasKey .Values "disableEnvoyVersionCheck" }} + disable-envoy-version-check: {{ .Values.disableEnvoyVersionCheck | quote }} +{{- end }} + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: {{ .Values.identityAllocationMode }} + identity-heartbeat-timeout: {{ include "validateDuration" .Values.operator.identityHeartbeatTimeout | quote }} + identity-gc-interval: {{ include "validateDuration" .Values.operator.identityGCInterval | quote }} + cilium-endpoint-gc-interval: {{ include "validateDuration" .Values.operator.endpointGCInterval | quote }} + nodes-gc-interval: {{ include "validateDuration" .Values.operator.nodeGCInterval | quote }} + +{{- if hasKey .Values.operator "skipCNPStatusStartupClean" }} + skip-cnp-status-startup-clean: "{{ .Values.operator.skipCNPStatusStartupClean }}" +{{- end }} + +{{- if eq .Values.disableEndpointCRD true }} + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "true" +{{- end }} + +{{- if .Values.identityChangeGracePeriod }} + # identity-change-grace-period is the grace period that needs to pass + # before an endpoint that has changed its identity will start using + # that new identity. During the grace period, the new identity has + # already been allocated and other nodes in the cluster have a chance + # to whitelist the new upcoming identity of the endpoint. + identity-change-grace-period: {{ include "validateDuration" .Values.identityChangeGracePeriod | quote }} +{{- end }} + +{{- if hasKey .Values "labels" }} + # To include or exclude matched resources from cilium identity evaluation + labels: {{ .Values.labels | quote }} +{{- end }} + + # If you want to run cilium in debug mode change this value to true + debug: {{ .Values.debug.enabled | quote }} + +{{- if hasKey .Values.debug "verbose" }} + debug-verbose: "{{ .Values.debug.verbose }}" +{{- end }} + +{{- if ne (int .Values.healthPort) 9879 }} + # Set the TCP port for the agent health status API. This is not the port used + # for cilium-health. + agent-health-port: "{{ .Values.healthPort }}" +{{- end }} + +{{- if hasKey .Values "clusterHealthPort" }} + # Set the TCP port for the agent health API. This port is used for cilium-health. + cluster-health-port: "{{ .Values.clusterHealthPort }}" +{{- end }} + +{{- if hasKey .Values "policyEnforcementMode" }} + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "{{ lower .Values.policyEnforcementMode }}" +{{- end }} + +{{- if hasKey .Values "policyCIDRMatchMode" }} + policy-cidr-match-mode: {{ join " " .Values.policyCIDRMatchMode | quote }} +{{- end}} + + +{{- if .Values.prometheus.enabled }} + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":{{ .Values.prometheus.port }}" + {{- if .Values.prometheus.metrics }} + # Metrics that should be enabled or disabled from the default metric + # list. (+metric_foo to enable metric_foo , -metric_bar to disable + # metric_bar). + metrics: {{- range .Values.prometheus.metrics }} + {{ . }} + {{- end }} + {{- end }} + {{- if .Values.prometheus.controllerGroupMetrics }} + # A space-separated list of controller groups for which to enable metrics. + # The special values of "all" and "none" are supported. + controller-group-metrics: {{- range .Values.prometheus.controllerGroupMetrics }} + {{ . }} + {{- end }} + {{- end }} +{{- end }} + +{{- if not .Values.envoy.enabled }} + # Port to expose Envoy metrics (e.g. "9964"). Envoy metrics listener will be disabled if this + # field is not set. + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} + proxy-prometheus-port: "{{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}" + {{- end }} +{{- end }} + +{{- if .Values.operator.prometheus.enabled }} + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":{{ .Values.operator.prometheus.port }}" + enable-metrics: "true" +{{- end }} + +{{- if .Values.operator.skipCRDCreation }} + skip-crd-creation: "true" +{{- end }} + +{{- if or .Values.envoyConfig.enabled .Values.ingressController.enabled .Values.gatewayAPI.enabled (and (hasKey .Values "loadBalancer") (eq .Values.loadBalancer.l7.backend "envoy")) }} + enable-envoy-config: "true" + {{- if .Values.envoyConfig.enabled }} + envoy-secrets-namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + {{- end }} +{{- end }} + +{{- if .Values.ingressController.enabled }} + enable-ingress-controller: "true" + enforce-ingress-https: {{ .Values.ingressController.enforceHttps | quote }} + enable-ingress-proxy-protocol: {{ .Values.ingressController.enableProxyProtocol | quote }} + enable-ingress-secrets-sync: {{ .Values.ingressController.secretsNamespace.sync | quote }} + ingress-secrets-namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + ingress-lb-annotation-prefixes: {{ .Values.ingressController.ingressLBAnnotationPrefixes | join " " | quote }} + ingress-default-lb-mode: {{ .Values.ingressController.loadbalancerMode }} + ingress-shared-lb-service-name: {{ .Values.ingressController.service.name }} + {{- if and .Values.ingressController.defaultSecretNamespace .Values.ingressController.defaultSecretName }} + ingress-default-secret-namespace: {{ .Values.ingressController.defaultSecretNamespace | quote }} + ingress-default-secret-name: {{ .Values.ingressController.defaultSecretName | quote }} + {{- end }} +{{- end }} + +{{- if .Values.gatewayAPI.enabled }} + enable-gateway-api: "true" + enable-gateway-api-secrets-sync: {{ .Values.gatewayAPI.secretsNamespace.sync | quote }} + gateway-api-secrets-namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} +{{- end }} + +{{- if hasKey .Values "loadBalancer" }} +{{- if eq .Values.loadBalancer.l7.backend "envoy" }} + loadbalancer-l7: "envoy" + loadbalancer-l7-ports: {{ .Values.loadBalancer.l7.ports | join " " | quote }} + loadbalancer-l7-algorithm: {{ .Values.loadBalancer.l7.algorithm | quote }} +{{- end }} +{{- end }} + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: {{ .Values.ipv4.enabled | quote }} + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: {{ .Values.ipv6.enabled | quote }} + +{{- if .Values.cleanState }} + # If a serious issue occurs during Cilium startup, this + # invasive option may be set to true to remove all persistent + # state. Endpoints will not be restored using knowledge from a + # prior Cilium run, so they may receive new IP addresses upon + # restart. This also triggers clean-cilium-bpf-state. + clean-cilium-state: "true" +{{- end }} + +{{- if .Values.cleanBpfState }} + # If you want to clean cilium BPF state, set this to true; + # Removes all BPF maps from the filesystem. Upon restart, + # endpoints are restored with the same IP addresses, however + # any ongoing connections may be disrupted briefly. + # Loadbalancing decisions will be reset, so any ongoing + # connections via a service may be loadbalanced to a different + # backend after restart. + clean-cilium-bpf-state: "true" +{{- end }} + +{{- if hasKey .Values.cni "customConf" }} + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "{{ .Values.cni.customConf }}" +{{- end }} + +{{- if hasKey .Values "bpfClockProbe" }} + enable-bpf-clock-probe: {{ .Values.bpfClockProbe | quote }} +{{- else if eq $defaultBpfClockProbe "true" }} + enable-bpf-clock-probe: {{ $defaultBpfClockProbe | quote }} +{{- end }} + +{{- if (not (kindIs "invalid" .Values.bpf.tproxy)) }} + enable-bpf-tproxy: {{ .Values.bpf.tproxy | quote }} +{{- else if eq $defaultBpfTProxy "true" }} + enable-bpf-tproxy: {{ $defaultBpfTProxy | quote }} +{{- end }} + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: {{ .Values.bpf.monitorAggregation }} + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: {{ include "validateDuration" .Values.bpf.monitorInterval | quote }} + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: {{ .Values.bpf.monitorFlags }} + + + + +{{- if .Values.bpf.mapDynamicSizeRatio }} + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: {{ .Values.bpf.mapDynamicSizeRatio | quote }} +{{- else if ne $defaultBpfMapDynamicSizeRatio 0.0 }} + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: {{ $defaultBpfMapDynamicSizeRatio | quote }} +{{- end }} + +{{- if (not (kindIs "invalid" .Values.bpf.hostLegacyRouting)) }} + enable-host-legacy-routing: {{ .Values.bpf.hostLegacyRouting | quote }} +{{- else if ne $cniChainingMode "none" }} + # In cni chaining mode, the other chained plugin is responsible for underlying connectivity, + # so cilium eBPF host routing shoud not work, and let it fall back to the legacy routing mode + enable-host-legacy-routing: "true" +{{- end }} + +{{- if .Values.bpf.authMapMax }} + # bpf-auth-map-max specifies the maximum number of entries in the auth map + bpf-auth-map-max: {{ .Values.bpf.authMapMax | quote }} +{{- end }} +{{- if or $bpfCtTcpMax $bpfCtAnyMax }} + # bpf-ct-global-*-max specifies the maximum number of connections + # supported across all endpoints, split by protocol: tcp or other. One pair + # of maps uses these values for IPv4 connections, and another pair of maps + # use these values for IPv6 connections. + # + # If these values are modified, then during the next Cilium startup the + # tracking of ongoing connections may be disrupted. As a result, reply + # packets may be dropped and the load-balancing decisions for established + # connections may change. + # + # For users upgrading from Cilium 1.2 or earlier, to minimize disruption + # during the upgrade process, set bpf-ct-global-tcp-max to 1000000. +{{- if $bpfCtTcpMax }} + bpf-ct-global-tcp-max: {{ $bpfCtTcpMax | quote }} +{{- end }} +{{- if $bpfCtAnyMax }} + bpf-ct-global-any-max: {{ $bpfCtAnyMax | quote }} +{{- end }} +{{- end }} +{{- if .Values.bpf.natMax }} + # bpf-nat-global-max specified the maximum number of entries in the + # BPF NAT table. + bpf-nat-global-max: "{{ .Values.bpf.natMax }}" +{{- end }} +{{- if .Values.bpf.neighMax }} + # bpf-neigh-global-max specified the maximum number of entries in the + # BPF neighbor table. + bpf-neigh-global-max: "{{ .Values.bpf.neighMax }}" +{{- end }} +{{- if hasKey .Values.bpf "policyMapMax" }} + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "{{ .Values.bpf.policyMapMax }}" +{{- end }} +{{- if hasKey .Values.bpf "lbMapMax" }} + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "{{ .Values.bpf.lbMapMax }}" +{{- end }} +{{- if hasKey .Values.bpf "lbExternalClusterIP" }} + bpf-lb-external-clusterip: {{ .Values.bpf.lbExternalClusterIP | quote }} +{{- end }} + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "{{ .Values.bpf.preallocateMaps }}" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "{{ .Values.proxy.sidecarImageRegex }}" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: {{ .Values.cluster.name }} + +{{- if hasKey .Values.cluster "id" }} + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "{{ .Values.cluster.id }}" +{{- end }} + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve +{{- if .Values.gke.enabled }} + {{- if ne (.Values.routingMode | default "native") "native" }} + {{- fail (printf "RoutingMode must be set to native when gke.enabled=true" )}} + {{- end }} + routing-mode: "native" + enable-endpoint-routes: "true" +{{- else if .Values.aksbyocni.enabled }} + {{- if ne (.Values.routingMode | default "tunnel") "tunnel" }} + {{- fail (printf "RoutingMode must be set to tunnel when aksbyocni.enabled=true" )}} + {{- end }} + routing-mode: "tunnel" + tunnel-protocol: "vxlan" +{{- else if .Values.routingMode }} + routing-mode: {{ .Values.routingMode | quote }} +{{- else }} + # Default case + routing-mode: "tunnel" + tunnel-protocol: "vxlan" +{{- end }} + +{{- if .Values.tunnelProtocol }} + tunnel-protocol: {{ .Values.tunnelProtocol | quote }} +{{- end }} + +{{- if .Values.tunnelPort }} + tunnel-port: {{ .Values.tunnelPort | quote }} +{{- end }} + +{{- if .Values.serviceNoBackendResponse }} + service-no-backend-response: "{{ .Values.serviceNoBackendResponse }}" +{{- end}} + +{{- if .Values.MTU }} + mtu: {{ .Values.MTU | quote }} +{{- end }} + +{{- if .Values.eni.enabled }} + enable-endpoint-routes: "true" + auto-create-cilium-node-resource: "true" +{{- if .Values.eni.updateEC2AdapterLimitViaAPI }} + update-ec2-adapter-limit-via-api: "true" +{{- end }} +{{- if .Values.eni.awsReleaseExcessIPs }} + aws-release-excess-ips: "true" +{{- end }} +{{- if .Values.eni.awsEnablePrefixDelegation }} + aws-enable-prefix-delegation: "true" +{{- end }} + ec2-api-endpoint: {{ .Values.eni.ec2APIEndpoint | quote }} + eni-tags: {{ .Values.eni.eniTags | toRawJson | quote }} +{{- if .Values.eni.subnetIDsFilter }} + subnet-ids-filter: {{ .Values.eni.subnetIDsFilter | join " " | quote }} +{{- end }} +{{- if .Values.eni.subnetTagsFilter }} + subnet-tags-filter: {{ .Values.eni.subnetTagsFilter | join " " | quote }} +{{- end }} +{{- if .Values.eni.instanceTagsFilter }} + instance-tags-filter: {{ .Values.eni.instanceTagsFilter | join " " | quote }} +{{- end }} +{{- end }} +{{ if .Values.eni.gcInterval }} + eni-gc-interval: {{ .Values.eni.gcInterval | quote }} +{{- end }} +{{ if .Values.eni.gcTags }} + eni-gc-tags: {{ .Values.eni.gcTags | toRawJson | quote }} +{{- end }} + +{{- if .Values.azure.enabled }} + enable-endpoint-routes: "true" + auto-create-cilium-node-resource: "true" +{{- if .Values.azure.userAssignedIdentityID }} + azure-user-assigned-identity-id: {{ .Values.azure.userAssignedIdentityID | quote }} +{{- end }} + azure-use-primary-address: {{ $azureUsePrimaryAddress | quote }} +{{- end }} + +{{- if .Values.alibabacloud.enabled }} + enable-endpoint-routes: "true" + auto-create-cilium-node-resource: "true" +{{- end }} + +{{- if hasKey .Values "l7Proxy" }} + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: {{ .Values.l7Proxy | quote }} +{{- end }} + +{{- if ne $cniChainingMode "none" }} + # Enable chaining with another CNI plugin + # + # Supported modes: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap (Enables HostPort support for Cilium) + cni-chaining-mode: {{ $cniChainingMode }} + +{{- if hasKey .Values "enableIdentityMark" }} + enable-identity-mark: {{ .Values.enableIdentityMark | quote }} +{{- else if (ne $enableIdentityMark "true") }} + enable-identity-mark: "false" +{{- end }} +{{- if ne $cniChainingMode "portmap" }} + # Disable the PodCIDR route to the cilium_host interface as it is not + # required. While chaining, it is the responsibility of the underlying plugin + # to enable routing. + enable-local-node-route: "false" +{{- end }} +{{- end }} + + enable-ipv4-masquerade: {{ .Values.enableIPv4Masquerade | quote }} + enable-ipv4-big-tcp: {{ .Values.enableIPv4BIGTCP | quote }} + enable-ipv6-big-tcp: {{ .Values.enableIPv6BIGTCP | quote }} + enable-ipv6-masquerade: {{ .Values.enableIPv6Masquerade | quote }} + +{{- if (not (kindIs "invalid" .Values.bpf.masquerade)) }} + enable-bpf-masquerade: {{ .Values.bpf.masquerade | quote }} +{{- else if eq $defaultBpfMasquerade "true" }} + enable-bpf-masquerade: {{ $defaultBpfMasquerade | quote }} +{{- end }} + enable-masquerade-to-route-source: {{ .Values.enableMasqueradeRouteSource | quote }} +{{- if hasKey .Values "egressMasqueradeInterfaces" }} + egress-masquerade-interfaces: {{ .Values.egressMasqueradeInterfaces }} +{{- end }} +{{- if and .Values.ipMasqAgent .Values.ipMasqAgent.enabled }} + enable-ip-masq-agent: "true" +{{- end }} + +{{- if .Values.encryption.enabled }} + {{- if eq .Values.encryption.type "ipsec" }} + enable-ipsec: {{ .Values.encryption.enabled | quote }} + + {{- if and .Values.encryption.ipsec.mountPath .Values.encryption.ipsec.keyFile }} + ipsec-key-file: {{ .Values.encryption.ipsec.mountPath }}/{{ .Values.encryption.ipsec.keyFile }} + {{- else }} + ipsec-key-file: {{ .Values.encryption.mountPath }}/{{ .Values.encryption.keyFile }} + {{- end }} + {{- if .Values.encryption.ipsec.interface }} + encrypt-interface: {{ .Values.encryption.ipsec.interface }} + {{- else if .Values.encryption.interface }} + encrypt-interface: {{ .Values.encryption.interface }} + {{- end }} + {{- if hasKey .Values.encryption.ipsec "keyWatcher" }} + enable-ipsec-key-watcher: {{ .Values.encryption.ipsec.keyWatcher | quote }} + {{- end }} + {{- if .Values.encryption.ipsec.keyRotationDuration }} + ipsec-key-rotation-duration: {{ include "validateDuration" .Values.encryption.ipsec.keyRotationDuration | quote }} + {{- end }} + {{- else if eq .Values.encryption.type "wireguard" }} + enable-wireguard: {{ .Values.encryption.enabled | quote }} + {{- if .Values.encryption.wireguard.userspaceFallback }} + enable-wireguard-userspace-fallback: {{ .Values.encryption.wireguard.userspaceFallback | quote }} + {{- end }} + {{- if .Values.encryption.wireguard.persistentKeepalive }} + wireguard-persistent-keepalive: {{ .Values.encryption.wireguard.persistentKeepalive | quote }} + {{- end }} + {{- end }} + {{- if .Values.encryption.nodeEncryption }} + encrypt-node: {{ .Values.encryption.nodeEncryption | quote }} + {{- end }} +{{- end }} + +{{- if .Values.encryption.strictMode.enabled }} + enable-encryption-strict-mode: {{ .Values.encryption.strictMode.enabled | quote }} + + encryption-strict-mode-cidr: {{ .Values.encryption.strictMode.cidr | quote }} + + encryption-strict-mode-allow-remote-node-identities: {{ .Values.encryption.strictMode.allowRemoteNodeIdentities | quote }} +{{- end }} + + enable-xt-socket-fallback: {{ .Values.enableXTSocketFallback | quote }} +{{- if or (.Values.azure.enabled) (.Values.eni.enabled) (.Values.gke.enabled) (ne $cniChainingMode "none") }} + install-no-conntrack-iptables-rules: "false" +{{- else }} + install-no-conntrack-iptables-rules: {{ .Values.installNoConntrackIptablesRules | quote }} +{{- end}} + +{{- if hasKey .Values "iptablesRandomFully" }} + iptables-random-fully: {{ .Values.iptablesRandomFully | quote }} +{{- end }} + +{{- if hasKey .Values "iptablesLockTimeout" }} + iptables-lock-timeout: {{ .Values.iptablesLockTimeout | quote }} +{{- end }} + + auto-direct-node-routes: {{ .Values.autoDirectNodeRoutes | quote }} + +{{- if hasKey .Values "bandwidthManager" }} +{{- if .Values.bandwidthManager.enabled }} + enable-bandwidth-manager: {{ .Values.bandwidthManager.enabled | quote }} + enable-bbr: {{ .Values.bandwidthManager.bbr | quote }} +{{- end }} +{{- end }} + +{{- if .Values.highScaleIPcache.enabled }} + enable-high-scale-ipcache: {{ .Values.highScaleIPcache.enabled | quote }} +{{- end }} + +{{- if hasKey .Values "localRedirectPolicy" }} + enable-local-redirect-policy: {{ .Values.localRedirectPolicy | quote }} +{{- end }} + +{{- if .Values.ipv4NativeRoutingCIDR }} + ipv4-native-routing-cidr: {{ .Values.ipv4NativeRoutingCIDR }} +{{- end }} + +{{- if .Values.ipv6NativeRoutingCIDR }} + ipv6-native-routing-cidr: {{ .Values.ipv6NativeRoutingCIDR }} +{{- end }} + +{{- if hasKey .Values "fragmentTracking" }} + enable-ipv4-fragment-tracking: {{ .Values.fragmentTracking | quote }} +{{- else if (ne $fragmentTracking "true") }} + enable-ipv4-fragment-tracking: "false" +{{- end }} + +{{- if .Values.nat46x64Gateway.enabled }} + enable-nat46x64-gateway: {{ .Values.nat46x64Gateway.enabled | quote }} +{{- end }} + +{{- if and .Values.hostFirewall .Values.hostFirewall.enabled }} + enable-host-firewall: {{ .Values.hostFirewall.enabled | quote }} +{{- end}} + +{{- if hasKey .Values "devices" }} + # List of devices used to attach bpf_host.o (implements BPF NodePort, + # host-firewall and BPF masquerading) + devices: {{ join " " .Values.devices | quote }} +{{- end }} + +{{- if .Values.enableRuntimeDeviceDetection }} + enable-runtime-device-detection: "true" +{{- end }} + + kube-proxy-replacement: {{ $kubeProxyReplacement | quote }} + +{{- if ne $kubeProxyReplacement "disabled" }} + kube-proxy-replacement-healthz-bind-address: {{ default "" .Values.kubeProxyReplacementHealthzBindAddr | quote}} +{{- end }} + +{{- if $socketLB }} +{{- if hasKey $socketLB "enabled" }} + bpf-lb-sock: {{ $socketLB.enabled | quote }} +{{- end }} +{{- if hasKey $socketLB "hostNamespaceOnly" }} + bpf-lb-sock-hostns-only: {{ $socketLB.hostNamespaceOnly | quote }} +{{- end }} +{{- end }} + +{{- if hasKey .Values "hostPort" }} +{{- if or (eq $kubeProxyReplacement "partial") (eq $kubeProxyReplacement "false") }} + enable-host-port: {{ .Values.hostPort.enabled | quote }} +{{- end }} +{{- end }} +{{- if hasKey .Values "externalIPs" }} +{{- if or (eq $kubeProxyReplacement "partial") (eq $kubeProxyReplacement "false") }} + enable-external-ips: {{ .Values.externalIPs.enabled | quote }} +{{- end }} +{{- end }} +{{- if hasKey .Values "nodePort" }} +{{- if or (eq $kubeProxyReplacement "partial") (eq $kubeProxyReplacement "false") }} + enable-node-port: {{ .Values.nodePort.enabled | quote }} +{{- end }} +{{- if hasKey .Values.nodePort "range" }} + node-port-range: {{ get .Values.nodePort "range" | quote }} +{{- end }} +{{- if hasKey .Values.nodePort "directRoutingDevice" }} + direct-routing-device: {{ .Values.nodePort.directRoutingDevice | quote }} +{{- end }} +{{- if hasKey .Values.nodePort "enableHealthCheck" }} + enable-health-check-nodeport: {{ .Values.nodePort.enableHealthCheck | quote}} +{{- end }} +{{- if .Values.gke.enabled }} + enable-health-check-loadbalancer-ip: "true" +{{- else if hasKey .Values.nodePort "enableHealthCheckLoadBalancerIP" }} + enable-health-check-loadbalancer-ip: {{ .Values.nodePort.enableHealthCheckLoadBalancerIP | quote}} +{{- end }} + node-port-bind-protection: {{ .Values.nodePort.bindProtection | quote }} + enable-auto-protect-node-port-range: {{ .Values.nodePort.autoProtectPortRange | quote }} +{{- end }} +{{- if hasKey .Values "loadBalancer" }} +{{- if .Values.loadBalancer.standalone }} + datapath-mode: lb-only +{{- end }} +{{- if hasKey .Values.loadBalancer "mode" }} + bpf-lb-mode: {{ .Values.loadBalancer.mode | quote }} +{{- end }} +{{- if hasKey .Values.loadBalancer "algorithm" }} + bpf-lb-algorithm: {{ .Values.loadBalancer.algorithm | quote }} +{{- end }} +{{- if hasKey .Values.loadBalancer "acceleration" }} + bpf-lb-acceleration: {{ .Values.loadBalancer.acceleration | quote }} +{{- end }} +{{- if hasKey .Values.loadBalancer "dsrDispatch" }} + bpf-lb-dsr-dispatch: {{ .Values.loadBalancer.dsrDispatch | quote }} +{{- end }} +{{- if hasKey .Values.loadBalancer "serviceTopology" }} + enable-service-topology: {{ .Values.loadBalancer.serviceTopology | quote }} +{{- end }} + +{{- end }} +{{- if hasKey .Values.maglev "tableSize" }} + bpf-lb-maglev-table-size: {{ .Values.maglev.tableSize | quote}} +{{- end }} +{{- if hasKey .Values.maglev "hashSeed" }} + bpf-lb-maglev-hash-seed: {{ .Values.maglev.hashSeed | quote}} +{{- end }} +{{- if .Values.sessionAffinity }} + enable-session-affinity: {{ .Values.sessionAffinity | quote }} +{{- end }} +{{- if .Values.svcSourceRangeCheck }} + enable-svc-source-range-check: {{ .Values.svcSourceRangeCheck | quote }} +{{- end }} + +{{- if hasKey .Values "l2NeighDiscovery" }} +{{- if hasKey .Values.l2NeighDiscovery "enabled" }} + enable-l2-neigh-discovery: {{ .Values.l2NeighDiscovery.enabled | quote }} +{{- end }} + arping-refresh-period: {{ include "validateDuration" .Values.l2NeighDiscovery.refreshPeriod | quote }} +{{- end }} + +{{- if .Values.pprof.enabled }} + pprof: {{ .Values.pprof.enabled | quote }} + pprof-address: {{ .Values.pprof.address | quote }} + pprof-port: {{ .Values.pprof.port | quote }} +{{- end }} + +{{- if .Values.operator.pprof.enabled }} + operator-pprof: {{ .Values.operator.pprof.enabled | quote }} + operator-pprof-address: {{ .Values.operator.pprof.address | quote }} + operator-pprof-port: {{ .Values.operator.pprof.port | quote }} +{{- end }} + +{{- if .Values.logSystemLoad }} + log-system-load: {{ .Values.logSystemLoad | quote }} +{{- end }} +{{- if .Values.logOptions }} + log-opt: {{ .Values.logOptions | toJson | quote }} +{{- end }} +{{- if hasKey .Values.k8s "requireIPv4PodCIDR" }} + k8s-require-ipv4-pod-cidr: {{ .Values.k8s.requireIPv4PodCIDR | quote }} +{{- end }} +{{- if hasKey .Values.k8s "requireIPv6PodCIDR" }} + k8s-require-ipv6-pod-cidr: {{ .Values.k8s.requireIPv6PodCIDR | quote }} +{{- end }} +{{- if .Values.endpointStatus.enabled }} + endpoint-status: {{ required "endpointStatus.status required: policy, health, controllers, log and / or state. For 2 or more options use a space: \"policy health\"" .Values.endpointStatus.status | quote }} +{{- end }} +{{- if and .Values.endpointRoutes .Values.endpointRoutes.enabled }} + enable-endpoint-routes: {{ .Values.endpointRoutes.enabled | quote }} +{{- end }} +{{- if and .Values.k8sNetworkPolicy .Values.k8sNetworkPolicy.enabled }} + enable-k8s-networkpolicy: {{ .Values.k8sNetworkPolicy.enabled | quote }} +{{- end }} +{{- if .Values.cni.configMap }} + read-cni-conf: {{ .Values.cni.confFileMountPath }}/{{ .Values.cni.configMapKey }} +{{- if .Values.cni.customConf }} + # legacy: v1.13 and before needed cni.customConf: true with cni.configMap + write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/05-cilium.conflist +{{- end }} +{{- else if .Values.cni.readCniConf }} + read-cni-conf: {{ .Values.cni.readCniConf }} +{{- end }} +{{- if and (not .Values.cni.customConf) .Values.cni.install }} + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/05-cilium.conflist + cni-exclusive: {{ .Values.cni.exclusive | quote }} + cni-log-file: {{ .Values.cni.logFile | quote }} +{{- end }} +{{- if .Values.cni.uninstall }} + cni-uninstall: {{ .Values.cni.uninstall | quote }} +{{- end }} +{{- if (not (kindIs "invalid" .Values.cni.chainingTarget)) }} + cni-chaining-target: {{ .Values.cni.chainingTarget | quote }} +{{- end}} +{{- if (not (kindIs "invalid" .Values.cni.externalRouting)) }} + cni-external-routing: {{ .Values.cni.externalRouting | quote }} +{{- end}} +{{- if .Values.kubeConfigPath }} + k8s-kubeconfig-path: {{ .Values.kubeConfigPath | quote }} +{{- end }} +{{- if and ( .Values.endpointHealthChecking.enabled ) (or (eq $cniChainingMode "portmap") (eq $cniChainingMode "none")) }} + enable-endpoint-health-checking: "true" +{{- else}} + # Disable health checking, when chaining mode is not set to portmap or none + enable-endpoint-health-checking: "false" +{{- end }} +{{- if hasKey .Values "healthChecking" }} + enable-health-checking: {{ .Values.healthChecking | quote }} +{{- end }} +{{- if or .Values.wellKnownIdentities.enabled .Values.etcd.managed }} + enable-well-known-identities: "true" +{{- else }} + enable-well-known-identities: "false" +{{- end }} + enable-remote-node-identity: {{ .Values.remoteNodeIdentity | quote }} + +{{- if hasKey .Values "synchronizeK8sNodes" }} + synchronize-k8s-nodes: {{ .Values.synchronizeK8sNodes | quote }} +{{- end }} + +{{- if hasKey .Values "policyAuditMode" }} + policy-audit-mode: {{ .Values.policyAuditMode | quote }} +{{- end }} + +{{- if ne $defaultOperatorApiServeAddr "localhost:9234" }} + operator-api-serve-addr: {{ $defaultOperatorApiServeAddr | quote }} +{{- end }} + +{{- if .Values.hubble.enabled }} + # Enable Hubble gRPC service. + enable-hubble: {{ .Values.hubble.enabled | quote }} + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: {{ .Values.hubble.socketPath | quote }} +{{- if hasKey .Values.hubble "eventQueueSize" }} + # Buffer size of the channel for Hubble to receive monitor events. If this field is not set, + # the buffer size is set to the default monitor queue size. + hubble-event-queue-size: {{ .Values.hubble.eventQueueSize | quote }} +{{- end }} +{{- if hasKey .Values.hubble "eventBufferCapacity" }} + # Capacity of the buffer to store recent events. + hubble-event-buffer-capacity: {{ .Values.hubble.eventBufferCapacity | quote }} +{{- end }} +{{- if .Values.hubble.metrics.enabled }} + # Address to expose Hubble metrics (e.g. ":7070"). Metrics server will be disabled if this + # field is not set. + hubble-metrics-server: ":{{ .Values.hubble.metrics.port }}" + # A space separated list of metrics to enable. See [0] for available metrics. + # + # https://github.com/cilium/hubble/blob/master/Documentation/metrics.md + hubble-metrics: {{- range .Values.hubble.metrics.enabled }} + {{.}} +{{- end }} + enable-hubble-open-metrics: {{ .Values.hubble.metrics.enableOpenMetrics | quote }} +{{- end }} +{{- if .Values.hubble.redact }} +{{- if eq .Values.hubble.redact.enabled true }} + # Enables hubble redact capabilities + hubble-redact-enabled: "true" +{{- if .Values.hubble.redact.http }} + # Enables redaction of the http URL query part in flows + hubble-redact-http-urlquery: {{ .Values.hubble.redact.http.urlQuery | quote }} + # Enables redaction of the http user info in flows + hubble-redact-http-userinfo: {{ .Values.hubble.redact.http.userInfo | quote }} +{{- if .Values.hubble.redact.http.headers }} +{{- if .Values.hubble.redact.http.headers.allow }} + # Redact all http headers that do not match this list + hubble-redact-http-headers-allow: {{- range .Values.hubble.redact.http.headers.allow }} + {{ . }} +{{- end }} +{{- end }} +{{- if .Values.hubble.redact.http.headers.deny }} + # Redact all http headers that match this list + hubble-redact-http-headers-deny: {{- range .Values.hubble.redact.http.headers.deny }} + {{ . }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hubble.redact.kafka }} + # Enables redaction of the Kafka API key part in flows + hubble-redact-kafka-apikey: {{ .Values.hubble.redact.kafka.apiKey | quote }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hubble.export }} + hubble-export-file-max-size-mb: {{ .Values.hubble.export.fileMaxSizeMb | quote }} + hubble-export-file-max-backups: {{ .Values.hubble.export.fileMaxBackups | quote }} +{{- if .Values.hubble.export.static.enabled }} + hubble-export-file-path: {{ .Values.hubble.export.static.filePath | quote }} + hubble-export-fieldmask: {{ .Values.hubble.export.static.fieldMask | join " " | quote }} + hubble-export-allowlist: {{ .Values.hubble.export.static.allowList | join "," | quote }} + hubble-export-denylist: {{ .Values.hubble.export.static.denyList | join "," | quote }} +{{- end }} +{{- if .Values.hubble.export.dynamic.enabled }} + hubble-flowlogs-config-path: /flowlog-config/flowlogs.yaml +{{- end }} +{{- end }} +{{- if hasKey .Values.hubble "listenAddress" }} + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: {{ .Values.hubble.listenAddress | quote }} +{{- if .Values.hubble.tls.enabled }} + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt +{{- else }} + hubble-disable-tls: "true" +{{- end }} +{{- end }} +{{- if .Values.hubble.preferIpv6 }} + hubble-prefer-ipv6: "true" +{{- end }} +{{- if (not (kindIs "invalid" .Values.hubble.skipUnknownCGroupIDs)) }} + hubble-skip-unknown-cgroup-ids: {{ .Values.hubble.skipUnknownCGroupIDs | quote }} +{{- end }} +{{- end }} +{{- if hasKey .Values "disableIptablesFeederRules" }} + # A space separated list of iptables chains to disable when installing feeder rules. + disable-iptables-feeder-rules: {{ .Values.disableIptablesFeederRules | join " " | quote }} +{{- end }} +{{- if .Values.aksbyocni.enabled }} + ipam: "cluster-pool" +{{- else }} + ipam: {{ $ipam | quote }} +{{- end }} + +{{- if .Values.ipam.ciliumNodeUpdateRate }} + ipam-cilium-node-update-rate: {{ include "validateDuration" .Values.ipam.ciliumNodeUpdateRate | quote }} +{{- end }} + +{{- if (eq $ipam "cluster-pool") }} +{{- if .Values.ipv4.enabled }} + {{- if hasKey .Values.ipam.operator "clusterPoolIPv4PodCIDR" }} + {{- /* ipam.operator.clusterPoolIPv4PodCIDR removed in v1.14, remove this failsafe around v1.17 */ -}} + {{- fail "Value ipam.operator.clusterPoolIPv4PodCIDR removed, use ipam.operator.clusterPoolIPv4PodCIDRList instead" }} + {{- end }} + cluster-pool-ipv4-cidr: {{ .Values.ipam.operator.clusterPoolIPv4PodCIDRList | join " " | quote }} + cluster-pool-ipv4-mask-size: {{ .Values.ipam.operator.clusterPoolIPv4MaskSize | quote }} +{{- end }} +{{- if .Values.ipv6.enabled }} + {{- if hasKey .Values.ipam.operator "clusterPoolIPv6PodCIDR" }} + {{- /* ipam.operator.clusterPoolIPv6PodCIDR removed in v1.14, remove this failsafe around v1.17 */ -}} + {{- fail "Value ipam.operator.clusterPoolIPv6PodCIDR removed, use ipam.operator.clusterPoolIPv6PodCIDRList instead" }} + {{- end }} + cluster-pool-ipv6-cidr: {{ .Values.ipam.operator.clusterPoolIPv6PodCIDRList | join " " | quote }} + cluster-pool-ipv6-mask-size: {{ .Values.ipam.operator.clusterPoolIPv6MaskSize | quote }} +{{- end }} +{{- end }} +{{- if (eq $ipam "multi-pool") }} + {{- $pools := list }} + {{- range $pool, $spec := .Values.ipam.operator.autoCreateCiliumPodIPPools }} + {{- $attrs := list }} + {{- if hasKey $spec "ipv4" }} + {{- $attrs = append $attrs (printf "ipv4-cidrs:%s" (join "," $spec.ipv4.cidrs)) }} + {{- $attrs = append $attrs (printf "ipv4-mask-size:%s" (toString $spec.ipv4.maskSize)) }} + {{- end }} + {{- if hasKey $spec "ipv6" }} + {{- $attrs = append $attrs (printf "ipv6-cidrs:%s" (join "," $spec.ipv6.cidrs)) }} + {{- $attrs = append $attrs (printf "ipv6-mask-size:%s" (toString $spec.ipv6.maskSize)) }} + {{- end }} + {{- $pools = append $pools (printf "%s=%s" $pool (join ";" $attrs)) }} + {{- end }} + auto-create-cilium-pod-ip-pools: {{ join "," $pools | quote }} +{{- end }} + +{{- if .Values.ipam.operator.externalAPILimitBurstSize }} + limit-ipam-api-burst: {{ .Values.ipam.operator.externalAPILimitBurstSize | quote }} +{{- end }} +{{- if .Values.ipam.operator.externalAPILimitQPS }} + limit-ipam-api-qps: {{ .Values.ipam.operator.externalAPILimitQPS | quote }} +{{- end }} + +{{- if .Values.apiRateLimit }} + api-rate-limit: {{ .Values.apiRateLimit | quote }} +{{- end }} + +{{- if .Values.egressGateway.enabled }} + enable-ipv4-egress-gateway: "true" +{{- end }} +{{- if .Values.egressGateway.installRoutes }} + install-egress-gateway-routes: "true" +{{- end }} +{{- if hasKey .Values.egressGateway "reconciliationTriggerInterval" }} + egress-gateway-reconciliation-trigger-interval: {{ .Values.egressGateway.reconciliationTriggerInterval | quote }} +{{- end }} +{{- if .Values.egressGateway.maxPolicyEntries }} + egress-gateway-policy-map-max: {{ .Values.egressGateway.maxPolicyEntries }} +{{- end }} + +{{- if hasKey .Values "vtep" }} + enable-vtep: {{ .Values.vtep.enabled | quote }} +{{- if hasKey .Values.vtep "endpoint" }} + vtep-endpoint: {{ .Values.vtep.endpoint | quote }} +{{- end }} +{{- if hasKey .Values.vtep "cidr" }} + vtep-cidr: {{ .Values.vtep.cidr | quote }} +{{- end }} +{{- if hasKey .Values.vtep "mask" }} + vtep-mask: {{ .Values.vtep.mask | quote }} +{{- end }} +{{- if hasKey .Values.vtep "mac" }} + vtep-mac: {{ .Values.vtep.mac | quote }} +{{- end }} +{{- end }} + +{{- if .Values.crdWaitTimeout }} + crd-wait-timeout: {{ include "validateDuration" .Values.crdWaitTimeout | quote }} +{{- end }} + +{{- if .Values.enableK8sEndpointSlice }} + enable-k8s-endpoint-slice: {{ .Values.enableK8sEndpointSlice | quote }} +{{- end }} + +{{- if hasKey .Values.k8s "serviceProxyName" }} + # Configure service proxy name for Cilium. + k8s-service-proxy-name: {{ .Values.k8s.serviceProxyName | quote }} +{{- end }} + +{{- if and .Values.customCalls .Values.customCalls.enabled }} + # Enable tail call hooks for custom eBPF programs. + enable-custom-calls: {{ .Values.customCalls.enabled | quote }} +{{- end }} + +{{- if .Values.l2announcements.enabled }} + # Enable L2 announcements + enable-l2-announcements: {{ .Values.l2announcements.enabled | quote }} + {{- if .Values.l2announcements.leaseDuration }} + l2-announcements-lease-duration: {{ .Values.l2announcements.leaseDuration | quote }} + {{- end}} + {{- if .Values.l2announcements.leaseRenewDeadline }} + l2-announcements-renew-deadline: {{ .Values.l2announcements.leaseRenewDeadline | quote }} + {{- end}} + {{- if .Values.l2announcements.leaseRetryPeriod }} + l2-announcements-retry-period: {{ .Values.l2announcements.leaseRetryPeriod | quote }} + {{- end}} +{{- end}} + +{{- if .Values.l2podAnnouncements.enabled }} + enable-l2-pod-announcements: {{ .Values.l2podAnnouncements.enabled | quote }} + l2-pod-announcements-interface: {{ .Values.l2podAnnouncements.interface | quote }} +{{- end }} + +{{- if and .Values.bgp.enabled (and (not .Values.bgp.announce.loadbalancerIP) (not .Values.bgp.announce.podCIDR)) }} + {{ fail "BGP was enabled, but no announcements were enabled. Please enable one or more announcements." }} +{{- end }} + +{{- if and .Values.bgp.enabled .Values.bgp.announce.loadbalancerIP }} + bgp-announce-lb-ip: {{ .Values.bgp.announce.loadbalancerIP | quote }} +{{- end }} + +{{- if and .Values.bgp.enabled .Values.bgp.announce.podCIDR }} + bgp-announce-pod-cidr: {{ .Values.bgp.announce.podCIDR | quote }} +{{- end}} + +{{- if .Values.bgpControlPlane.enabled }} + enable-bgp-control-plane: "true" + bgp-secrets-namespace: {{ .Values.bgpControlPlane.secretsNamespace.name | quote }} +{{- else }} + enable-bgp-control-plane: "false" +{{- end }} + +{{- if .Values.pmtuDiscovery.enabled }} + enable-pmtu-discovery: "true" +{{- end }} + +{{- if not .Values.securityContext.privileged }} + procfs: "/host/proc" +{{- end }} + +{{- if hasKey .Values.bpf "root" }} + bpf-root: {{ .Values.bpf.root | quote }} +{{- end }} + +{{- if hasKey .Values.cgroup "hostRoot" }} + cgroup-root: {{ .Values.cgroup.hostRoot | quote }} +{{- end }} + +{{- if .Values.bpf.vlanBypass }} + # A space separated list of explicitly allowed vlan id's + vlan-bpf-bypass: {{ .Values.bpf.vlanBypass | join " " | quote }} +{{- end }} + +{{- if .Values.enableCiliumEndpointSlice }} + enable-cilium-endpoint-slice: "true" +{{- end }} + +{{- if hasKey .Values "enableK8sTerminatingEndpoint" }} + enable-k8s-terminating-endpoint: {{ .Values.enableK8sTerminatingEndpoint | quote }} +{{- end }} + +{{- if hasKey .Values.sctp "enabled" }} + enable-sctp: {{ .Values.sctp.enabled | quote }} +{{- end }} + +{{- if hasKey .Values "dnsPolicyUnloadOnShutdown" }} + # Unload DNS policy rules on graceful shutdown + dns-policy-unload-on-shutdown: {{.Values.dnsPolicyUnloadOnShutdown | quote }} +{{- end }} + +{{- if .Values.annotateK8sNode }} + annotate-k8s-node: "true" +{{- end }} + + k8s-client-qps: {{ .Values.k8sClientRateLimit.qps | default $defaultK8sClientQPS | quote}} + k8s-client-burst: {{ .Values.k8sClientRateLimit.burst | default $defaultK8sClientBurst | quote }} + +{{- if and .Values.operator.setNodeTaints (not .Values.operator.removeNodeTaints) -}} + {{ fail "Cannot have operator.setNodeTaintsMaxNodes and not operator.removeNodeTaints = false" }} +{{- end -}} +{{- if .Values.operator.removeNodeTaints }} + remove-cilium-node-taints: "true" +{{- end }} +{{- /* set node taints if setNodeTaints is explicitly enabled or removeNodeTaints is set */ -}} +{{- if or .Values.operator.setNodeTaints + ( and (kindIs "invalid" .Values.operator.setNodeTaints) + .Values.operator.removeNodeTaints ) }} + set-cilium-node-taints: "true" +{{- end }} +{{- if .Values.operator.setNodeNetworkStatus }} + set-cilium-is-up-condition: "true" +{{- end }} + +{{- if .Values.operator.unmanagedPodWatcher.restart }} + unmanaged-pod-watcher-interval: {{ .Values.operator.unmanagedPodWatcher.intervalSeconds | quote }} +{{- else }} + unmanaged-pod-watcher-interval: "0" +{{- end }} + +{{- if .Values.dnsProxy }} + {{- if hasKey .Values.dnsProxy "enableTransparentMode" }} + # explicit setting gets precedence + dnsproxy-enable-transparent-mode: {{ .Values.dnsProxy.enableTransparentMode | quote }} + {{- else if eq $cniChainingMode "none" }} + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: {{ $defaultDNSProxyEnableTransparentMode | quote }} + {{- end }} + {{- if .Values.dnsProxy.dnsRejectResponseCode }} + tofqdns-dns-reject-response-code: {{ .Values.dnsProxy.dnsRejectResponseCode | quote }} + {{- end }} + {{- if hasKey .Values.dnsProxy "enableDnsCompression" }} + tofqdns-enable-dns-compression: {{ .Values.dnsProxy.enableDnsCompression | quote }} + {{- end }} + {{- if .Values.dnsProxy.endpointMaxIpPerHostname }} + tofqdns-endpoint-max-ip-per-hostname: {{ .Values.dnsProxy.endpointMaxIpPerHostname | quote }} + {{- end }} + {{- if .Values.dnsProxy.idleConnectionGracePeriod }} + tofqdns-idle-connection-grace-period: {{ .Values.dnsProxy.idleConnectionGracePeriod | quote }} + {{- end }} + {{- if .Values.dnsProxy.maxDeferredConnectionDeletes }} + tofqdns-max-deferred-connection-deletes: {{ .Values.dnsProxy.maxDeferredConnectionDeletes | quote }} + {{- end }} + {{- if .Values.dnsProxy.minTtl }} + tofqdns-min-ttl: {{ .Values.dnsProxy.minTtl | quote }} + {{- end }} + {{- if .Values.dnsProxy.preCache }} + tofqdns-pre-cache: {{ .Values.dnsProxy.preCache | quote }} + {{- end }} + {{- if .Values.dnsProxy.proxyPort }} + tofqdns-proxy-port: {{ .Values.dnsProxy.proxyPort | quote }} + {{- end }} + {{- if .Values.dnsProxy.proxyResponseMaxDelay }} + tofqdns-proxy-response-max-delay: {{ .Values.dnsProxy.proxyResponseMaxDelay | quote }} + {{- end }} +{{- end }} + +{{- if hasKey .Values "agentNotReadyTaintKey" }} + agent-not-ready-taint-key: {{ .Values.agentNotReadyTaintKey | quote }} +{{- end }} + + mesh-auth-enabled: {{ .Values.authentication.enabled | quote }} + mesh-auth-queue-size: {{ .Values.authentication.queueSize | quote }} + mesh-auth-rotated-identities-queue-size: {{ .Values.authentication.rotatedIdentitiesQueueSize | quote }} + mesh-auth-gc-interval: {{ include "validateDuration" .Values.authentication.gcInterval | quote }} + +{{- if .Values.authentication.mutual.spire.enabled }} + mesh-auth-mutual-enabled: "true" + mesh-auth-mutual-listener-port: {{ .Values.authentication.mutual.port | quote }} + mesh-auth-spire-agent-socket: {{ .Values.authentication.mutual.spire.agentSocketPath | quote }} + mesh-auth-mutual-connect-timeout: {{ include "validateDuration" .Values.authentication.mutual.connectTimeout | quote }} + {{- if .Values.authentication.mutual.spire.serverAddress }} + mesh-auth-spire-server-address: {{ .Values.authentication.mutual.spire.serverAddress | quote }} + {{- else }} + mesh-auth-spire-server-address: "spire-server.{{ .Values.authentication.mutual.spire.install.namespace}}.svc:8081" + {{- end }} + mesh-auth-spire-server-connection-timeout: {{ .Values.authentication.mutual.spire.connectionTimeout }} + mesh-auth-spire-admin-socket: {{ .Values.authentication.mutual.spire.adminSocketPath | quote }} + mesh-auth-spiffe-trust-domain: {{ .Values.authentication.mutual.spire.trustDomain | quote }} +{{- end }} + + proxy-connect-timeout: {{ .Values.envoy.connectTimeoutSeconds | quote }} + proxy-max-requests-per-connection: {{ .Values.envoy.maxRequestsPerConnection | quote }} + proxy-max-connection-duration-seconds: {{ .Values.envoy.maxConnectionDurationSeconds | quote }} + + external-envoy-proxy: {{ .Values.envoy.enabled | quote }} + +{{- if .Values.envoy.log.path }} + envoy-log: {{ .Values.envoy.log.path | quote }} +{{- end }} + +{{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + max-connected-clusters: {{ .Values.clustermesh.maxConnectedClusters | quote }} +{{- end }} + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. +{{- if .Values.extraConfig }} + {{ toYaml .Values.extraConfig | nindent 2 }} +{{- end }} + +{{- end }} +--- +{{- if and .Values.ipMasqAgent.enabled .Values.ipMasqAgent.config }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: ip-masq-agent + namespace: {{ .Release.Namespace }} +data: + config: |- +{{ toJson .Values.ipMasqAgent.config | indent 4 }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-envoy/configmap.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/configmap.yaml new file mode 100644 index 00000000..990cf951 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/configmap.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: {{ .Release.Namespace }} + {{- with .Values.envoy.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- (tpl (.Files.Glob "files/cilium-envoy/configmap/bootstrap-config.json").AsConfig .) | nindent 2 }} + +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-envoy/daemonset.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/daemonset.yaml new file mode 100644 index 00000000..3a7f7b4b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/daemonset.yaml @@ -0,0 +1,266 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: {{ .Release.Namespace }} + {{- with .Values.envoy.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + {{- with .Values.envoy.updateStrategy }} + updateStrategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.prometheus.serviceMonitor.enabled) }} + prometheus.io/port: "{{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}" + prometheus.io/scrape: "true" + {{- end }} + {{- if .Values.envoy.rollOutPods }} + # ensure pods roll when configmap updates + cilium.io/cilium-envoy-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-envoy/configmap.yaml") . | sha256sum | quote }} + {{- end }} + {{- if not .Values.envoy.securityContext.privileged }} + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-envoy: "unconfined" + {{- end }} + {{- with .Values.envoy.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + {{- with .Values.envoy.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.envoy.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cilium-envoy + image: {{ include "cilium.image" .Values.envoy.image | quote }} + imagePullPolicy: {{ .Values.envoy.image.pullPolicy }} + command: + - /usr/bin/cilium-envoy-starter + args: + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + {{- if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "envoy" ( splitList " " .Values.debug.verbose )) }} + - '--log-level trace' + {{- else if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "flow" ( splitList " " .Values.debug.verbose )) }} + - '--log-level debug' + {{- else }} + - '--log-level info' + {{- end }} + - '--log-format {{ .Values.envoy.log.format }}' + {{- if .Values.envoy.log.path }} + - '--log-path {{ .Values.envoy.log.path }}' + {{- end }} + {{- with .Values.envoy.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} + startupProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: {{ .Values.envoy.healthPort }} + scheme: HTTP + failureThreshold: {{ .Values.envoy.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.envoy.startupProbe.periodSeconds }} + successThreshold: 1 + initialDelaySeconds: 5 + {{- end }} + livenessProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: {{ .Values.envoy.healthPort }} + scheme: HTTP + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 120 + {{- end }} + periodSeconds: {{ .Values.envoy.livenessProbe.periodSeconds }} + successThreshold: 1 + failureThreshold: {{ .Values.envoy.livenessProbe.failureThreshold }} + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: {{ .Values.envoy.healthPort }} + scheme: HTTP + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + initialDelaySeconds: 5 + {{- end }} + periodSeconds: {{ .Values.envoy.readinessProbe.periodSeconds }} + successThreshold: 1 + failureThreshold: {{ .Values.envoy.readinessProbe.failureThreshold }} + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- with .Values.envoy.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.envoy.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} + ports: + - name: envoy-metrics + containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + {{- end }} + securityContext: + {{- if .Values.envoy.securityContext.privileged }} + privileged: true + {{- else }} + seLinuxOptions: + {{- with .Values.envoy.securityContext.seLinuxOptions }} + {{- toYaml . | nindent 12 }} + {{- end }} + capabilities: + add: + {{- with .Values.envoy.securityContext.capabilities.envoy }} + {{- toYaml . | nindent 14 }} + {{- end }} + drop: + - ALL + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + {{- if .Values.bpf.autoMount.enabled }} + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + {{- end }} + {{- range .Values.envoy.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- if .mountPropagation }} + mountPropagation: {{ .mountPropagation }} + {{- end }} + {{- end }} + {{- with .Values.envoy.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.envoy.extraContainers }} + {{- toYaml .Values.envoy.extraContainers | nindent 6 }} + {{- end }} + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.envoy.priorityClassName "system-node-critical") }} + serviceAccount: {{ .Values.serviceAccounts.envoy.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.envoy.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.envoy.automount }} + terminationGracePeriodSeconds: {{ .Values.envoy.terminationGracePeriodSeconds }} + hostNetwork: true + {{- if .Values.envoy.dnsPolicy }} + dnsPolicy: {{ .Values.envoy.dnsPolicy }} + {{- end }} + {{- with .Values.envoy.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.envoy.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.envoy.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + - name: envoy-sockets + hostPath: + path: "{{ .Values.daemon.runPath }}/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "{{ .Values.daemon.runPath }}/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: cilium-envoy-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + {{- if and .Values.bpf.autoMount.enabled }} + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + {{- end }} + {{- range .Values.envoy.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- if .hostPathType }} + type: {{ .hostPathType }} + {{- end }} + {{- end }} + {{- with .Values.envoy.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-envoy/service.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/service.yaml new file mode 100644 index 00000000..d238c62e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/service.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: {{ .Release.Namespace }} + {{- if or (not .Values.envoy.prometheus.serviceMonitor.enabled) .Values.envoy.annotations }} + annotations: + {{- if not .Values.envoy.prometheus.serviceMonitor.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} + {{- end }} + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }} + protocol: TCP + targetPort: envoy-metrics +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-envoy/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/serviceaccount.yaml new file mode 100644 index 00000000..607dabca --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) .Values.serviceAccounts.envoy.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.envoy.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.envoy.annotations .Values.envoy.annotations }} + annotations: + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-envoy/servicemonitor.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/servicemonitor.yaml new file mode 100644 index 00000000..10f84d82 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-envoy/servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.envoy.enabled (not .Values.preflight.enabled) .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled .Values.envoy.prometheus.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cilium-envoy + namespace: {{ .Values.envoy.prometheus.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + {{- with .Values.envoy.prometheus.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.envoy.prometheus.serviceMonitor.annotations .Values.envoy.annotations }} + annotations: + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.envoy.prometheus.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: envoy-metrics + interval: {{ .Values.envoy.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.envoy.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.envoy.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + targetLabels: + - k8s-app +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-flowlog-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-flowlog-configmap.yaml new file mode 100644 index 00000000..8a1341e4 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-flowlog-configmap.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.hubble.export.dynamic.enabled .Values.hubble.export.dynamic.config.createConfigMap }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.hubble.export.dynamic.config.configMapName }} + namespace: {{ .Release.Namespace }} +data: + flowlogs.yaml: | + flowLogs: +{{ .Values.hubble.export.dynamic.config.content | toYaml | indent 4 }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-gateway-api-class.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-gateway-api-class.yaml new file mode 100644 index 00000000..74a78d8d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-gateway-api-class.yaml @@ -0,0 +1,10 @@ +{{- if .Values.gatewayAPI.enabled -}} +{{- if .Capabilities.APIVersions.Has "gateway.networking.k8s.io/v1/GatewayClass" }} +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: cilium +spec: + controllerName: io.cilium/gateway-controller +{{- end}} +{{- end}} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-ingress-class.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-ingress-class.yaml new file mode 100644 index 00000000..9243b377 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-ingress-class.yaml @@ -0,0 +1,12 @@ +{{- if .Values.ingressController.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: cilium + {{- if .Values.ingressController.default }} + annotations: + ingressclass.kubernetes.io/is-default-class: "true" + {{- end}} +spec: + controller: cilium.io/ingress-controller +{{- end}} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-ingress-service.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-ingress-service.yaml new file mode 100644 index 00000000..ff6269d2 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-ingress-service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.ingressController.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.ingressController.service.name }} + namespace: {{ .Release.Namespace }} + labels: + cilium.io/ingress: "true" + {{- if .Values.ingressController.service.labels }} + {{- toYaml .Values.ingressController.service.labels | nindent 4 }} + {{- end }} + {{- if .Values.ingressController.service.annotations }} + annotations: + {{- toYaml .Values.ingressController.service.annotations | nindent 4 }} + {{- end }} +spec: + ports: + - name: http + port: 80 + protocol: TCP + nodePort: {{ .Values.ingressController.service.insecureNodePort }} + - name: https + port: 443 + protocol: TCP + nodePort: {{ .Values.ingressController.service.secureNodePort }} + type: {{ .Values.ingressController.service.type }} + {{- if semverCompare ">=1.24-0" .Capabilities.KubeVersion.Version -}} + {{- if .Values.ingressController.service.loadBalancerClass }} + loadBalancerClass: {{ .Values.ingressController.service.loadBalancerClass }} + {{- end }} + {{- if (not (kindIs "invalid" .Values.ingressController.service.allocateLoadBalancerNodePorts)) }} + allocateLoadBalancerNodePorts: {{ .Values.ingressController.service.allocateLoadBalancerNodePorts }} + {{- end }} + {{- end -}} + {{- if .Values.ingressController.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.ingressController.service.loadBalancerIP }} + {{- end }} +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ .Values.ingressController.service.name }} + namespace: {{ .Release.Namespace }} + {{- if .Values.ingressController.service.labels }} + labels: + {{- toYaml .Values.ingressController.service.labels | nindent 4 }} + {{- end }} + {{- if .Values.ingressController.service.annotations }} + annotations: + {{- toYaml .Values.ingressController.service.annotations | nindent 4 }} + {{- end }} +subsets: +- addresses: + - ip: "192.192.192.192" + ports: + - port: 9999 +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/daemonset.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/daemonset.yaml new file mode 100644 index 00000000..76f1a20d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/daemonset.yaml @@ -0,0 +1,114 @@ +{{- if .Values.nodeinit.enabled }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: cilium-node-init + namespace: {{ .Release.Namespace }} + {{- with .Values.nodeinit.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app: cilium-node-init + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-node-init +spec: + selector: + matchLabels: + app: cilium-node-init + {{- with .Values.nodeinit.updateStrategy }} + updateStrategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- with .Values.nodeinit.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not .Values.securityContext.privileged }} + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/node-init: "unconfined" + {{- end }} + labels: + app: cilium-node-init + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-node-init + {{- with .Values.nodeinit.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: node-init + image: {{ include "cilium.image" .Values.nodeinit.image | quote }} + imagePullPolicy: {{ .Values.nodeinit.image.pullPolicy }} + lifecycle: + {{- if .Values.nodeinit.revertReconfigureKubelet }} + preStop: + exec: + command: + - nsenter + - --target=1 + - --mount + - -- + - /bin/bash + - -c + - | + {{- tpl (.Files.Get "files/nodeinit/prestop.bash") . | nindent 20 }} + {{- end }} + env: + {{- with .Values.nodeinit.extraEnv }} + {{- toYaml . | trim | nindent 10 }} + {{- end }} + # STARTUP_SCRIPT is the script run on node bootstrap. Node + # bootstrapping can be customized in this script. This script is invoked + # using nsenter, so it runs in the host's network and mount namespace using + # the host's userland tools! + - name: STARTUP_SCRIPT + value: | + {{- tpl (.Files.Get "files/nodeinit/startup.bash") . | nindent 14 }} + {{- with .Values.nodeinit.resources }} + resources: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + {{- with .Values.nodeinit.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.nodeinit.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.nodeinit.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeinit.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.nodeinit.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + hostPID: true + hostNetwork: true + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.nodeinit.priorityClassName "system-node-critical") }} + {{- if .Values.serviceAccounts.nodeinit.enabled }} + serviceAccount: {{ .Values.serviceAccounts.nodeinit.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.nodeinit.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.nodeinit.automount }} + {{- end }} + {{- with .Values.nodeinit.extraVolumes }} + volumes: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml new file mode 100644 index 00000000..eb503ee9 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.nodeinit.enabled .Values.serviceAccounts.nodeinit.enabled .Values.serviceAccounts.nodeinit.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.nodeinit.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.nodeinit.annotations .Values.nodeinit.annotations }} + annotations: + {{- with .Values.nodeinit.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.nodeinit.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/_helpers.tpl b/magnum_cluster_api/charts/cilium/templates/cilium-operator/_helpers.tpl new file mode 100644 index 00000000..0910de63 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/_helpers.tpl @@ -0,0 +1,36 @@ +{{- define "cilium.operator.cloud" -}} +{{- $cloud := "generic" -}} +{{- if .Values.eni.enabled -}} + {{- $cloud = "aws" -}} +{{- else if .Values.azure.enabled -}} + {{- $cloud = "azure" -}} +{{- else if .Values.alibabacloud.enabled -}} + {{- $cloud = "alibabacloud" -}} +{{- end -}} +{{- $cloud -}} +{{- end -}} + +{{- define "cilium.operator.imageDigestName" -}} +{{- $imageDigest := (.Values.operator.image.useDigest | default false) | ternary (printf "@%s" .Values.operator.image.genericDigest) "" -}} +{{- if .Values.eni.enabled -}} + {{- $imageDigest = (.Values.operator.image.useDigest | default false) | ternary (printf "@%s" .Values.operator.image.awsDigest) "" -}} +{{- else if .Values.azure.enabled -}} + {{- $imageDigest = (.Values.operator.image.useDigest | default false) | ternary (printf "@%s" .Values.operator.image.azureDigest) "" -}} +{{- else if .Values.alibabacloud.enabled -}} + {{- $imageDigest = (.Values.operator.image.useDigest | default false) | ternary (printf "@%s" .Values.operator.image.alibabacloudDigest) "" -}} +{{- end -}} +{{- $imageDigest -}} +{{- end -}} + +{{/* +Return cilium operator image +*/}} +{{- define "cilium.operator.image" -}} +{{- if .Values.operator.image.override -}} +{{- printf "%s" .Values.operator.image.override -}} +{{- else -}} +{{- $cloud := include "cilium.operator.cloud" . }} +{{- $imageDigest := include "cilium.operator.imageDigestName" . }} +{{- printf "%s-%s%s:%s%s" .Values.operator.image.repository $cloud .Values.operator.image.suffix .Values.operator.image.tag $imageDigest -}} +{{- end -}} +{{- end -}} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrole.yaml new file mode 100644 index 00000000..7e9080b5 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrole.yaml @@ -0,0 +1,294 @@ +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +{{- if hasKey .Values "disableEndpointCRD" }} +{{- if not .Values.disableEndpointCRD }} +{{- if (and .Values.operator.unmanagedPodWatcher.restart (ne (.Values.operator.unmanagedPodWatcher.intervalSeconds | int64) 0 ) ) }} + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +{{- end }} +{{- end }} +{{- end }} +{{- if or .Values.operator.removeNodeTaints .Values.operator.setNodeNetworkStatus (include "hasDuration" .Values.operator.endpointGCInterval) }} +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +{{- end }} +{{- if or .Values.operator.removeNodeTaints .Values.operator.setNodeNetworkStatus }} +- apiGroups: + - "" + resources: +{{- if .Values.operator.removeNodeTaints }} + # To remove node taints + - nodes +{{- end }} +{{- if .Values.operator.setNodeNetworkStatus }} + # To set NetworkUnavailable false on startup + - nodes/status +{{- end }} + verbs: + - patch +{{- end }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces +{{- if or .Values.ingressController.enabled .Values.gatewayAPI.enabled }} + - secrets +{{- end }} + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +{{- if or .Values.ingressController.enabled .Values.gatewayAPI.enabled }} + - create + - update + - delete + - patch +{{- end }} +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch +{{- if include "hasDuration" .Values.operator.nodeGCInterval }} + # To perform CiliumNode garbage collector + - delete +{{- end }} +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +{{- if .Values.ingressController.enabled }} +- apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status # To update ingress status with load balancer IP. + verbs: + - update +{{- end }} +{{- if .Values.gatewayAPI.enabled }} +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - tlsroutes + - httproutes + - grpcroutes + - referencegrants + - referencepolicies + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + - gateways/status + - httproutes/status + - grpcroutes/status + - tlsroutes/status + verbs: + - update + - patch +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrolebinding.yaml new file mode 100644 index 00000000..f0d00e2e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.operator.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml new file mode 100644 index 00000000..c4b90a27 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml @@ -0,0 +1,30 @@ +{{- if .Values.operator.dashboards.enabled }} +{{- $files := .Files.Glob "files/cilium-operator/dashboards/*.json" }} +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Values.operator.dashboards.namespace | default $.Release.Namespace }} + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-operator + app.kubernetes.io/part-of: cilium + {{- if $.Values.operator.dashboards.label }} + {{ $.Values.operator.dashboards.label }}: {{ ternary $.Values.operator.dashboards.labelValue "1" (not (empty $.Values.operator.dashboards.labelValue)) | quote }} + {{- end }} + {{- if or $.Values.operator.dashboards.annotations $.Values.operator.annotations }} + annotations: + {{- with $.Values.operator.dashboards.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/deployment.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/deployment.yaml new file mode 100644 index 00000000..4f4450e5 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/deployment.yaml @@ -0,0 +1,337 @@ +{{- if .Values.operator.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: {{ .Values.operator.replicas }} + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + {{- if and (eq (.Values.operator.replicas | toString) "1") (eq .Values.operator.updateStrategy.type "RollingUpdate") }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.operator.updateStrategy.rollingUpdate.maxSurge }} + maxUnavailable: 100% + type: RollingUpdate + {{- else }} + {{- with .Values.operator.updateStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- end }} + template: + metadata: + annotations: + {{- if .Values.operator.rollOutPods }} + # ensure pods roll when configmap updates + cilium.io/cilium-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-configmap.yaml") . | sha256sum | quote }} + {{- end }} + {{- if and .Values.operator.prometheus.enabled (not .Values.operator.prometheus.serviceMonitor.enabled) }} + prometheus.io/port: {{ .Values.operator.prometheus.port | quote }} + prometheus.io/scrape: "true" + {{- end }} + {{- with .Values.operator.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + {{- with .Values.operator.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cilium-operator + image: {{ include "cilium.operator.image" . | quote }} + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + command: + - cilium-operator-{{ include "cilium.operator.cloud" . }} + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + {{- with .Values.operator.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + {{- if and .Values.eni.enabled (not .Values.eni.iamRole ) }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cilium-aws + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: cilium-aws + key: AWS_SECRET_ACCESS_KEY + optional: true + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + name: cilium-aws + key: AWS_DEFAULT_REGION + optional: true + {{- end }} + {{- if .Values.alibabacloud.enabled }} + - name: ALIBABA_CLOUD_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cilium-alibabacloud + key: ALIBABA_CLOUD_ACCESS_KEY_ID + optional: true + - name: ALIBABA_CLOUD_ACCESS_KEY_SECRET + valueFrom: + secretKeyRef: + name: cilium-alibabacloud + key: ALIBABA_CLOUD_ACCESS_KEY_SECRET + optional: true + {{- end }} + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- if .Values.azure.enabled }} + {{- if .Values.azure.subscriptionID }} + - name: AZURE_SUBSCRIPTION_ID + value: {{ .Values.azure.subscriptionID }} + {{- end }} + {{- if .Values.azure.tenantID }} + - name: AZURE_TENANT_ID + value: {{ .Values.azure.tenantID }} + {{- end }} + {{- if .Values.azure.resourceGroup }} + - name: AZURE_RESOURCE_GROUP + value: {{ .Values.azure.resourceGroup }} + {{- end }} + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: cilium-azure + key: AZURE_CLIENT_ID + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: cilium-azure + key: AZURE_CLIENT_SECRET + {{- end }} + {{- with .Values.operator.extraEnv }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.operator.prometheus.enabled }} + ports: + - name: prometheus + containerPort: {{ .Values.operator.prometheus.port }} + hostPort: {{ .Values.operator.prometheus.port }} + protocol: TCP + {{- end }} + livenessProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + {{- if .Values.etcd.enabled }} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + - name: etcd-secrets + mountPath: /var/lib/etcd-secrets + readOnly: true + {{- end }} + {{- end }} + {{- if .Values.kubeConfigPath }} + - name: kube-config + mountPath: {{ .Values.kubeConfigPath }} + readOnly: true + {{- end }} + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + mountPath: {{ dir .Values.authentication.mutual.spire.agentSocketPath }} + readOnly: true + {{- end }} + {{- range .Values.operator.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- if .mountPropagation }} + mountPropagation: {{ .mountPropagation }} + {{- end }} + {{- end }} + {{- if .Values.bgp.enabled }} + - name: bgp-config-path + mountPath: /var/lib/cilium/bgp + readOnly: true + {{- end }} + {{- with .Values.operator.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- with .Values.operator.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + {{- if and .Values.etcd.managed (not .Values.etcd.k8sService) }} + # In managed etcd mode, Cilium must be able to resolve the DNS name of + # the etcd service + dnsPolicy: ClusterFirstWithHostNet + {{- else if .Values.operator.dnsPolicy }} + dnsPolicy: {{ .Values.operator.dnsPolicy }} + {{- end }} + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.operator.priorityClassName "system-cluster-critical") }} + serviceAccount: {{ .Values.serviceAccounts.operator.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.operator.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.operator.automount }} + {{- with .Values.operator.affinity }} + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.operator.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range $constraint := . }} + - {{ toYaml $constraint | nindent 8 | trim }} + {{- if not $constraint.labelSelector }} + labelSelector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config + {{- if .Values.etcd.enabled }} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: etcd-config + path: etcd.config + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + # To read the k8s etcd secrets in case the user might want to use TLS + - name: etcd-secrets + secret: + secretName: cilium-etcd-secrets + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + {{- end }} + {{- end }} + {{- if .Values.kubeConfigPath }} + - name: kube-config + hostPath: + path: {{ .Values.kubeConfigPath }} + type: FileOrCreate + {{- end }} + {{- range .Values.operator.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- if .hostPathType }} + type: {{ .hostPathType }} + {{- end }} + {{- end }} + {{- if .Values.bgp.enabled }} + - name: bgp-config-path + configMap: + name: bgp-config + {{- end }} + {{- if .Values.authentication.mutual.spire.enabled }} + - name: spire-agent-socket + hostPath: + path: {{ dir .Values.authentication.mutual.spire.agentSocketPath }} + type: DirectoryOrCreate + {{- end }} + {{- with .Values.operator.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml new file mode 100644 index 00000000..a224b9e6 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.operator.enabled .Values.operator.podDisruptionBudget.enabled }} +{{- $component := .Values.operator.podDisruptionBudget }} +apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: cilium-operator + namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/name: cilium-operator + app.kubernetes.io/part-of: cilium +spec: + {{- with $component.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + {{- with $component.minAvailable }} + minAvailable: {{ . }} + {{- end }} + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/role.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/role.yaml new file mode 100644 index 00000000..20101b02 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/role.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create .Values.ingressController.enabled .Values.ingressController.secretsNamespace.sync .Values.ingressController.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-ingress-secrets + namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch +{{- end }} + +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.sync .Values.gatewayAPI.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-gateway-secrets + namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/rolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/rolebinding.yaml new file mode 100644 index 00000000..8d8f73d8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/rolebinding.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create .Values.ingressController.enabled .Values.ingressController.secretsNamespace.sync .Values.ingressController.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-ingress-secrets + namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-ingress-secrets +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccounts.operator.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} + +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.sync .Values.gatewayAPI.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-gateway-secrets + namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-gateway-secrets +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.operator.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/secret.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/secret.yaml new file mode 100644 index 00000000..31e917ec --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.operator.enabled }} +{{- if .Values.azure.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: cilium-azure + namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + AZURE_CLIENT_ID: {{ default "" .Values.azure.clientID | b64enc | quote }} + AZURE_CLIENT_SECRET: {{ default "" .Values.azure.clientSecret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/service.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/service.yaml new file mode 100644 index 00000000..8c201099 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.operator.enabled .Values.operator.prometheus.enabled .Values.operator.prometheus.serviceMonitor.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: cilium-operator + namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + clusterIP: None + type: ClusterIP + ports: + - name: metrics + port: 9963 + protocol: TCP + targetPort: prometheus + selector: + io.cilium/app: operator + name: cilium-operator +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/serviceaccount.yaml new file mode 100644 index 00000000..b083907e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.operator.enabled .Values.serviceAccounts.operator.create }} +{{- if and .Values.eni.enabled .Values.eni.iamRole }} + {{ $_ := set .Values.serviceAccounts.operator.annotations "eks.amazonaws.com/role-arn" .Values.eni.iamRole }} +{{- end}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.operator.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.operator.annotations .Values.operator.annotations }} + annotations: + {{- with .Values.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-operator/servicemonitor.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-operator/servicemonitor.yaml new file mode 100644 index 00000000..742993b8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-operator/servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.operator.enabled .Values.operator.prometheus.enabled .Values.operator.prometheus.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cilium-operator + namespace: {{ .Values.operator.prometheus.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + {{- with .Values.operator.prometheus.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.operator.prometheus.serviceMonitor.annotations .Values.operator.annotations }} + annotations: + {{- with .Values.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.operator.prometheus.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: metrics + interval: {{ .Values.operator.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.operator.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.operator.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + targetLabels: + - io.cilium/app +{{- if .Values.operator.prometheus.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.operator.prometheus.serviceMonitor.jobLabel | quote }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrole.yaml new file mode 100644 index 00000000..d2a8f4ca --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrole.yaml @@ -0,0 +1,150 @@ +{{- if .Values.preflight.enabled }} +{{- /* +Keep file in sync with cilium-agent/clusterrole.yaml +*/ -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-pre-flight + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +{{- if .Values.annotateK8sNode }} +- apiGroups: + - "" + resources: + - nodes/status + verbs: + # To annotate the k8s node with Cilium's metadata + - patch +{{- end }} +{{- if .Values.l2announcements.enabled }} +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + - list + - delete +{{- end }} +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +{{- if eq "k8s" .Values.tls.secretsBackend }} +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +{{- end }} +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrolebinding.yaml new file mode 100644 index 00000000..1237d6ac --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.preflight.enabled .Values.serviceAccounts.preflight.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-pre-flight + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-pre-flight +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.preflight.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-preflight/daemonset.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/daemonset.yaml new file mode 100644 index 00000000..bafd2700 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/daemonset.yaml @@ -0,0 +1,220 @@ +{{- if .Values.preflight.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-pre-flight-check + namespace: {{ .Release.Namespace }} + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: cilium-pre-flight-check + kubernetes.io/cluster-service: "true" + template: + metadata: + {{- with .Values.preflight.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium + k8s-app: cilium-pre-flight-check + app.kubernetes.io/name: cilium-pre-flight-check + kubernetes.io/cluster-service: "true" + {{- with .Values.preflight.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.preflight.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.preflight.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: clean-cilium-state + image: {{ include "cilium.image" .Values.preflight.image | quote }} + imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} + command: ["/bin/echo"] + args: + - "hello" + terminationMessagePolicy: FallbackToLogsOnError + containers: + - name: cilium-pre-flight-check + image: {{ include "cilium.image" .Values.preflight.image | quote }} + imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} + command: ["/bin/sh"] + args: + - -c + - "touch /tmp/ready; sleep 1h" + livenessProbe: + exec: + command: + - cat + - /tmp/ready + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + exec: + command: + - cat + - /tmp/ready + initialDelaySeconds: 5 + periodSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + {{- with .Values.preflight.extraEnv }} + {{- toYaml . | trim | nindent 12 }} + {{- end }} + volumeMounts: + - name: cilium-run + mountPath: /var/run/cilium + {{- if .Values.etcd.enabled }} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + - name: etcd-secrets + mountPath: /var/lib/etcd-secrets + readOnly: true + {{- end }} + {{- end }} + {{- with .Values.preflight.extraVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.preflight.resources }} + resources: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + {{- with .Values.preflight.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 14 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- if ne .Values.preflight.tofqdnsPreCache "" }} + - name: cilium-pre-flight-fqdn-precache + image: {{ include "cilium.image" .Values.preflight.image | quote }} + imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} + name: cilium-pre-flight-fqdn-precache + command: ["/bin/sh"] + args: + - -ec + - | + cilium-dbg preflight fqdn-poller --tofqdns-pre-cache {{ .Values.preflight.tofqdnsPreCache }}; + touch /tmp/ready-tofqdns-precache; + livenessProbe: + exec: + command: + - cat + - /tmp/read-tofqdns-precachey + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + exec: + command: + - cat + - /tmp/read-tofqdns-precachey + initialDelaySeconds: 5 + periodSeconds: 5 + env: + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + volumeMounts: + - name: cilium-run + mountPath: /var/run/cilium + {{- if .Values.etcd.enabled }} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + - name: etcd-secrets + mountPath: /var/lib/etcd-secrets + readOnly: true + {{- end }} + {{- end }} + {{- with .Values.preflight.extraVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.preflight.extraEnv }} + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- with .Values.preflight.resources }} + resources: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + {{- with .Values.preflight.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 14 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- end }} + hostNetwork: true + # This is here to seamlessly allow migrate-identity to work with + # etcd-operator setups. The assumption is that other cases would also + # work since the cluster DNS would forward the request on. + # This differs from the cilium-agent daemonset, where this is only + # enabled when etcd.managed=true + dnsPolicy: ClusterFirstWithHostNet + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.preflight.priorityClassName "system-node-critical") }} + serviceAccount: {{ .Values.serviceAccounts.preflight.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.preflight.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.preflight.automount }} + terminationGracePeriodSeconds: {{ .Values.preflight.terminationGracePeriodSeconds }} + {{- with .Values.preflight.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + {{- if .Values.etcd.enabled }} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: etcd-config + path: etcd.config + # To read the k8s etcd secrets in case the user might want to use TLS + {{- if or .Values.etcd.ssl .Values.etcd.managed }} + - name: etcd-secrets + secret: + secretName: cilium-etcd-secrets + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + {{- end }} + {{- with .Values.preflight.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-preflight/deployment.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/deployment.yaml new file mode 100644 index 00000000..af0a31ba --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/deployment.yaml @@ -0,0 +1,107 @@ +{{- if and .Values.preflight.enabled .Values.preflight.validateCNPs }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-pre-flight-check + namespace: {{ .Release.Namespace }} + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-pre-flight-check +spec: + selector: + matchLabels: + k8s-app: cilium-pre-flight-check-deployment + kubernetes.io/cluster-service: "true" + template: + metadata: + {{- with .Values.preflight.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium + k8s-app: cilium-pre-flight-check-deployment + kubernetes.io/cluster-service: "true" + app.kubernetes.io/name: cilium-pre-flight-check + {{- with .Values.preflight.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cnp-validator + image: {{ include "cilium.image" .Values.preflight.image | quote }} + imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} + command: ["/bin/sh"] + args: + - -ec + - | + cilium-dbg preflight validate-cnp; + touch /tmp/ready-validate-cnp; + sleep 1h; + livenessProbe: + exec: + command: + - cat + - /tmp/ready-validate-cnp + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + exec: + command: + - cat + - /tmp/ready-validate-cnp + initialDelaySeconds: 5 + periodSeconds: 5 + {{- with .Values.preflight.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + env: + {{- if .Values.k8sServiceHost }} + - name: KUBERNETES_SERVICE_HOST + value: {{ .Values.k8sServiceHost | quote }} + {{- end }} + {{- if .Values.k8sServicePort }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.k8sServicePort | quote }} + {{- end }} + {{- with .Values.preflight.extraEnv }} + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- with .Values.preflight.resources }} + resources: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.preflight.extraVolumes }} + volumes: + {{- toYaml . | trim | nindent 6 }} + {{- end }} + hostNetwork: true + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.preflight.priorityClassName "system-cluster-critical") }} + serviceAccount: {{ .Values.serviceAccounts.preflight.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.preflight.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.preflight.automount }} + terminationGracePeriodSeconds: {{ .Values.preflight.terminationGracePeriodSeconds }} + {{- with .Values.preflight.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.preflight.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.preflight.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml new file mode 100644 index 00000000..4b3c7cb0 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.preflight.enabled .Values.preflight.validateCNPs .Values.preflight.podDisruptionBudget.enabled }} +{{- $component := .Values.preflight.podDisruptionBudget }} +apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: cilium-pre-flight-check + namespace: {{ .Release.Namespace }} + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: cilium-pre-flight-check-deployment + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-pre-flight-check + kubernetes.io/cluster-service: "true" +spec: + {{- with $component.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + {{- with $component.minAvailable }} + minAvailable: {{ . }} + {{- end }} + selector: + matchLabels: + k8s-app: cilium-pre-flight-check-deployment + kubernetes.io/cluster-service: "true" +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-preflight/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/serviceaccount.yaml new file mode 100644 index 00000000..b64a3296 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-preflight/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.preflight.enabled .Values.serviceAccounts.preflight.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.preflight.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.preflight.annotations .Values.preflight.annotations }} + annotations: + {{- with .Values.preflight.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.preflight.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-resource-quota.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-resource-quota.yaml new file mode 100644 index 00000000..a9856967 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-resource-quota.yaml @@ -0,0 +1,35 @@ +{{- if or .Values.resourceQuotas.enabled (and (ne .Release.Namespace "kube-system") .Values.gke.enabled) }} +{{- if .Values.agent }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: cilium-resource-quota + namespace: {{ .Release.Namespace }} +spec: + hard: + pods: {{ .Values.resourceQuotas.cilium.hard.pods | quote }} + scopeSelector: + matchExpressions: + - operator: In + scopeName: PriorityClass + values: + - system-node-critical +{{- end }} +{{- if .Values.operator.enabled }} +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: cilium-operator-resource-quota + namespace: {{ .Release.Namespace }} +spec: + hard: + pods: {{ .Values.resourceQuotas.operator.hard.pods | quote }} + scopeSelector: + matchExpressions: + - operator: In + scopeName: PriorityClass + values: + - system-cluster-critical +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/cilium-secrets-namespace.yaml b/magnum_cluster_api/charts/cilium/templates/cilium-secrets-namespace.yaml new file mode 100644 index 00000000..d99967fa --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/cilium-secrets-namespace.yaml @@ -0,0 +1,14 @@ +{{- $secretNamespaces := dict -}} +{{- range $cfg := tuple .Values.ingressController .Values.gatewayAPI .Values.envoyConfig .Values.bgpControlPlane -}} +{{- if and $cfg.enabled $cfg.secretsNamespace.create $cfg.secretsNamespace.name -}} +{{- $_ := set $secretNamespaces $cfg.secretsNamespace.name 1 -}} +{{- end -}} +{{- end -}} + +{{- range $name, $_ := $secretNamespaces }} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $name | quote }} +{{- end}} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl new file mode 100644 index 00000000..19cce1f7 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/_helpers.tpl @@ -0,0 +1,11 @@ +{{- define "clustermesh-apiserver-generate-certs.admin-common-name" -}} +admin-{{ .Values.cluster.name }} +{{- end -}} + +{{- define "clustermesh-apiserver-generate-certs.remote-common-name" -}} +{{- if eq .Values.clustermesh.apiserver.tls.authMode "cluster" -}} +remote-{{ .Values.cluster.name }} +{{- else -}} +remote +{{- end -}} +{{- end -}} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrole.yaml new file mode 100644 index 00000000..c5df0163 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrole.yaml @@ -0,0 +1,72 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.serviceAccounts.clustermeshApiserver.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clustermesh-apiserver + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumendpoints + - ciliumidentities + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumexternalworkloads/status + - ciliumnodes + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumendpoints/status + verbs: + - patch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumexternalworkloads + - ciliumendpoints + - ciliumnodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - endpoints + - namespaces + - services + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml new file mode 100644 index 00000000..edc3cf9c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.serviceAccounts.clustermeshApiserver.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clustermesh-apiserver + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clustermesh-apiserver +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/deployment.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/deployment.yaml new file mode 100644 index 00000000..8668888d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/deployment.yaml @@ -0,0 +1,431 @@ +{{- if (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) }} +{{- if not (list "legacy" "migration" "cluster" | has .Values.clustermesh.apiserver.tls.authMode) -}} +{{- fail ".Values.clustermesh.apiserver.tls.authMode must be one of legacy, migration, cluster" -}} +{{- end -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clustermesh-apiserver + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: clustermesh-apiserver + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: clustermesh-apiserver +spec: + replicas: {{ .Values.clustermesh.apiserver.replicas }} + selector: + matchLabels: + k8s-app: clustermesh-apiserver + {{- with .Values.clustermesh.apiserver.updateStrategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- with .Values.clustermesh.apiserver.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: clustermesh-apiserver + k8s-app: clustermesh-apiserver + {{- with .Values.clustermesh.apiserver.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: etcd-init + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} + command: + - /usr/bin/clustermesh-apiserver + args: + - etcdinit + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + # These need to match the equivalent arguments to etcd in the main container. + - --etcd-cluster-name=clustermesh-apiserver + - --etcd-initial-cluster-token=clustermesh-apiserver + - --etcd-data-dir=/var/run/etcd + {{- with .Values.clustermesh.apiserver.etcd.init.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + env: + # The Cilium cluster name (specified via the `CILIUM_CLUSTER_NAME` environment variable) and the etcd cluster + # name (specified via the `--etcd-cluster-name` argument) are very different concepts. The Cilium cluster name + # is the name of the overall Cilium cluster, and is used to set the admin account username. The etcd cluster + # name is a concept that's only relevant for etcd itself. The etcd cluster name must be the same for both this + # command and the actual invocation of etcd in the main containers of this Pod, but it's otherwise not + # relevant to Cilium. + - name: CILIUM_CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-name + {{- with .Values.clustermesh.apiserver.etcd.init.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumeMounts: + - name: etcd-data-dir + mountPath: /var/run/etcd + {{- with .Values.clustermesh.apiserver.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.clustermesh.apiserver.etcd.init.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + containers: + - name: etcd + # The clustermesh-apiserver container image includes an etcd binary. + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} + command: + - /usr/bin/etcd + args: + - --data-dir=/var/run/etcd + - --name=clustermesh-apiserver + - --client-cert-auth + - --trusted-ca-file=/var/lib/etcd-secrets/ca.crt + - --cert-file=/var/lib/etcd-secrets/tls.crt + - --key-file=/var/lib/etcd-secrets/tls.key + # Surrounding the IPv4 address with brackets works in this case, since etcd + # uses net.SplitHostPort() internally and it accepts the that format. + - --listen-client-urls=https://127.0.0.1:2379,https://[$(HOSTNAME_IP)]:2379 + - --advertise-client-urls=https://[$(HOSTNAME_IP)]:2379 + - --initial-cluster-token=clustermesh-apiserver + - --auto-compaction-retention=1 + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - --listen-metrics-urls=http://[$(HOSTNAME_IP)]:{{ .Values.clustermesh.apiserver.metrics.etcd.port }} + - --metrics={{ .Values.clustermesh.apiserver.metrics.etcd.mode }} + {{- end }} + env: + - name: ETCDCTL_API + value: "3" + - name: HOSTNAME_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - name: etcd + containerPort: 2379 + protocol: TCP + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - name: etcd-metrics + containerPort: {{ .Values.clustermesh.apiserver.metrics.etcd.port }} + protocol: TCP + {{- end }} + volumeMounts: + - name: etcd-server-secrets + mountPath: /var/lib/etcd-secrets + readOnly: true + - name: etcd-data-dir + mountPath: /var/run/etcd + {{- with .Values.clustermesh.apiserver.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.clustermesh.apiserver.etcd.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.etcd.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.etcd.lifecycle }} + lifecycle: + {{- toYaml . | nindent 10 }} + {{- end }} + - name: apiserver + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} + command: + - /usr/bin/clustermesh-apiserver + args: + - clustermesh + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + - --cluster-name=$(CLUSTER_NAME) + - --cluster-id=$(CLUSTER_ID) + - --kvstore-opt + - etcd.config=/var/lib/cilium/etcd-config.yaml + {{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + - --max-connected-clusters={{ .Values.clustermesh.maxConnectedClusters }} + {{- end }} + {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} + - --cluster-users-enabled + - --cluster-users-config-path=/var/lib/cilium/etcd-config/users.yaml + {{- end }} + - --enable-external-workloads={{ .Values.externalWorkloads.enabled }} + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.port }} + - --controller-group-metrics=all + {{- end }} + {{- with .Values.clustermesh.apiserver.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + env: + - name: CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-name + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-id + optional: true + - name: IDENTITY_ALLOCATION_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: identity-allocation-mode + - name: ENABLE_K8S_ENDPOINT_SLICE + valueFrom: + configMapKeyRef: + name: cilium-config + key: enable-k8s-endpoint-slice + optional: true + {{- with .Values.clustermesh.apiserver.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + ports: + - name: apiserv-metrics + containerPort: {{ .Values.clustermesh.apiserver.metrics.port }} + protocol: TCP + {{- end }} + {{- with .Values.clustermesh.apiserver.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: etcd-admin-client + mountPath: /var/lib/cilium/etcd-secrets + readOnly: true + {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} + - name: etcd-users-config + mountPath: /var/lib/cilium/etcd-config + readOnly: true + {{- end }} + {{- with .Values.clustermesh.apiserver.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.clustermesh.apiserver.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.lifecycle }} + lifecycle: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} + - name: kvstoremesh + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} + command: + - /usr/bin/clustermesh-apiserver + args: + - kvstoremesh + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + - --cluster-name=$(CLUSTER_NAME) + - --cluster-id=$(CLUSTER_ID) + - --kvstore-opt=etcd.config=/var/lib/cilium/etcd-config.yaml + - --kvstore-opt=etcd.qps=100 + - --kvstore-opt=etcd.maxInflight=10 + - --clustermesh-config=/var/lib/cilium/clustermesh + {{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + - --max-connected-clusters={{ .Values.clustermesh.maxConnectedClusters }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} + - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + - --controller-group-metrics=all + {{- end }} + {{- with .Values.clustermesh.apiserver.kvstoremesh.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + env: + - name: CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-name + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-id + {{- with .Values.clustermesh.apiserver.kvstoremesh.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} + ports: + - name: kvmesh-metrics + containerPort: {{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + protocol: TCP + {{- end }} + {{- with .Values.clustermesh.apiserver.kvstoremesh.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: etcd-admin-client + mountPath: /var/lib/cilium/etcd-secrets + readOnly: true + - name: kvstoremesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + {{- with .Values.clustermesh.apiserver.kvstoremesh.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.clustermesh.apiserver.kvstoremesh.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.kvstoremesh.lifecycle }} + lifecycle: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + volumes: + - name: etcd-server-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: clustermesh-apiserver-server-cert + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: ca.crt + {{- end }} + - name: etcd-admin-client + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: clustermesh-apiserver-admin-cert + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: ca.crt + {{- end }} + {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} + - name: etcd-users-config + configMap: + name: clustermesh-remote-users + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + {{- end }} + - name: etcd-data-dir + emptyDir: {} + {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} + - name: kvstoremesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-kvstoremesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of this pod + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + {{- end }} + {{- with .Values.clustermesh.apiserver.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} + restartPolicy: Always + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.clustermesh.apiserver.priorityClassName "system-cluster-critical") }} + serviceAccount: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} + terminationGracePeriodSeconds: {{ .Values.clustermesh.apiserver.terminationGracePeriodSeconds }} + automountServiceAccountToken: {{ .Values.serviceAccounts.clustermeshApiserver.automount }} + {{- with .Values.clustermesh.apiserver.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range $constraint := . }} + - {{ toYaml $constraint | nindent 8 | trim }} + {{- if not $constraint.labelSelector }} + labelSelector: + matchLabels: + k8s-app: clustermesh-apiserver + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.clustermesh.apiserver.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and .Values.clustermesh.config.enabled .Values.clustermesh.apiserver.kvstoremesh.enabled }} + hostAliases: + {{- range $cluster := .Values.clustermesh.config.clusters }} + {{- range $ip := $cluster.ips }} + - ip: {{ $ip }} + hostnames: [ "{{ $cluster.name }}.{{ $.Values.clustermesh.config.domain }}" ] + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml new file mode 100644 index 00000000..0781af1e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml @@ -0,0 +1,43 @@ +{{- $kvstoreMetricsEnabled := and .Values.clustermesh.apiserver.kvstoremesh.enabled .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled -}} +{{- if and + (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) + (or .Values.clustermesh.apiserver.metrics.enabled $kvstoreMetricsEnabled .Values.clustermesh.apiserver.metrics.etcd.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: clustermesh-apiserver-metrics + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: clustermesh-apiserver + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: clustermesh-apiserver + app.kubernetes.io/component: metrics +spec: + clusterIP: None + type: ClusterIP + ports: + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + - name: apiserv-metrics + port: {{ .Values.clustermesh.apiserver.metrics.port }} + protocol: TCP + targetPort: apiserv-metrics + {{- end }} + {{- if $kvstoreMetricsEnabled }} + - name: kvmesh-metrics + port: {{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + protocol: TCP + targetPort: kvmesh-metrics + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - name: etcd-metrics + port: {{ .Values.clustermesh.apiserver.metrics.etcd.port }} + protocol: TCP + targetPort: etcd-metrics + {{- end }} + selector: + k8s-app: clustermesh-apiserver +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml new file mode 100644 index 00000000..4a1bbf7e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml @@ -0,0 +1,26 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.podDisruptionBudget.enabled }} +{{- $component := .Values.clustermesh.apiserver.podDisruptionBudget }} +apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: clustermesh-apiserver + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: clustermesh-apiserver + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: clustermesh-apiserver +spec: + {{- with $component.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + {{- with $component.minAvailable }} + minAvailable: {{ . }} + {{- end }} + selector: + matchLabels: + k8s-app: clustermesh-apiserver +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/service.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/service.yaml new file mode 100644 index 00000000..0a7028c5 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/service.yaml @@ -0,0 +1,38 @@ +{{- if (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) }} +apiVersion: v1 +kind: Service +metadata: + name: clustermesh-apiserver + namespace: {{ .Release.Namespace }} + labels: + k8s-app: clustermesh-apiserver + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: clustermesh-apiserver + {{- if or .Values.clustermesh.apiserver.service.annotations .Values.clustermesh.annotations }} + annotations: + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.clustermesh.apiserver.service.type }} + selector: + k8s-app: clustermesh-apiserver + ports: + - port: 2379 + {{- if and (eq "NodePort" .Values.clustermesh.apiserver.service.type) .Values.clustermesh.apiserver.service.nodePort }} + nodePort: {{ .Values.clustermesh.apiserver.service.nodePort }} + {{- end }} + {{- if and (eq "LoadBalancer" .Values.clustermesh.apiserver.service.type) .Values.clustermesh.apiserver.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.clustermesh.apiserver.service.loadBalancerIP }} + {{- end }} + {{- if .Values.clustermesh.apiserver.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.clustermesh.apiserver.service.externalTrafficPolicy }} + {{- end }} + {{- if .Values.clustermesh.apiserver.service.internalTrafficPolicy }} + internalTrafficPolicy: {{ .Values.clustermesh.apiserver.service.internalTrafficPolicy }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/serviceaccount.yaml new file mode 100644 index 00000000..0c8992ba --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.serviceAccounts.clustermeshApiserver.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.clustermeshApiserver.annotations .Values.clustermesh.annotations }} + annotations: + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.clustermeshApiserver.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml new file mode 100644 index 00000000..8ecf150b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml @@ -0,0 +1,77 @@ +{{- $kvstoreMetricsEnabled := and .Values.clustermesh.apiserver.kvstoremesh.enabled .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled -}} +{{- if and + (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) + (or .Values.clustermesh.apiserver.metrics.enabled $kvstoreMetricsEnabled .Values.clustermesh.apiserver.metrics.etcd.enabled) + .Values.clustermesh.apiserver.metrics.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: clustermesh-apiserver + namespace: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.clustermesh.apiserver.metrics.serviceMonitor.annotations .Values.clustermesh.annotations }} + annotations: + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: clustermesh-apiserver + app.kubernetes.io/component: metrics + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + {{- if .Values.clustermesh.apiserver.metrics.enabled }} + - port: apiserv-metrics + interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- if $kvstoreMetricsEnabled }} + - port: kvmesh-metrics + interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.kvstoremesh.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} + - port: etcd-metrics + interval: {{ .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.etcd.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml new file mode 100644 index 00000000..097cb14e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml @@ -0,0 +1,20 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: clustermesh-apiserver-admin-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: clustermesh-apiserver-admin-cert + commonName: {{ include "clustermesh-apiserver-generate-certs.admin-common-name" . }} + dnsNames: + - localhost + duration: {{ printf "%dh0m0s" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24) }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml new file mode 100644 index 00000000..f290fe8e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.externalWorkloads.enabled .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: clustermesh-apiserver-client-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: clustermesh-apiserver-client-cert + commonName: externalworkload + duration: {{ printf "%dh0m0s" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24) }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml new file mode 100644 index 00000000..06bb8bd2 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: clustermesh-apiserver-remote-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: clustermesh-apiserver-remote-cert + commonName: {{ include "clustermesh-apiserver-generate-certs.remote-common-name" . }} + duration: {{ printf "%dh0m0s" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24) }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml new file mode 100644 index 00000000..f7cfd7ad --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml @@ -0,0 +1,31 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: clustermesh-apiserver-server-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: clustermesh-apiserver-server-cert + commonName: clustermesh-apiserver.cilium.io + dnsNames: + - clustermesh-apiserver.cilium.io + - "*.mesh.cilium.io" + - "clustermesh-apiserver.{{ .Release.Namespace }}.svc" + {{- range $dns := .Values.clustermesh.apiserver.tls.server.extraDnsNames }} + - {{ $dns | quote }} + {{- end }} + ipAddresses: + - "127.0.0.1" + - "::1" + {{- range $ip := .Values.clustermesh.apiserver.tls.server.extraIpAddresses }} + - {{ $ip | quote }} + {{- end }} + duration: {{ printf "%dh0m0s" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24) }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl new file mode 100644 index 00000000..52a99089 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl @@ -0,0 +1,73 @@ +{{- define "clustermesh-apiserver-generate-certs.job.spec" }} +{{- $certValiditySecondsStr := printf "%ds" (mul .Values.clustermesh.apiserver.tls.auto.certValidityDuration 24 60 60) -}} +{{- $clustermeshServerSANs := concat (list "*.mesh.cilium.io" (printf "clustermesh-apiserver.%s.svc" .Release.Namespace)) + .Values.clustermesh.apiserver.tls.server.extraDnsNames + .Values.clustermesh.apiserver.tls.server.extraIpAddresses +-}} +spec: + template: + metadata: + labels: + k8s-app: clustermesh-apiserver-generate-certs + {{- with .Values.clustermesh.apiserver.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: certgen + image: {{ include "cilium.image" .Values.certgen.image | quote }} + imagePullPolicy: {{ .Values.certgen.image.pullPolicy }} + command: + - "/usr/bin/cilium-certgen" + args: + - "--cilium-namespace={{ .Release.Namespace }}" + {{- if .Values.debug.enabled }} + - "--debug" + {{- end }} + - "--ca-generate" + - "--ca-reuse-secret" + {{- if and .Values.tls.ca.cert .Values.tls.ca.key }} + - "--ca-secret-name=cilium-ca" + {{- end }} + - "--clustermesh-apiserver-server-cert-generate" + - "--clustermesh-apiserver-server-cert-validity-duration={{ $certValiditySecondsStr }}" + - "--clustermesh-apiserver-server-cert-sans={{ join "," $clustermeshServerSANs }}" + - "--clustermesh-apiserver-admin-cert-generate" + - "--clustermesh-apiserver-admin-cert-validity-duration={{ $certValiditySecondsStr }}" + - "--clustermesh-apiserver-admin-cert-common-name={{ include "clustermesh-apiserver-generate-certs.admin-common-name" . }}" + {{- if .Values.externalWorkloads.enabled }} + - "--clustermesh-apiserver-client-cert-generate" + - "--clustermesh-apiserver-client-cert-validity-duration={{ $certValiditySecondsStr }}" + {{- end }} + {{- if .Values.clustermesh.useAPIServer }} + - "--clustermesh-apiserver-remote-cert-generate" + - "--clustermesh-apiserver-remote-cert-validity-duration={{ $certValiditySecondsStr }}" + - "--clustermesh-apiserver-remote-cert-common-name={{ include "clustermesh-apiserver-generate-certs.remote-common-name" . }}" + {{- end }} + {{- with .Values.certgen.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + hostNetwork: true + {{- with .Values.certgen.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccount: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.clustermeshcertgen.automount }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + {{- with .Values.certgen.extraVolumes }} + volumes: + {{- toYaml . | nindent 6 }} + {{- end }} + affinity: + {{- with .Values.certgen.affinity }} + {{- toYaml . | nindent 8 }} + {{- end }} + ttlSecondsAfterFinished: {{ .Values.certgen.ttlSecondsAfterFinished }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml new file mode 100644 index 00000000..946602b4 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml @@ -0,0 +1,19 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") .Values.clustermesh.apiserver.tls.auto.schedule }} +apiVersion: {{ include "cronjob.apiVersion" . }} +kind: CronJob +metadata: + name: clustermesh-apiserver-generate-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: clustermesh-apiserver-generate-certs + app.kubernetes.io/part-of: cilium +spec: + schedule: {{ .Values.clustermesh.apiserver.tls.auto.schedule | quote }} + concurrencyPolicy: Forbid + jobTemplate: +{{- include "clustermesh-apiserver-generate-certs.job.spec" . | nindent 4 }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml new file mode 100644 index 00000000..b6dd5bf1 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml @@ -0,0 +1,20 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: clustermesh-apiserver-generate-certs + namespace: {{ .Release.Namespace }} + labels: + k8s-app: clustermesh-apiserver-generate-certs + app.kubernetes.io/part-of: cilium + annotations: + "helm.sh/hook": post-install,post-upgrade + {{- with .Values.certgen.annotations.job }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "clustermesh-apiserver-generate-certs.job.spec" . }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml new file mode 100644 index 00000000..79960cba --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml @@ -0,0 +1,40 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") .Values.serviceAccounts.clustermeshcertgen.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: clustermesh-apiserver-generate-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - cilium-ca + verbs: + - get + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - clustermesh-apiserver-server-cert + - clustermesh-apiserver-admin-cert + - clustermesh-apiserver-remote-cert + - clustermesh-apiserver-client-cert + verbs: + - update +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml new file mode 100644 index 00000000..4e67e047 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") .Values.serviceAccounts.clustermeshcertgen.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: clustermesh-apiserver-generate-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: clustermesh-apiserver-generate-certs +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml new file mode 100644 index 00000000..8dfaf52c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") .Values.serviceAccounts.clustermeshcertgen.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.clustermeshcertgen.annotations .Values.clustermesh.annotations }} + annotations: + {{- with .Values.serviceAccounts.clustermeshcertgen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml new file mode 100644 index 00000000..63282c98 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml @@ -0,0 +1,21 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} +{{- $_ := include "cilium.ca.setup" . -}} +{{- $cn := include "clustermesh-apiserver-generate-certs.admin-common-name" . -}} +{{- $dns := list "localhost" }} +{{- $cert := genSignedCert $cn nil $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-admin-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .commonCA.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml new file mode 100644 index 00000000..85137111 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.externalWorkloads.enabled .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} +{{- $_ := include "cilium.ca.setup" . -}} +{{- $cn := "externalworkload" }} +{{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-client-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .commonCA.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml new file mode 100644 index 00000000..7bfb1c5a --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} +{{- $_ := include "cilium.ca.setup" . -}} +{{- $cn := include "clustermesh-apiserver-generate-certs.remote-common-name" . -}} +{{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-remote-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .commonCA.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml new file mode 100644 index 00000000..f958f353 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml @@ -0,0 +1,22 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} +{{- $_ := include "cilium.ca.setup" . -}} +{{- $cn := "clustermesh-apiserver.cilium.io" }} +{{- $ip := concat (list "127.0.0.1" "::1") .Values.clustermesh.apiserver.tls.server.extraIpAddresses }} +{{- $dns := concat (list $cn "*.mesh.cilium.io" (printf "clustermesh-apiserver.%s.svc" .Release.Namespace)) .Values.clustermesh.apiserver.tls.server.extraDnsNames }} +{{- $cert := genSignedCert $cn $ip $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-server-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .commonCA.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml new file mode 100644 index 00000000..de038ca8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) (not .Values.clustermesh.apiserver.tls.auto.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-admin-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.clustermesh.apiserver.tls.admin.cert | required "missing clustermesh.apiserver.tls.admin.cert" }} + tls.key: {{ .Values.clustermesh.apiserver.tls.admin.key | required "missing clustermesh.apiserver.tls.admin.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml new file mode 100644 index 00000000..b8ea7b60 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.externalWorkloads.enabled (not .Values.clustermesh.apiserver.tls.auto.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-client-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.clustermesh.apiserver.tls.client.cert | required "missing clustermesh.apiserver.tls.client.cert" }} + tls.key: {{ .Values.clustermesh.apiserver.tls.client.key | required "missing clustermesh.apiserver.tls.client.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml new file mode 100644 index 00000000..8a253eb5 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.clustermesh.useAPIServer (not .Values.clustermesh.apiserver.tls.auto.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-remote-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.clustermesh.apiserver.tls.remote.cert | required "missing clustermesh.apiserver.tls.remote.cert" }} + tls.key: {{ .Values.clustermesh.apiserver.tls.remote.key | required "missing clustermesh.apiserver.tls.remote.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml new file mode 100644 index 00000000..8c239b07 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) (not .Values.clustermesh.apiserver.tls.auto.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: clustermesh-apiserver-server-cert + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.clustermesh.apiserver.tls.server.cert | required "missing clustermesh.apiserver.tls.server.cert" }} + tls.key: {{ .Values.clustermesh.apiserver.tls.server.key | required "missing clustermesh.apiserver.tls.server.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml new file mode 100644 index 00000000..e1cbf95f --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml @@ -0,0 +1,25 @@ +{{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clustermesh-remote-users + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +data: + users.yaml: | + users: + {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} + - name: remote-{{ .Values.cluster.name }} + role: remote + {{- end }} + {{- range .Values.clustermesh.config.clusters }} + - name: remote-{{ .name }} + role: remote + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-config/_helpers.tpl b/magnum_cluster_api/charts/cilium/templates/clustermesh-config/_helpers.tpl new file mode 100644 index 00000000..5cd31486 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-config/_helpers.tpl @@ -0,0 +1,24 @@ +{{- define "clustermesh-config-generate-etcd-cfg" }} +{{- $cluster := index . 0 -}} +{{- $domain := index . 1 -}} +{{- $override := index . 2 -}} +{{- /* The parenthesis around $cluster.tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} +{{- $prefix := ternary "common-" (printf "%s." $cluster.name) (or (ne $override "") (empty ($cluster.tls).cert) (empty ($cluster.tls).key)) -}} + +endpoints: +{{- if ne $override "" }} +- {{ $override }} +{{- else if $cluster.ips }} +- https://{{ $cluster.name }}.{{ $domain }}:{{ $cluster.port }} +{{- else }} +- https://{{ $cluster.address | required "missing clustermesh.apiserver.config.clusters.address" }}:{{ $cluster.port }} +{{- end }} +{{- if not (empty ($cluster.tls).caCert) }} +{{- /* The custom CA configuration takes effect only if a custom certificate and key are also set */}} +trusted-ca-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client-ca.crt +{{- else }} +trusted-ca-file: /var/lib/cilium/clustermesh/common-etcd-client-ca.crt +{{- end }} +key-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client.key +cert-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client.crt +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml new file mode 100644 index 00000000..78ddecd9 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.clustermesh.config.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cilium-clustermesh + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- $kvstoremesh := and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.kvstoremesh.enabled }} + {{- $override := ternary (printf "https://clustermesh-apiserver.%s.svc:2379" .Release.Namespace) "" $kvstoremesh }} + {{- range .Values.clustermesh.config.clusters }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $override) | b64enc }} + {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} + {{- if and (eq $override "") (.tls).cert (.tls).key }} + {{- if .tls.caCert }} + {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert }} + {{- end }} + {{ .name }}.etcd-client.key: {{ .tls.key }} + {{ .name }}.etcd-client.crt: {{ .tls.cert }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml b/magnum_cluster_api/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml new file mode 100644 index 00000000..91ae0320 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled .Values.clustermesh.apiserver.kvstoremesh.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: cilium-kvstoremesh + namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- range .Values.clustermesh.config.clusters }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain "") | b64enc }} + {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} + {{- if and (.tls).cert (.tls).key }} + {{- if .tls.caCert }} + {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert }} + {{- end }} + {{ .name }}.etcd-client.key: {{ .tls.key }} + {{ .name }}.etcd-client.crt: {{ .tls.cert }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml new file mode 100644 index 00000000..22a81c2c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml @@ -0,0 +1,79 @@ +{{- if .Values.etcd.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - etcd.database.coreos.com + resources: + - etcdclusters + verbs: + - get + - delete + - create + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - delete + - get + - create +- apiGroups: + - "" + resources: + - deployments + verbs: + - delete + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - delete +- apiGroups: + - apps + resources: + - deployments + verbs: + - delete + - create + - get + - update +- apiGroups: + - "" + resources: + - componentstatuses + verbs: + - get +- apiGroups: + - extensions + resources: + - deployments + verbs: + - delete + - create + - get + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - delete +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml new file mode 100644 index 00000000..25a92e10 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.etcd.managed .Values.serviceAccounts.etcd.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-etcd-operator +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.etcd.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml new file mode 100644 index 00000000..5946219f --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml @@ -0,0 +1,128 @@ +{{- if .Values.etcd.managed }} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator + app.kubernetes.io/name: cilium-etcd-operator + app.kubernetes.io/part-of: cilium + name: cilium-etcd-operator + namespace: {{ .Release.Namespace }} + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator +{{- with .Values.etcd.updateStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} +{{- end }} + template: + metadata: +{{- with .Values.etcd.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} +{{- end }} + labels: + io.cilium/app: etcd-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-etcd-operator + name: cilium-etcd-operator +{{- with .Values.etcd.podLabels }} + {{- toYaml . | nindent 8 }} +{{- end }} + spec: +{{- if .Values.etcd.affinity }} + affinity: +{{ toYaml .Values.etcd.affinity | indent 8 }} +{{- end }} +{{- with .Values.etcd.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range $constraint := . }} + - {{ toYaml $constraint | nindent 8 | trim }} + {{- if not $constraint.labelSelector }} + labelSelector: + matchLabels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.imagePullSecrets | indent 8 }} +{{- end }} +{{- with .Values.etcd.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} +{{- end }} + containers: + - args: +{{- with .Values.etcd.extraArgs }} + {{- toYaml . | trim | nindent 8 }} +{{- end }} + #- --etcd-node-selector=disktype=ssd,cputype=high + command: + - /usr/bin/cilium-etcd-operator + env: + - name: CILIUM_ETCD_OPERATOR_CLUSTER_DOMAIN + value: "{{ .Values.etcd.clusterDomain }}" + - name: CILIUM_ETCD_OPERATOR_ETCD_CLUSTER_SIZE + value: "{{ .Values.etcd.clusterSize }}" + - name: CILIUM_ETCD_OPERATOR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_ETCD_OPERATOR_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: CILIUM_ETCD_OPERATOR_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_MODE + value: "revision" + - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_RETENTION + value: "25000" + image: {{ include "cilium.image" .Values.etcd.image | quote }} + imagePullPolicy: {{ .Values.etcd.image.pullPolicy }} + name: cilium-etcd-operator + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.etcd.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.etcd.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 8 }} + {{- end }} + dnsPolicy: ClusterFirst + hostNetwork: true + priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.clustermesh.apiserver.priorityClassName "system-cluster-critical") }} + restartPolicy: Always + serviceAccount: {{ .Values.serviceAccounts.etcd.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.etcd.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.etcd.automount }} +{{- with .Values.etcd.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} +{{- end }} +{{- with .Values.etcd.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 6 }} +{{- end }} + {{- with .Values.etcd.extraVolumes }} + volumes: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml new file mode 100644 index 00000000..7d7b765a --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.etcd.managed .Values.serviceAccounts.etcd.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.etcd.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.etcd.annotations .Values.etcd.annotations }} + annotations: + {{- with .Values.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml new file mode 100644 index 00000000..72cb4e6e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml @@ -0,0 +1,60 @@ +{{- if .Values.etcd.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - etcd.database.coreos.com + resources: + - etcdclusters + - etcdbackups + - etcdrestores + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + - deployments + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments + verbs: + - '*' +- apiGroups: + - extensions + resources: + - deployments + verbs: + - create + - get + - list + - patch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml new file mode 100644 index 00000000..b729ea71 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.etcd.managed }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: etcd-operator +subjects: +- kind: ServiceAccount + name: cilium-etcd-sa + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml new file mode 100644 index 00000000..06ee39c1 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.etcd.managed }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium-etcd-sa + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.etcd.annotations .Values.etcd.annotations }} + annotations: + {{- with .Values.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml b/magnum_cluster_api/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml new file mode 100644 index 00000000..5939b4ae --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.etcd.managed .Values.etcd.podDisruptionBudget.enabled }} +{{- $component := .Values.etcd.podDisruptionBudget }} +apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: cilium-etcd-operator + namespace: {{ .Release.Namespace }} + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator + app.kubernetes.io/name: cilium-etcd-operator + app.kubernetes.io/part-of: cilium +spec: + {{- with $component.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + {{- with $component.minAvailable }} + minAvailable: {{ . }} + {{- end }} + selector: + matchLabels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/configmap.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/configmap.yaml new file mode 100644 index 00000000..93f5b8d8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/configmap.yaml @@ -0,0 +1,51 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled }} +{{- $peerSvcPort := .Values.hubble.peerService.servicePort -}} +{{- if not .Values.hubble.peerService.servicePort }} +{{- $peerSvcPort = (.Values.hubble.tls.enabled | ternary 443 80) -}} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yaml: | + cluster-name: {{ .Values.cluster.name }} + peer-service: "hubble-peer.{{ .Release.Namespace }}.svc.{{ .Values.hubble.peerService.clusterDomain }}:{{ $peerSvcPort }}" + listen-address: {{ .Values.hubble.relay.listenHost }}:{{ .Values.hubble.relay.listenPort }} + gops: {{ .Values.hubble.relay.gops.enabled }} + gops-port: {{ .Values.hubble.relay.gops.port | quote }} + {{- if .Values.hubble.relay.pprof.enabled }} + pprof: {{ .Values.hubble.relay.pprof.enabled | quote }} + pprof-address: {{ .Values.hubble.relay.pprof.address | quote }} + pprof-port: {{ .Values.hubble.relay.pprof.port | quote }} + {{- end }} + {{- if .Values.hubble.relay.prometheus.enabled }} + metrics-listen-address: ":{{ .Values.hubble.relay.prometheus.port }}" + {{- end }} + dial-timeout: {{ .Values.hubble.relay.dialTimeout }} + retry-timeout: {{ .Values.hubble.relay.retryTimeout }} + sort-buffer-len-max: {{ .Values.hubble.relay.sortBufferLenMax }} + sort-buffer-drain-timeout: {{ .Values.hubble.relay.sortBufferDrainTimeout }} + {{- if .Values.hubble.tls.enabled }} + tls-hubble-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-hubble-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + {{- else }} + disable-client-tls: true + {{- end }} + {{- if and .Values.hubble.tls.enabled .Values.hubble.relay.tls.server.enabled }} + tls-relay-server-cert-file: /var/lib/hubble-relay/tls/server.crt + tls-relay-server-key-file: /var/lib/hubble-relay/tls/server.key + {{- if .Values.hubble.relay.tls.server.mtls }} + tls-relay-client-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + {{- end }} + {{- else }} + disable-server-tls: true + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/deployment.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/deployment.yaml new file mode 100644 index 00000000..52b9eba5 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/deployment.yaml @@ -0,0 +1,201 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: hubble-relay + app.kubernetes.io/name: hubble-relay + app.kubernetes.io/part-of: cilium +spec: + replicas: {{ .Values.hubble.relay.replicas }} + selector: + matchLabels: + k8s-app: hubble-relay + {{- with .Values.hubble.relay.updateStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- if .Values.hubble.relay.rollOutPods }} + # ensure pods roll when configmap updates + cilium.io/hubble-relay-configmap-checksum: {{ include (print $.Template.BasePath "/hubble-relay/configmap.yaml") . | sha256sum | quote }} + {{- end }} + {{- with .Values.hubble.relay.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + k8s-app: hubble-relay + app.kubernetes.io/name: hubble-relay + app.kubernetes.io/part-of: cilium + {{- with .Values.hubble.relay.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.hubble.relay.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: hubble-relay + {{- with .Values.hubble.relay.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ include "cilium.image" .Values.hubble.relay.image | quote }} + imagePullPolicy: {{ .Values.hubble.relay.image.pullPolicy }} + command: + - hubble-relay + args: + - serve + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.hubble.relay.listenPort }} + {{- if .Values.hubble.relay.prometheus.enabled }} + - name: prometheus + containerPort: {{ .Values.hubble.relay.prometheus.port }} + protocol: TCP + {{- end }} + readinessProbe: + {{- include "hubble-relay.probe" . | nindent 12 }} + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 5 + {{- end }} + livenessProbe: + {{- include "hubble-relay.probe" . | nindent 12 }} + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 60 + {{- end }} + {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} + startupProbe: + # give the relay one minute to start up + {{- include "hubble-relay.probe" . | nindent 12 }} + failureThreshold: 20 + periodSeconds: 3 + {{- end }} + {{- with .Values.hubble.relay.extraEnv }} + env: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + {{- with .Values.hubble.relay.resources }} + resources: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/hubble-relay + readOnly: true + {{- if .Values.hubble.tls.enabled }} + - name: tls + mountPath: /var/lib/hubble-relay/tls + readOnly: true + {{- end }} + {{- with .Values.hubble.relay.extraVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + restartPolicy: Always + priorityClassName: {{ .Values.hubble.relay.priorityClassName }} + serviceAccount: {{ .Values.serviceAccounts.relay.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.relay.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.relay.automount }} + terminationGracePeriodSeconds: {{ .Values.hubble.relay.terminationGracePeriodSeconds }} + {{- with .Values.hubble.relay.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hubble.relay.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range $constraint := . }} + - {{ toYaml $constraint | nindent 8 | trim }} + {{- if not $constraint.labelSelector }} + labelSelector: + matchLabels: + k8s-app: hubble-relay + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.hubble.relay.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.hubble.relay.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + {{- if .Values.hubble.tls.enabled }} + - name: tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: hubble-server-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: hubble-server-ca.crt + {{- end }} + {{- if .Values.hubble.relay.tls.server.enabled }} + - secret: + name: hubble-relay-server-certs + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + {{- end }} + {{- end }} + {{- with .Values.hubble.relay.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} + +{{- define "hubble-relay.probe" }} +{{- /* This distinction can be removed once we drop support for k8s 1.23 */}} +{{- if semverCompare ">=1.24-0" .Capabilities.KubeVersion.Version -}} +grpc: + port: 4222 +{{- else }} +exec: + command: + - grpc_health_probe + - -addr=localhost:4222 +{{- end }} +timeoutSeconds: 3 +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/metrics-service.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/metrics-service.yaml new file mode 100644 index 00000000..1066c6c4 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/metrics-service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled .Values.hubble.relay.prometheus.enabled }} +# We use a separate service from hubble-relay which can be exposed externally +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay-metrics + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: hubble-relay +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - name: metrics + port: {{ .Values.hubble.relay.prometheus.port }} + protocol: TCP + targetPort: prometheus +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml new file mode 100644 index 00000000..4fd6da9b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled .Values.hubble.relay.podDisruptionBudget.enabled }} +{{- $component := .Values.hubble.relay.podDisruptionBudget }} +apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: hubble-relay + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: hubble-relay + app.kubernetes.io/name: hubble-relay + app.kubernetes.io/part-of: cilium +spec: + {{- with $component.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + {{- with $component.minAvailable }} + minAvailable: {{ . }} + {{- end }} + selector: + matchLabels: + k8s-app: hubble-relay +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/service.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/service.yaml new file mode 100644 index 00000000..39259473 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/service.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: hubble-relay + app.kubernetes.io/name: hubble-relay + app.kubernetes.io/part-of: cilium +spec: + type: {{ .Values.hubble.relay.service.type | quote }} + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + {{- if .Values.hubble.relay.servicePort }} + port: {{ .Values.hubble.relay.servicePort }} + {{- else }} + port: {{ .Values.hubble.relay.tls.server.enabled | ternary 443 80 }} + {{- end }} + targetPort: {{ .Values.hubble.relay.listenPort }} + {{- if and (eq "NodePort" .Values.hubble.relay.service.type) .Values.hubble.relay.service.nodePort }} + nodePort: {{ .Values.hubble.relay.service.nodePort }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/serviceaccount.yaml new file mode 100644 index 00000000..cf56d631 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled .Values.serviceAccounts.relay.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.relay.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.relay.annotations .Values.hubble.relay.annotations }} + annotations: + {{- with .Values.hubble.relay.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.relay.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-relay/servicemonitor.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-relay/servicemonitor.yaml new file mode 100644 index 00000000..4e41fdf3 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-relay/servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.hubble.enabled .Values.hubble.relay.enabled .Values.hubble.relay.prometheus.enabled .Values.hubble.relay.prometheus.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: hubble-relay + namespace: {{ .Values.hubble.relay.prometheus.serviceMonitor.namespace | default .Release.Namespace }} + labels: + {{- with .Values.hubble.relay.prometheus.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.hubble.relay.prometheus.serviceMonitor.annotations .Values.hubble.relay.annotations }} + annotations: + {{- with .Values.hubble.relay.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.relay.prometheus.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: hubble-relay + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: metrics + interval: {{ .Values.hubble.relay.prometheus.serviceMonitor.interval | quote }} + path: /metrics + {{- with .Values.hubble.relay.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.relay.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/_nginx.tpl b/magnum_cluster_api/charts/cilium/templates/hubble-ui/_nginx.tpl new file mode 100644 index 00000000..e787b5aa --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/_nginx.tpl @@ -0,0 +1,61 @@ +{{- define "hubble-ui.nginx.conf" }} +server { + listen 8081; +{{- if .Values.hubble.ui.frontend.server.ipv6.enabled }} + listen [::]:8081; +{{- end }} + server_name localhost; + root /app; + index index.html; + client_max_body_size 1G; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + # CORS + add_header Access-Control-Allow-Methods "GET, POST, PUT, HEAD, DELETE, OPTIONS"; + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Max-Age 1728000; + add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message; + add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout; + if ($request_method = OPTIONS) { + return 204; + } + # /CORS + + location {{ .Values.hubble.ui.baseUrl }}api { + {{- if not (eq .Values.hubble.ui.baseUrl "/") }} + rewrite ^{{ (trimSuffix "/" .Values.hubble.ui.baseUrl) }}(/.*)$ $1 break; + {{- end }} + proxy_http_version 1.1; + proxy_pass_request_headers on; + proxy_hide_header Access-Control-Allow-Origin; + {{- if eq .Values.hubble.ui.baseUrl "/" }} + proxy_pass http://127.0.0.1:8090; + {{- else }} + proxy_pass http://127.0.0.1:8090/; + {{- end }} + } + + {{- if not (eq .Values.hubble.ui.baseUrl "/") }} + sub_filter_once on; + sub_filter '' ''; + {{- end }} + location {{ .Values.hubble.ui.baseUrl }} { + {{- if not (eq .Values.hubble.ui.baseUrl "/") }} + rewrite ^{{ (trimSuffix "/" .Values.hubble.ui.baseUrl) }}(/.*)$ $1 break; + {{- end }} + # double `/index.html` is required here + try_files $uri $uri/ /index.html /index.html; + } + + # Liveness probe + location /healthz { + access_log off; + add_header Content-Type text/plain; + return 200 'ok'; + } + } +} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrole.yaml new file mode 100644 index 00000000..5df709f7 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrole.yaml @@ -0,0 +1,50 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.serviceAccounts.ui.create }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrolebinding.yaml new file mode 100644 index 00000000..d091786b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.serviceAccounts.ui.create }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.ui.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/configmap.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/configmap.yaml new file mode 100644 index 00000000..8b5f0141 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/configmap.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-nginx + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + nginx.conf: {{ include "hubble-ui.nginx.conf" . | trim | quote }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/deployment.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/deployment.yaml new file mode 100644 index 00000000..a7dd5cb8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/deployment.yaml @@ -0,0 +1,216 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled }} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: hubble-ui + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: hubble-ui + app.kubernetes.io/name: hubble-ui + app.kubernetes.io/part-of: cilium +spec: + replicas: {{ .Values.hubble.ui.replicas }} + selector: + matchLabels: + k8s-app: hubble-ui + {{- with .Values.hubble.ui.updateStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- if .Values.hubble.ui.rollOutPods }} + # ensure pods roll when configmap updates + cilium.io/hubble-ui-nginx-configmap-checksum: {{ include (print $.Template.BasePath "/hubble-ui/configmap.yaml") . | sha256sum | quote }} + {{- end }} + {{- with .Values.hubble.ui.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + k8s-app: hubble-ui + app.kubernetes.io/name: hubble-ui + app.kubernetes.io/part-of: cilium + {{- with .Values.hubble.ui.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.hubble.ui.securityContext }} + {{- if .enabled }} + securityContext: + {{- omit . "enabled" | toYaml | nindent 8 }} + {{- end}} + {{- end }} + priorityClassName: {{ .Values.hubble.ui.priorityClassName }} + serviceAccount: {{ .Values.serviceAccounts.ui.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.ui.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: frontend + image: {{ include "cilium.image" .Values.hubble.ui.frontend.image | quote }} + imagePullPolicy: {{ .Values.hubble.ui.frontend.image.pullPolicy }} + ports: + - name: http + containerPort: 8081 + {{- with .Values.hubble.ui.frontend.extraEnv }} + env: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + readinessProbe: + httpGet: + path: / + port: 8081 + {{- with .Values.hubble.ui.frontend.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: tmp-dir + mountPath: /tmp + {{- with .Values.hubble.ui.frontend.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.hubble.ui.frontend.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + - name: backend + image: {{ include "cilium.image" .Values.hubble.ui.backend.image | quote }} + imagePullPolicy: {{ .Values.hubble.ui.backend.image.pullPolicy }} + env: + - name: EVENTS_SERVER_PORT + value: "8090" + {{- if .Values.hubble.relay.tls.server.enabled }} + - name: FLOWS_API_ADDR + value: "hubble-relay:443" + - name: TLS_TO_RELAY_ENABLED + value: "true" + - name: TLS_RELAY_SERVER_NAME + value: {{ .Values.hubble.relay.tls.server.relayName }} + - name: TLS_RELAY_CA_CERT_FILES + value: /var/lib/hubble-ui/certs/hubble-relay-ca.crt + - name: TLS_RELAY_CLIENT_CERT_FILE + value: /var/lib/hubble-ui/certs/client.crt + - name: TLS_RELAY_CLIENT_KEY_FILE + value: /var/lib/hubble-ui/certs/client.key + {{- else }} + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + {{- end }} + {{- with .Values.hubble.ui.backend.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- if .Values.hubble.ui.backend.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 8090 + {{- end }} + {{- if .Values.hubble.ui.backend.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 8090 + {{- end }} + ports: + - name: grpc + containerPort: 8090 + {{- with .Values.hubble.ui.backend.resources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.hubble.relay.tls.server.enabled }} + - name: hubble-ui-client-certs + mountPath: /var/lib/hubble-ui/certs + readOnly: true + {{- end }} + {{- with .Values.hubble.ui.backend.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.hubble.ui.backend.securityContext }} + securityContext: + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- with .Values.hubble.ui.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hubble.ui.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range $constraint := . }} + - {{ toYaml $constraint | nindent 8 | trim }} + {{- if not $constraint.labelSelector }} + labelSelector: + matchLabels: + k8s-app: hubble-ui + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.hubble.ui.nodeSelector }} + nodeSelector: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.hubble.ui.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf + - emptyDir: {} + name: tmp-dir + {{- if .Values.hubble.relay.tls.server.enabled }} + - name: hubble-ui-client-certs + {{- if .Values.hubble.ui.standalone.enabled }} + {{- toYaml .Values.hubble.ui.standalone.tls.certsVolume | nindent 8 }} + {{- else }} + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-ui-client-certs + items: + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + {{- if not .Values.tls.caBundle.enabled }} + - key: ca.crt + path: hubble-relay-ca.crt + {{- else }} + - {{ .Values.tls.caBundle.useSecret | ternary "secret" "configMap" }}: + name: {{ .Values.tls.caBundle.name }} + items: + - key: {{ .Values.tls.caBundle.key }} + path: hubble-relay-ca.crt + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.hubble.ui.frontend.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.hubble.ui.backend.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/ingress.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/ingress.yaml new file mode 100644 index 00000000..2c0ff7d3 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/ingress.yaml @@ -0,0 +1,40 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.hubble.ui.ingress.enabled }} +{{- $baseUrl := .Values.hubble.ui.baseUrl -}} +apiVersion: {{ template "ingress.apiVersion" . }} +kind: Ingress +metadata: + name: hubble-ui + namespace: {{ .Release.Namespace }} + labels: + k8s-app: hubble-ui + app.kubernetes.io/name: hubble-ui + app.kubernetes.io/part-of: cilium + {{- with .Values.hubble.ui.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.hubble.ui.ingress.annotations .Values.hubble.ui.annotations }} + annotations: + {{- with .Values.hubble.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.ui.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + {{- if .Values.hubble.ui.ingress.className }} + ingressClassName: {{ .Values.hubble.ui.ingress.className }} + {{- end }} + {{- if .Values.hubble.ui.ingress.tls }} + tls: + {{- toYaml .Values.hubble.ui.ingress.tls | nindent 4 }} + {{- end }} + rules: + {{- range .Values.hubble.ui.ingress.hosts }} + - host: {{ . }} + http: + paths: + - path: {{ $baseUrl | quote }} + {{- include "ingress.paths" $ | nindent 12 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml new file mode 100644 index 00000000..af3b6705 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml @@ -0,0 +1,26 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.hubble.ui.podDisruptionBudget.enabled }} +{{- $component := .Values.hubble.ui.podDisruptionBudget }} +apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: hubble-ui + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: hubble-ui + app.kubernetes.io/name: hubble-ui + app.kubernetes.io/part-of: cilium +spec: + {{- with $component.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + {{- with $component.minAvailable }} + minAvailable: {{ . }} + {{- end }} + selector: + matchLabels: + k8s-app: hubble-ui +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/service.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/service.yaml new file mode 100644 index 00000000..a820b342 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/service.yaml @@ -0,0 +1,31 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + namespace: {{ .Release.Namespace }} + {{- if or .Values.hubble.ui.service.annotations .Values.hubble.ui.annotations }} + annotations: + {{- with .Values.hubble.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.ui.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + k8s-app: hubble-ui + app.kubernetes.io/name: hubble-ui + app.kubernetes.io/part-of: cilium +spec: + type: {{ .Values.hubble.ui.service.type | quote }} + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 + {{- if and (eq "NodePort" .Values.hubble.ui.service.type) .Values.hubble.ui.service.nodePort }} + nodePort: {{ .Values.hubble.ui.service.nodePort }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble-ui/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/hubble-ui/serviceaccount.yaml new file mode 100644 index 00000000..dc02ea2b --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble-ui/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.serviceAccounts.ui.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.ui.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.ui.annotations .Values.hubble.ui.annotations }} + annotations: + {{- with .Values.hubble.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/dashboards-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/dashboards-configmap.yaml new file mode 100644 index 00000000..c668ebfd --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/dashboards-configmap.yaml @@ -0,0 +1,30 @@ +{{- if .Values.hubble.metrics.dashboards.enabled }} +{{- $files := .Files.Glob "files/hubble/dashboards/*.json" }} +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Values.hubble.metrics.dashboards.namespace | default $.Release.Namespace }} + labels: + k8s-app: hubble + app.kubernetes.io/name: hubble + app.kubernetes.io/part-of: cilium + {{- if $.Values.hubble.metrics.dashboards.label }} + {{ $.Values.hubble.metrics.dashboards.label }}: {{ ternary $.Values.hubble.metrics.dashboards.labelValue "1" (not (empty $.Values.hubble.metrics.dashboards.labelValue)) | quote }} + {{- end }} + {{- if or $.Values.hubble.metrics.dashboards.annotations $.Values.hubble.annotations }} + annotations: + {{- with $.Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.hubble.metrics.dashboards.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/metrics-service.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/metrics-service.yaml new file mode 100644 index 00000000..ab5cfbab --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/metrics-service.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.hubble.enabled .Values.hubble.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: hubble-metrics + namespace: {{ .Release.Namespace }} + labels: + k8s-app: hubble + app.kubernetes.io/name: hubble + app.kubernetes.io/part-of: cilium + annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.metrics.serviceAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if not .Values.hubble.metrics.serviceMonitor.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.hubble.metrics.port | quote }} + {{- end }} +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: {{ .Values.hubble.metrics.port }} + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/peer-service.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/peer-service.yaml new file mode 100644 index 00000000..7ba56456 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/peer-service.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.agent .Values.hubble.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + {{- if .Values.hubble.peerService.servicePort }} + port: {{ .Values.hubble.peerService.servicePort }} + {{- else }} + port: {{ .Values.hubble.tls.enabled | ternary 443 80 }} + {{- end }} + protocol: TCP + targetPort: {{ .Values.hubble.peerService.targetPort }} +{{- if semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion }} + internalTrafficPolicy: Local +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/servicemonitor.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/servicemonitor.yaml new file mode 100644 index 00000000..3b3ba8ba --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/servicemonitor.yaml @@ -0,0 +1,44 @@ +{{- if and .Values.hubble.enabled .Values.hubble.metrics.enabled .Values.hubble.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: hubble + namespace: {{ .Values.prometheus.serviceMonitor.namespace | default .Release.Namespace }} + labels: + app.kubernetes.io/part-of: cilium + {{- with .Values.hubble.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.hubble.metrics.serviceMonitor.annotations .Values.hubble.annotations }} + annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.metrics.serviceMonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + selector: + matchLabels: + k8s-app: hubble + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: hubble-metrics + interval: {{ .Values.hubble.metrics.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.hubble.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.hubble.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.hubble.metrics.serviceMonitor.jobLabel | quote }} +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml new file mode 100644 index 00000000..1dd96b18 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "certmanager") .Values.hubble.relay.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: hubble-relay-client-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: hubble-relay-client-certs + commonName: "*.hubble-relay.cilium.io" + dnsNames: + - "*.hubble-relay.cilium.io" + duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} + privateKey: + rotationPolicy: Always +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml new file mode 100644 index 00000000..845b4fb8 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "certmanager") .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: hubble-relay-server-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: hubble-relay-server-certs + commonName: "*.hubble-relay.cilium.io" + dnsNames: + - "*.hubble-relay.cilium.io" + {{- range $dns := .Values.hubble.relay.tls.server.extraDnsNames }} + - {{ $dns | quote }} + {{- end }} + {{- if .Values.hubble.relay.tls.server.extraIpAddresses }} + ipAddresses: + {{- range $ip := .Values.hubble.relay.tls.server.extraIpAddresses }} + - {{ $ip | quote }} + {{- end }} + {{- end }} + duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} + privateKey: + rotationPolicy: Always +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml new file mode 100644 index 00000000..5f202e10 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "certmanager") }} +{{- $cn := list "*" (.Values.cluster.name | replace "." "-") "hubble-grpc.cilium.io" | join "." }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: hubble-server-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: hubble-server-certs + commonName: {{ $cn | quote }} + dnsNames: + - {{ $cn | quote }} + {{- range $dns := .Values.hubble.tls.server.extraDnsNames }} + - {{ $dns | quote }} + {{- end }} + {{- if .Values.hubble.tls.server.extraIpAddresses }} + ipAddresses: + {{- range $ip := .Values.hubble.tls.server.extraIpAddresses }} + - {{ $ip | quote }} + {{- end }} + {{- end }} + duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} + privateKey: + rotationPolicy: Always +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml new file mode 100644 index 00000000..5006666e --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "certmanager") .Values.hubble.ui.enabled .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: hubble-ui-client-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + issuerRef: + {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} + secretName: hubble-ui-client-certs + commonName: "*.hubble-ui.cilium.io" + dnsNames: + - "*.hubble-ui.cilium.io" + duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} + privateKey: + rotationPolicy: Always +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl new file mode 100644 index 00000000..4bee347a --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/_job-spec.tpl @@ -0,0 +1,68 @@ +{{- define "hubble-generate-certs.job.spec" }} +{{- $certValiditySecondsStr := printf "%ds" (mul .Values.hubble.tls.auto.certValidityDuration 24 60 60) -}} +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + {{- with .Values.certgen.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: certgen + image: {{ include "cilium.image" .Values.certgen.image | quote }} + imagePullPolicy: {{ .Values.certgen.image.pullPolicy }} + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace={{ .Release.Namespace }}" + {{- if .Values.debug.enabled }} + - "--debug" + {{- end }} + - "--ca-generate" + - "--ca-reuse-secret" + {{- if and .Values.tls.ca.cert .Values.tls.ca.key }} + - "--ca-secret-name=cilium-ca" + {{- end }} + - "--hubble-server-cert-generate" + - "--hubble-server-cert-common-name={{ list "*" (.Values.cluster.name | replace "." "-") "hubble-grpc.cilium.io" | join "." }}" + - "--hubble-server-cert-validity-duration={{ $certValiditySecondsStr }}" + {{- if .Values.hubble.relay.enabled }} + - "--hubble-relay-client-cert-generate" + - "--hubble-relay-client-cert-validity-duration={{ $certValiditySecondsStr }}" + {{- end }} + {{- if and .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} + - "--hubble-relay-server-cert-generate" + - "--hubble-relay-server-cert-validity-duration={{ $certValiditySecondsStr }}" + {{- end }} + {{- with .Values.certgen.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + hostNetwork: true + {{- with .Values.certgen.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccount: {{ .Values.serviceAccounts.hubblecertgen.name | quote }} + serviceAccountName: {{ .Values.serviceAccounts.hubblecertgen.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.hubblecertgen.automount }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + {{- with .Values.certgen.extraVolumes }} + volumes: + {{- toYaml . | nindent 6 }} + {{- end }} + affinity: + {{- with .Values.certgen.affinity }} + {{- toYaml . | nindent 8 }} + {{- end }} + ttlSecondsAfterFinished: {{ .Values.certgen.ttlSecondsAfterFinished }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml new file mode 100644 index 00000000..74d07831 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrole.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") .Values.serviceAccounts.hubblecertgen.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - cilium-ca + verbs: + - get + - update +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml new file mode 100644 index 00000000..5938f16c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") .Values.serviceAccounts.hubblecertgen.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.hubblecertgen.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml new file mode 100644 index 00000000..fa996608 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") .Values.hubble.tls.auto.schedule }} +apiVersion: {{ include "cronjob.apiVersion" . }} +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: {{ .Release.Namespace }} + labels: + k8s-app: hubble-generate-certs + app.kubernetes.io/name: hubble-generate-certs + app.kubernetes.io/part-of: cilium + {{- if or .Values.certgen.annotations.cronJob .Values.hubble.annotations }} + annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.certgen.annotations.cronJob }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + schedule: {{ .Values.hubble.tls.auto.schedule | quote }} + concurrencyPolicy: Forbid + jobTemplate: + {{- include "hubble-generate-certs.job.spec" . | nindent 4 }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/job.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/job.yaml new file mode 100644 index 00000000..9831fdc6 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/job.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs + namespace: {{ .Release.Namespace }} + labels: + k8s-app: hubble-generate-certs + app.kubernetes.io/name: hubble-generate-certs + app.kubernetes.io/part-of: cilium + annotations: + "helm.sh/hook": post-install,post-upgrade + {{- with .Values.certgen.annotations.job }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "hubble-generate-certs.job.spec" . }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml new file mode 100644 index 00000000..62a8de80 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") .Values.serviceAccounts.hubblecertgen.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccounts.hubblecertgen.name | quote }} + namespace: {{ .Release.Namespace }} + {{- if or .Values.serviceAccounts.hubblecertgen.annotations .Values.hubble.annotations }} + annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.hubblecertgen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/_helpers.tpl b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/_helpers.tpl new file mode 100644 index 00000000..79babf94 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/_helpers.tpl @@ -0,0 +1,31 @@ +{{/* +Generate TLS certificates for Hubble Server and Hubble Relay. + +Note: Always use this template as follows: + + {{- $_ := include "hubble-generate-certs.helm.setup-ca" . -}} + +The assignment to `$_` is required because we store the generated CI in a global `ca` variable. +Please, don't try to "simplify" this, as without this trick, every generated +certificate would be signed by a different CA. +*/}} +{{- define "hubble-generate-certs.helm.setup-ca" }} + {{- if not .ca }} + {{- $ca := "" -}} + {{- $crt := .Values.tls.ca.cert -}} + {{- $key := .Values.tls.ca.key -}} + {{- if and $crt $key }} + {{- $ca = buildCustomCert $crt $key -}} + {{- else }} + {{- $_ := include "cilium.ca.setup" . -}} + {{- with lookup "v1" "Secret" .Release.Namespace .commonCASecretName }} + {{- $crt := index .data "ca.crt" }} + {{- $key := index .data "ca.key" }} + {{- $ca = buildCustomCert $crt $key -}} + {{- else }} + {{- $ca = .commonCA -}} + {{- end }} + {{- end }} + {{- $_ := set . "ca" $ca -}} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-client-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-client-secret.yaml new file mode 100644 index 00000000..e1d6e876 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-client-secret.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm") .Values.hubble.relay.enabled }} +{{- $_ := include "hubble-generate-certs.helm.setup-ca" . -}} +{{- $cn := "*.hubble-relay.cilium.io" }} +{{- $dns := list $cn }} +{{- $cert := genSignedCert $cn nil $dns (.Values.hubble.tls.auto.certValidityDuration | int) .ca -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: hubble-relay-client-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .ca.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-server-secret.yaml new file mode 100644 index 00000000..902c2be4 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/relay-server-secret.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm") .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} +{{- $_ := include "hubble-generate-certs.helm.setup-ca" . -}} +{{- $cn := "*.hubble-relay.cilium.io" }} +{{- $ip := .Values.hubble.relay.tls.server.extraIpAddresses }} +{{- $dns := prepend .Values.hubble.relay.tls.server.extraDnsNames $cn }} +{{- $cert := genSignedCert $cn $ip $dns (.Values.hubble.tls.auto.certValidityDuration | int) .ca -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: hubble-relay-server-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .ca.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/server-secret.yaml new file mode 100644 index 00000000..a05c3266 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/server-secret.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.agent .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm") }} +{{- $_ := include "hubble-generate-certs.helm.setup-ca" . -}} +{{- $cn := list "*" (.Values.cluster.name | replace "." "-") "hubble-grpc.cilium.io" | join "." }} +{{- $ip := .Values.hubble.tls.server.extraIpAddresses }} +{{- $dns := prepend .Values.hubble.tls.server.extraDnsNames $cn }} +{{- $cert := genSignedCert $cn $ip $dns (.Values.hubble.tls.auto.certValidityDuration | int) .ca -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .ca.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/ui-client-certs.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/ui-client-certs.yaml new file mode 100644 index 00000000..7b385b26 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-helm/ui-client-certs.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm") .Values.hubble.ui.enabled .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} +{{- $_ := include "hubble-generate-certs.helm.setup-ca" . -}} +{{- $cn := "*.hubble-ui.cilium.io" }} +{{- $dns := list $cn }} +{{- $cert := genSignedCert $cn nil $dns (.Values.hubble.tls.auto.certValidityDuration | int) .ca -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: hubble-ui-client-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .ca.Cert | b64enc }} + tls.crt: {{ $cert.Cert | b64enc }} + tls.key: {{ $cert.Key | b64enc }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml new file mode 100644 index 00000000..b1512ed2 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-client-secret.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled (not .Values.hubble.tls.auto.enabled) .Values.hubble.relay.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: hubble-relay-client-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.hubble.relay.tls.client.cert | required "missing hubble.relay.tls.client.cert" }} + tls.key: {{ .Values.hubble.relay.tls.client.key | required "missing hubble.relay.tls.client.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml new file mode 100644 index 00000000..07059143 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled (not .Values.hubble.tls.auto.enabled) .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: hubble-relay-server-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.hubble.relay.tls.server.cert | required "missing hubble.relay.tls.server.cert" }} + tls.key: {{ .Values.hubble.relay.tls.server.key | required "missing hubble.relay.tls.server.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/server-secret.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/server-secret.yaml new file mode 100644 index 00000000..f659a1fb --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/server-secret.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.agent .Values.hubble.enabled .Values.hubble.tls.enabled (not .Values.hubble.tls.auto.enabled) }} +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.hubble.tls.server.cert | required "missing hubble.tls.server.cert" }} + tls.key: {{ .Values.hubble.tls.server.key | required "missing hubble.tls.server.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml new file mode 100644 index 00000000..ca7683a4 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled (not .Values.hubble.tls.auto.enabled) .Values.hubble.ui.enabled .Values.hubble.relay.enabled .Values.hubble.relay.tls.server.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: hubble-ui-client-certs + namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.ca.cert }} + tls.crt: {{ .Values.hubble.ui.tls.client.cert | required "missing hubble.ui.tls.client.cert" }} + tls.key: {{ .Values.hubble.ui.tls.client.key | required "missing hubble.ui.tls.client.key" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrole.yaml new file mode 100644 index 00000000..90e2165d --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrole.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.agent.serviceAccount.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +# Required cluster role to allow spire-agent to query k8s API server +- apiGroups: [ "" ] + resources: [ "pods","nodes","nodes/proxy" ] + verbs: [ "get" ] +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrolebinding.yaml new file mode 100644 index 00000000..50fe9588 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/agent/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.agent.serviceAccount.create -}} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + kind: ClusterRole + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/agent/configmap.yaml b/magnum_cluster_api/charts/cilium/templates/spire/agent/configmap.yaml new file mode 100644 index 00000000..7d365ba6 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/agent/configmap.yaml @@ -0,0 +1,53 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-agent + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + agent.conf: | + agent { + data_dir = "/run/spire" + log_level = "ERROR" + server_address = "spire-server" + server_port = "8081" + socket_path = {{ .Values.authentication.mutual.spire.agentSocketPath | quote }} + admin_socket_path = {{ .Values.authentication.mutual.spire.adminSocketPath | quote }} + trust_bundle_path = "/run/spire/bundle/bundle.crt" + trust_domain = {{ .Values.authentication.mutual.spire.trustDomain | quote }} + authorized_delegates = [ + "spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-agent", + ] + } + + plugins { + NodeAttestor "k8s_psat" { + plugin_data { + cluster = {{ .Values.cluster.name | quote }} + } + } + + KeyManager "memory" { + plugin_data { + } + } + + WorkloadAttestor "k8s" { + plugin_data { + skip_kubelet_verification = {{ .Values.authentication.mutual.spire.install.agent.skipKubeletVerification }} + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "4251" + live_path = "/live" + ready_path = "/ready" + } +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/agent/daemonset.yaml b/magnum_cluster_api/charts/cilium/templates/spire/agent/daemonset.yaml new file mode 100644 index 00000000..6c0bffe7 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/agent/daemonset.yaml @@ -0,0 +1,126 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: spire-agent + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- if or .Values.authentication.mutual.spire.install.server.annotations .Values.authentication.mutual.spire.annotations }} + annotations: + {{- with .Values.authentication.mutual.spire.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + app: spire-agent + {{- with .Values.authentication.mutual.spire.install.agent.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: spire-agent + template: + metadata: + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + labels: + app: spire-agent + {{- with .Values.authentication.mutual.spire.install.agent.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + hostPID: true + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.install.agent.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: init + image: {{ include "cilium.image" .Values.authentication.mutual.spire.install.initImage | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.initImage.pullPolicy }} + command: + - /bin/sh + - -c + - | + {{- tpl (.Files.Get "files/spire/wait-for-spire.bash") . | nindent 14 }} + containers: + - name: spire-agent + {{- if eq (typeOf .Values.authentication.mutual.spire.install.agent.image) "string" }} + image: {{ .Values.authentication.mutual.spire.install.agent.image }} + {{- else }} + image: {{ include "cilium.image" .Values.authentication.mutual.spire.install.agent.image | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.agent.image.pullPolicy }} + {{- end }} + args: ["-config", "/run/spire/config/agent.conf"] + {{- with .Values.authentication.mutual.spire.install.agent.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + - name: spire-bundle + mountPath: /run/spire/bundle + - name: spire-agent-socket + mountPath: /run/spire/sockets + readOnly: false + - name: spire-agent + mountPath: /var/run/secrets/tokens + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: status.hostIP + livenessProbe: + httpGet: + path: /live + port: 4251 + failureThreshold: 2 + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /ready + port: 4251 + initialDelaySeconds: 5 + periodSeconds: 5 + {{- with .Values.authentication.mutual.spire.install.agent.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.agent.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- with .Values.authentication.mutual.spire.install.agent.tolerations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + - key: {{ .Values.agentNotReadyTaintKey | default "node.cilium.io/agent-not-ready" }} + effect: NoSchedule + volumes: + - name: spire-config + configMap: + name: spire-agent + - name: spire-bundle + configMap: + name: spire-bundle + - name: spire-agent-socket + hostPath: + path: /run/spire/sockets + type: DirectoryOrCreate + - name: spire-agent + projected: + sources: + - serviceAccountToken: + path: spire-agent + expirationSeconds: 600 + audience: spire-server +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/agent/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/spire/agent/serviceaccount.yaml new file mode 100644 index 00000000..95f7e431 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/agent/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.agent.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/bundle-configmap.yaml b/magnum_cluster_api/charts/cilium/templates/spire/bundle-configmap.yaml new file mode 100644 index 00000000..389b8d58 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/bundle-configmap.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-bundle + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/namespace.yaml b/magnum_cluster_api/charts/cilium/templates/spire/namespace.yaml new file mode 100644 index 00000000..ccd38680 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/namespace.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled (not .Values.authentication.mutual.spire.install.existingNamespace) -}} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/clusterrole.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/clusterrole.yaml new file mode 100644 index 00000000..31bc2de1 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/clusterrole.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +# ClusterRole to allow spire-server node attestor to query Token Review API +- apiGroups: [ "authentication.k8s.io" ] + resources: [ "tokenreviews" ] + verbs: [ "create" ] +# Required cluster role to allow spire-server to query k8s API server +# for pods for psat attestation +- apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get" ] +# Required cluster role to allow spire-server to query k8s API server +# for nodes for psat attestation +- apiGroups: [ "" ] + resources: [ "nodes","nodes/proxy" ] + verbs: [ "get" ] +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/clusterrolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/clusterrolebinding.yaml new file mode 100644 index 00000000..4a04d80c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + kind: ClusterRole + name: spire-server + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/configmap.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/configmap.yaml new file mode 100644 index 00000000..d7347595 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/configmap.yaml @@ -0,0 +1,72 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-server + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + server.conf: | + server { + bind_address = "0.0.0.0" + bind_port = "8081" + socket_path = "/tmp/spire-server/private/api.sock" + trust_domain = {{ .Values.authentication.mutual.spire.trustDomain | quote }} + data_dir = "/run/spire/data" + log_level = "INFO" + ca_key_type = {{ .Values.authentication.mutual.spire.install.server.ca.keyType | quote }} + + ca_subject = { + country = [{{ .Values.authentication.mutual.spire.install.server.ca.subject.country | quote }}], + organization = [{{ .Values.authentication.mutual.spire.install.server.ca.subject.organization | quote }}], + common_name = {{ .Values.authentication.mutual.spire.install.server.ca.subject.commonName | quote }}, + } + + admin_ids = [ + "spiffe://{{ .Values.authentication.mutual.spire.trustDomain }}/cilium-operator", + ] + } + + plugins { + DataStore "sql" { + plugin_data { + database_type = "sqlite3" + connection_string = "/run/spire/data/datastore.sqlite3" + } + } + + NodeAttestor "k8s_psat" { + plugin_data { + clusters = { + {{ .Values.cluster.name | quote }} = { + use_token_review_api_validation = true + service_account_allow_list = ["{{ .Values.authentication.mutual.spire.install.namespace}}:{{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }}"] + } + } + } + } + + KeyManager "disk" { + plugin_data { + keys_path = "/run/spire/data/keys.json" + } + } + + Notifier "k8sbundle" { + plugin_data { + namespace = {{ .Values.authentication.mutual.spire.install.namespace | quote }} + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "8080" + live_path = "/live" + ready_path = "/ready" + } +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/role.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/role.yaml new file mode 100644 index 00000000..2d38fe14 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/role.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +# Role (namespace scoped) to be able to push certificate bundles to a configmap +- apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "patch", "get", "list" ] +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/rolebinding.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/rolebinding.yaml new file mode 100644 index 00000000..497bd60c --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/rolebinding.yaml @@ -0,0 +1,37 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spire-server +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }}-pod + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }}-pod +subjects: +- kind: ServiceAccount + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/service.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/service.yaml new file mode 100644 index 00000000..32288a44 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: spire-server + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- if or .Values.authentication.mutual.spire.install.server.service.annotations .Values.authentication.mutual.spire.annotations }} + annotations: + {{- with .Values.authentication.mutual.spire.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.service.labels }} + labels: + {{- toYaml . | nindent 8 }} + {{- end }} +spec: + type: {{ .Values.authentication.mutual.spire.install.server.service.type }} + ports: + - name: grpc + port: 8081 + targetPort: grpc + protocol: TCP + selector: + app: spire-server +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/serviceaccount.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/serviceaccount.yaml new file mode 100644 index 00000000..cd365c89 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled .Values.authentication.mutual.spire.install.server.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/spire/server/statefulset.yaml b/magnum_cluster_api/charts/cilium/templates/spire/server/statefulset.yaml new file mode 100644 index 00000000..49ab9690 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/spire/server/statefulset.yaml @@ -0,0 +1,129 @@ +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: spire-server + namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- if or .Values.authentication.mutual.spire.install.server.annotations .Values.authentication.mutual.spire.annotations }} + annotations: + {{- with .Values.authentication.mutual.spire.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + app: spire-server + {{- with .Values.authentication.mutual.spire.install.server.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + app: spire-server + serviceName: spire-server + template: + metadata: + labels: + app: spire-server + {{- with .Values.authentication.mutual.spire.install.server.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + shareProcessNamespace: true + {{- with .Values.authentication.mutual.spire.install.server.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if gt (len .Values.authentication.mutual.spire.install.server.initContainers) 0 }} + initContainers: + {{- toYaml .Values.authentication.mutual.spire.install.server.initContainers | nindent 8 }} + {{- end }} + containers: + - name: cilium-init + image: {{ include "cilium.image" .Values.authentication.mutual.spire.install.initImage | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.initImage.pullPolicy }} + command: + - /bin/sh + - -c + - | + {{- tpl (.Files.Get "files/spire/init.bash") . | nindent 12 }} + - name: spire-server + {{- if eq (typeOf .Values.authentication.mutual.spire.install.server.image) "string" }} + image: {{ .Values.authentication.mutual.spire.install.server.image }} + {{- else }} + image: {{ include "cilium.image" .Values.authentication.mutual.spire.install.server.image | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.server.image.pullPolicy }} + {{- end }} + args: + - -config + - /run/spire/config/server.conf + ports: + - name: grpc + containerPort: 8081 + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + {{- if .Values.authentication.mutual.spire.install.server.dataStorage.enabled }} + - name: spire-data + mountPath: /run/spire/data + readOnly: false + {{- end }} + - name: spire-server-socket + mountPath: /tmp/spire-server/private + readOnly: false + livenessProbe: + httpGet: + path: /live + port: 8080 + failureThreshold: 2 + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + {{- with .Values.authentication.mutual.spire.install.server.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.tolerations }} + tolerations: + {{- toYaml . | trim | nindent 8 }} + {{- end }} + volumes: + - name: spire-config + configMap: + name: spire-server + - name: spire-server-socket + hostPath: + path: /var/run/spire-server/sockets + type: DirectoryOrCreate + {{- if .Values.authentication.mutual.spire.install.server.dataStorage.enabled }} + volumeClaimTemplates: + - metadata: + name: spire-data + spec: + accessModes: + - {{ .Values.authentication.mutual.spire.install.server.dataStorage.accessMode | default "ReadWriteOnce" }} + resources: + requests: + storage: {{ .Values.authentication.mutual.spire.install.server.dataStorage.size }} + storageClassName: {{ .Values.authentication.mutual.spire.install.server.dataStorage.storageClass }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/templates/validate.yaml b/magnum_cluster_api/charts/cilium/templates/validate.yaml new file mode 100644 index 00000000..3c89e4e3 --- /dev/null +++ b/magnum_cluster_api/charts/cilium/templates/validate.yaml @@ -0,0 +1,103 @@ +{{/* validate hubble config */}} +{{- if and .Values.hubble.ui.enabled (not .Values.hubble.ui.standalone.enabled) }} + {{- if not .Values.hubble.relay.enabled }} + {{ fail "Hubble UI requires .Values.hubble.relay.enabled=true" }} + {{- end }} +{{- end }} +{{- if and .Values.hubble.ui.enabled .Values.hubble.ui.standalone.enabled .Values.hubble.relay.tls.server.enabled }} + {{- if not .Values.hubble.ui.standalone.tls.certsVolume }} + {{ fail "Hubble UI in standalone with Hubble Relay server TLS enabled requires providing .Values.hubble.ui.standalone.tls.certsVolume for mounting client certificates in the backend pod" }} + {{- end }} +{{- end }} +{{- if .Values.hubble.relay.enabled }} + {{- if not .Values.hubble.enabled }} + {{ fail "Hubble Relay requires .Values.hubble.enabled=true" }} + {{- end }} +{{- end }} + +{{/* validate service monitoring CRDs */}} +{{- if or (and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.enabled) (and .Values.operator.prometheus.enabled .Values.operator.prometheus.serviceMonitor.enabled) (and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled .Values.envoy.prometheus.serviceMonitor.enabled) (and .Values.proxy.prometheus.enabled .Values.hubble.relay.prometheus.enabled .Values.hubble.relay.prometheus.serviceMonitor.enabled) }} + {{- if not (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }} + {{- if not .Values.prometheus.serviceMonitor.trustCRDsExist }} + {{ fail "Service Monitor requires monitoring.coreos.com/v1 CRDs. Please refer to https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml or set .Values.prometheus.serviceMonitor.trustCRDsExist=true" }} + {{- end }} + {{- end }} +{{- end }} + +{{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "certmanager") }} + {{- if not .Values.hubble.tls.auto.certManagerIssuerRef }} + {{ fail "Hubble TLS certgen method=certmanager requires that user specifies .Values.hubble.tls.auto.certManagerIssuerRef" }} + {{- end }} +{{- end }} + +{{- if and .Values.hubble.redact.http.headers.allow .Values.hubble.redact.http.headers.deny }} + {{ fail "Only one of .Values.hubble.redact.http.headers.allow, .Values.hubble.redact.http.headers.deny can be specified"}} +{{- end }} + +{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} + {{- if not .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef }} + {{ fail "ClusterMesh TLS certgen method=certmanager requires that user specifies .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef" }} + {{- end }} +{{- end }} + +{{/* validate hubble-ui specific config */}} +{{- if and .Values.hubble.ui.enabled + (ne .Values.hubble.ui.backend.image.tag "latest") + (ne .Values.hubble.ui.frontend.image.tag "latest") }} + {{- if regexReplaceAll "@.*$" .Values.hubble.ui.backend.image.tag "" | trimPrefix "v" | semverCompare "<0.9.0" }} + {{ fail "Hubble UI requires hubble.ui.backend.image.tag to be '>=v0.9.0'" }} + {{- end }} + {{- if regexReplaceAll "@.*$" .Values.hubble.ui.frontend.image.tag "" | trimPrefix "v" | semverCompare "<0.9.0" }} + {{ fail "Hubble UI requires hubble.ui.frontend.image.tag to be '>=v0.9.0'" }} + {{- end }} +{{- end }} + +{{- if or .Values.ingressController.enabled .Values.gatewayAPI.enabled (eq .Values.loadBalancer.l7.backend "envoy") }} + {{- if hasKey .Values "l7Proxy" }} + {{- if not .Values.l7Proxy }} + {{ fail "Ingress or Gateway API controller or Envoy L7 Load Balancer requires .Values.l7Proxy to be set to 'true'" }} + {{- end }} + {{- end }} +{{- end }} + +{{- if or .Values.envoyConfig.enabled .Values.ingressController.enabled .Values.gatewayAPI.enabled }} + {{- if or (eq (toString .Values.kubeProxyReplacement) "disabled") (and (not (hasKey .Values "kubeProxyReplacement")) (not (semverCompare ">=1.14" (default "1.14" .Values.upgradeCompatibility)))) }} + {{ fail "Ingress/Gateway API controller and EnvoyConfig require .Values.kubeProxyReplacement to be explicitly set to 'false' or 'true'" }} + {{- end }} +{{- end }} + +{{- if .Values.authentication.mutual.spire.enabled }} + {{- if not .Values.authentication.enabled }} + {{ fail "SPIRE integration requires .Values.authentication.enabled=true and .Values.authentication.mutual.spire.enabled=true" }} + {{- end }} +{{- end }} + +{{/* validate Cilium operator */}} +{{- if eq .Values.enableCiliumEndpointSlice true }} + {{- if eq .Values.disableEndpointCRD true }} + {{ fail "if Cilium Endpoint Slice is enabled (.Values.enableCiliumEndpointSlice=true), it requires .Values.disableEndpointCRD=false" }} + {{- end }} +{{- end }} + +{{/* validate clustermesh-apiserver */}} +{{- if .Values.clustermesh.useAPIServer }} + {{- if ne .Values.identityAllocationMode "crd" }} + {{ fail (printf "The clustermesh-apiserver cannot be enabled in combination with .Values.identityAllocationMode=%s. To establish a Cluster Mesh, directly configure the parameters to access the remote kvstore through .Values.clustermesh.config" .Values.identityAllocationMode ) }} + {{- end }} + {{- if .Values.disableEndpointCRD }} + {{ fail "The clustermesh-apiserver cannot be enabled in combination with .Values.disableEndpointCRD=true" }} + {{- end }} +{{- end }} +{{- if .Values.externalWorkloads.enabled }} + {{- if ne .Values.identityAllocationMode "crd" }} + {{ fail (printf "External workloads support cannot be enabled in combination with .Values.identityAllocationMode=%s" .Values.identityAllocationMode ) }} + {{- end }} + {{- if .Values.disableEndpointCRD }} + {{ fail "External workloads support cannot be enabled in combination with .Values.disableEndpointCRD=true" }} + {{- end }} +{{- end }} + +{{/*validate ClusterMesh */}} +{{- if and (ne (int .Values.clustermesh.maxConnectedClusters) 255) (ne (int .Values.clustermesh.maxConnectedClusters) 511) }} + {{- fail "max-connected-clusters must be set to 255 or 511" }} +{{- end }} diff --git a/magnum_cluster_api/charts/cilium/values.yaml b/magnum_cluster_api/charts/cilium/values.yaml new file mode 100644 index 00000000..2e1d10bc --- /dev/null +++ b/magnum_cluster_api/charts/cilium/values.yaml @@ -0,0 +1,3516 @@ +# File generated by install/kubernetes/Makefile; DO NOT EDIT. +# This file is based on install/kubernetes/cilium/*values.yaml.tmpl. + + +# upgradeCompatibility helps users upgrading to ensure that the configMap for +# Cilium will not change critical values to ensure continued operation +# This flag is not required for new installations. +# For example: 1.7, 1.8, 1.9 +# upgradeCompatibility: '1.8' + +debug: + # -- Enable debug logging + enabled: false + # -- Configure verbosity levels for debug logging + # This option is used to enable debug messages for operations related to such + # sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is + # for enabling debug messages emitted per request, message and connection. + # Multiple values can be set via a space-separated string (e.g. "datapath envoy"). + # + # Applicable values: + # - flow + # - kvstore + # - envoy + # - datapath + # - policy + verbose: ~ + +rbac: + # -- Enable creation of Resource-Based Access Control configuration. + create: true + +# -- Configure image pull secrets for pulling container images +imagePullSecrets: +# - name: "image-pull-secret" + +# -- (string) Kubernetes config path +# @default -- `"~/.kube/config"` +kubeConfigPath: "" +# -- (string) Kubernetes service host +k8sServiceHost: "" +# -- (string) Kubernetes service port +k8sServicePort: "" + +# -- Configure the client side rate limit for the agent and operator +# +# If the amount of requests to the Kubernetes API server exceeds the configured +# rate limit, the agent and operator will start to throttle requests by delaying +# them until there is budget or the request times out. +k8sClientRateLimit: + # -- (int) The sustained request rate in requests per second. + # @default -- 5 for k8s up to 1.26. 10 for k8s version 1.27+ + qps: + # -- (int) The burst request rate in requests per second. + # The rate limiter will allow short bursts with a higher rate. + # @default -- 10 for k8s up to 1.26. 20 for k8s version 1.27+ + burst: + +cluster: + # -- Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. + name: default + # -- (int) Unique ID of the cluster. Must be unique across all connected + # clusters and in the range of 1 to 255. Only required for Cluster Mesh, + # may be 0 if Cluster Mesh is not used. + id: 0 + +# -- Define serviceAccount names for components. +# @default -- Component's fully qualified name. +serviceAccounts: + cilium: + create: true + name: cilium + automount: true + annotations: {} + nodeinit: + create: true + # -- Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. + # Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by + # this issue. Name and automount can be configured, if enabled is set to true. + # Otherwise, they are ignored. Enabled can be removed once the issue is fixed. + # Cilium-nodeinit DS must also be fixed. + enabled: false + name: cilium-nodeinit + automount: true + annotations: {} + envoy: + create: true + name: cilium-envoy + automount: true + annotations: {} + etcd: + create: true + name: cilium-etcd-operator + automount: true + annotations: {} + operator: + create: true + name: cilium-operator + automount: true + annotations: {} + preflight: + create: true + name: cilium-pre-flight + automount: true + annotations: {} + relay: + create: true + name: hubble-relay + automount: false + annotations: {} + ui: + create: true + name: hubble-ui + automount: true + annotations: {} + clustermeshApiserver: + create: true + name: clustermesh-apiserver + automount: true + annotations: {} + # -- Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob + clustermeshcertgen: + create: true + name: clustermesh-apiserver-generate-certs + automount: true + annotations: {} + # -- Hubblecertgen is used if hubble.tls.auto.method=cronJob + hubblecertgen: + create: true + name: hubble-generate-certs + automount: true + annotations: {} + +# -- Configure termination grace period for cilium-agent DaemonSet. +terminationGracePeriodSeconds: 1 + +# -- Install the cilium agent resources. +agent: true + +# -- Agent container name. +name: cilium + +# -- Roll out cilium agent pods automatically when configmap is updated. +rollOutCiliumPods: false + +# -- Agent container image. +image: + override: ~ + repository: "quay.io/cilium/cilium" + tag: "v1.15.3" + pullPolicy: "IfNotPresent" + # cilium-digest + digest: "sha256:da74ab61d1bc665c1c088dff41d5be388d252ca5800f30c7d88844e6b5e440b0" + useDigest: true + +# -- Affinity for cilium-agent. +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + +# -- Node selector for cilium-agent. +nodeSelector: + kubernetes.io/os: linux + +# -- Node tolerations for agent scheduling to nodes with taints +# ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ +tolerations: +- operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + +# -- The priority class to use for cilium-agent. +priorityClassName: "" + +# -- DNS policy for Cilium agent pods. +# Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy +dnsPolicy: "" + +# -- Additional containers added to the cilium DaemonSet. +extraContainers: [] + +# -- Additional agent container arguments. +extraArgs: [] + +# -- Additional agent container environment variables. +extraEnv: [] + +# -- Additional agent hostPath mounts. +extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + +# -- Additional agent volumes. +extraVolumes: [] + +# -- Additional agent volumeMounts. +extraVolumeMounts: [] + +# -- extraConfig allows you to specify additional configuration parameters to be +# included in the cilium-config configmap. +extraConfig: {} +# my-config-a: "1234" +# my-config-b: |- +# test 1 +# test 2 +# test 3 + +# -- Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) +annotations: {} + +# -- Security Context for cilium-agent pods. +podSecurityContext: {} + +# -- Annotations to be added to agent pods +podAnnotations: {} + +# -- Labels to be added to agent pods +podLabels: {} + +# -- Agent resource limits & requests +# ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + +# -- resources & limits for the agent init containers +initResources: {} + +securityContext: + # -- User to run the pod with + # runAsUser: 0 + # -- Run the pod with elevated privileges + privileged: false + # -- SELinux options for the `cilium-agent` and init containers + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + # -- Capabilities for the `cilium-agent` container + ciliumAgent: + # Use to set socket permission + - CHOWN + # Used to terminate envoy child process + - KILL + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used since cilium creates raw sockets, etc... + - NET_RAW + # Used since cilium monitor uses mmap + - IPC_LOCK + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + # Allow discretionary access control (e.g. required for package installation) + - DAC_OVERRIDE + # Allow to set Access Control Lists (ACLs) on arbitrary files (e.g. required for package installation) + - FOWNER + # Allow to execute program that changes GID (e.g. required for package installation) + - SETGID + # Allow to execute program that changes UID (e.g. required for package installation) + - SETUID + # -- Capabilities for the `mount-cgroup` init container + mountCgroup: + # Only used for 'mount' cgroup + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # -- capabilities for the `apply-sysctl-overwrites` init container + applySysctlOverwrites: + # Required in order to access host's /etc/sysctl.d dir + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # -- Capabilities for the `clean-cilium-state` init container + cleanCiliumState: + # Most of the capabilities here are the same ones used in the + # cilium-agent's container because this container can be used to + # uninstall all Cilium resources, and therefore it is likely that + # will need the same capabilities. + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + +# -- Cilium agent update strategy +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + +# Configuration Values for cilium-agent + +aksbyocni: + # -- Enable AKS BYOCNI integration. + # Note that this is incompatible with AKS clusters not created in BYOCNI mode: + # use Azure integration (`azure.enabled`) instead. + enabled: false + +# -- Enable installation of PodCIDR routes between worker +# nodes if worker nodes share a common L2 network segment. +autoDirectNodeRoutes: false + +# -- Annotate k8s node upon initialization with Cilium's metadata. +annotateK8sNode: false + +azure: + # -- Enable Azure integration. + # Note that this is incompatible with AKS clusters created in BYOCNI mode: use + # AKS BYOCNI integration (`aksbyocni.enabled`) instead. + enabled: false + # usePrimaryAddress: false + # resourceGroup: group1 + # subscriptionID: 00000000-0000-0000-0000-000000000000 + # tenantID: 00000000-0000-0000-0000-000000000000 + # clientID: 00000000-0000-0000-0000-000000000000 + # clientSecret: 00000000-0000-0000-0000-000000000000 + # userAssignedIdentityID: 00000000-0000-0000-0000-000000000000 + +alibabacloud: + # -- Enable AlibabaCloud ENI integration + enabled: false + +# -- Enable bandwidth manager to optimize TCP and UDP workloads and allow +# for rate-limiting traffic from individual Pods with EDT (Earliest Departure +# Time) through the "kubernetes.io/egress-bandwidth" Pod annotation. +bandwidthManager: + # -- Enable bandwidth manager infrastructure (also prerequirement for BBR) + enabled: false + # -- Activate BBR TCP congestion control for Pods + bbr: false + +# -- Configure standalone NAT46/NAT64 gateway +nat46x64Gateway: + # -- Enable RFC8215-prefixed translation + enabled: false + +# -- EnableHighScaleIPcache enables the special ipcache mode for high scale +# clusters. The ipcache content will be reduced to the strict minimum and +# traffic will be encapsulated to carry security identities. +highScaleIPcache: + # -- Enable the high scale mode for the ipcache. + enabled: false + +# -- Configure L2 announcements +l2announcements: + # -- Enable L2 announcements + enabled: false + # -- If a lease is not renewed for X duration, the current leader is considered dead, a new leader is picked + # leaseDuration: 15s + # -- The interval at which the leader will renew the lease + # leaseRenewDeadline: 5s + # -- The timeout between retries if renewal fails + # leaseRetryPeriod: 2s + +# -- Configure L2 pod announcements +l2podAnnouncements: + # -- Enable L2 pod announcements + enabled: false + # -- Interface used for sending Gratuitous ARP pod announcements + interface: "eth0" + +# -- Configure BGP +bgp: + # -- Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside + # cilium-agent and cilium-operator + enabled: false + announce: + # -- Enable allocation and announcement of service LoadBalancer IPs + loadbalancerIP: false + # -- Enable announcement of node pod CIDR + podCIDR: false + +# -- This feature set enables virtual BGP routers to be created via +# CiliumBGPPeeringPolicy CRDs. +bgpControlPlane: + # -- Enables the BGP control plane. + enabled: false + # -- SecretsNamespace is the namespace which BGP support will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for BGP secrets. + create: false + # -- The name of the secret namespace to which Cilium agents are given read access + name: kube-system + +pmtuDiscovery: + # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to + # the client. + enabled: false + +bpf: + autoMount: + # -- Enable automatic mount of BPF filesystem + # When `autoMount` is enabled, the BPF filesystem is mounted at + # `bpf.root` path on the underlying host and inside the cilium agent pod. + # If users disable `autoMount`, it's expected that users have mounted + # bpffs filesystem at the specified `bpf.root` volume, and then the + # volume will be mounted inside the cilium agent pod at the same path. + enabled: true + # -- Configure the mount point for the BPF filesystem + root: /sys/fs/bpf + + # -- Enables pre-allocation of eBPF map values. This increases + # memory usage but can reduce latency. + preallocateMaps: false + + # -- (int) Configure the maximum number of entries in auth map. + # @default -- `524288` + authMapMax: ~ + + # -- (int) Configure the maximum number of entries in the TCP connection tracking + # table. + # @default -- `524288` + ctTcpMax: ~ + + # -- (int) Configure the maximum number of entries for the non-TCP connection + # tracking table. + # @default -- `262144` + ctAnyMax: ~ + + # -- Configure the maximum number of service entries in the + # load balancer maps. + lbMapMax: 65536 + + # -- (int) Configure the maximum number of entries for the NAT table. + # @default -- `524288` + natMax: ~ + + # -- (int) Configure the maximum number of entries for the neighbor table. + # @default -- `524288` + neighMax: ~ + + # -- Configure the maximum number of entries in endpoint policy map (per endpoint). + policyMapMax: 16384 + + # -- (float64) Configure auto-sizing for all BPF maps based on available memory. + # ref: https://docs.cilium.io/en/stable/network/ebpf/maps/ + # @default -- `0.0025` + mapDynamicSizeRatio: ~ + + # -- Configure the level of aggregation for monitor notifications. + # Valid options are none, low, medium, maximum. + monitorAggregation: medium + + # -- Configure the typical time between monitor notifications for + # active connections. + monitorInterval: "5s" + + # -- Configure which TCP flags trigger notifications when seen for the + # first time in a connection. + monitorFlags: "all" + + # -- Allow cluster external access to ClusterIP services. + lbExternalClusterIP: false + + # -- (bool) Enable native IP masquerade support in eBPF + # @default -- `false` + masquerade: ~ + + # -- (bool) Configure whether direct routing mode should route traffic via + # host stack (true) or directly and more efficiently out of BPF (false) if + # the kernel supports it. The latter has the implication that it will also + # bypass netfilter in the host namespace. + # @default -- `false` + hostLegacyRouting: ~ + + # -- (bool) Configure the eBPF-based TPROXY to reduce reliance on iptables rules + # for implementing Layer 7 policy. + # @default -- `false` + tproxy: ~ + + # -- (list) Configure explicitly allowed VLAN id's for bpf logic bypass. + # [0] will allow all VLAN id's without any filtering. + # @default -- `[]` + vlanBypass: ~ + +# -- Enable BPF clock source probing for more efficient tick retrieval. +bpfClockProbe: false + +# -- Clean all eBPF datapath state from the initContainer of the cilium-agent +# DaemonSet. +# +# WARNING: Use with care! +cleanBpfState: false + +# -- Clean all local Cilium state from the initContainer of the cilium-agent +# DaemonSet. Implies cleanBpfState: true. +# +# WARNING: Use with care! +cleanState: false + +# -- Wait for KUBE-PROXY-CANARY iptables rule to appear in "wait-for-kube-proxy" +# init container before launching cilium-agent. +# More context can be found in the commit message of below PR +# https://github.com/cilium/cilium/pull/20123 +waitForKubeProxy: false + +cni: + # -- Install the CNI configuration and binary files into the filesystem. + install: true + + # -- Remove the CNI configuration and binary files on agent shutdown. Enable this + # if you're removing Cilium from the cluster. Disable this to prevent the CNI + # configuration file from being removed during agent upgrade, which can cause + # nodes to go unmanageable. + uninstall: false + + # -- Configure chaining on top of other CNI plugins. Possible values: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap + chainingMode: ~ + + # -- A CNI network name in to which the Cilium plugin should be added as a chained plugin. + # This will cause the agent to watch for a CNI network with this network name. When it is + # found, this will be used as the basis for Cilium's CNI configuration file. If this is + # set, it assumes a chaining mode of generic-veth. As a special case, a chaining mode + # of aws-cni implies a chainingTarget of aws-cni. + chainingTarget: ~ + + # -- Make Cilium take ownership over the `/etc/cni/net.d` directory on the + # node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. + # This ensures no Pods can be scheduled using other CNI plugins during Cilium + # agent downtime. + exclusive: true + + # -- Configure the log file for CNI logging with retention policy of 7 days. + # Disable CNI file logging by setting this field to empty explicitly. + logFile: /var/run/cilium/cilium-cni.log + + # -- Skip writing of the CNI configuration. This can be used if + # writing of the CNI configuration is performed by external automation. + customConf: false + + # -- Configure the path to the CNI configuration directory on the host. + confPath: /etc/cni/net.d + + # -- Configure the path to the CNI binary directory on the host. + binPath: /opt/cni/bin + + # -- Specify the path to a CNI config to read from on agent start. + # This can be useful if you want to manage your CNI + # configuration outside of a Kubernetes environment. This parameter is + # mutually exclusive with the 'cni.configMap' parameter. The agent will + # write this to 05-cilium.conflist on startup. + # readCniConf: /host/etc/cni/net.d/05-sample.conflist.input + + # -- When defined, configMap will mount the provided value as ConfigMap and + # interpret the cniConf variable as CNI configuration file and write it + # when the agent starts up + # configMap: cni-configuration + + # -- Configure the key in the CNI ConfigMap to read the contents of + # the CNI configuration from. + configMapKey: cni-config + + # -- Configure the path to where to mount the ConfigMap inside the agent pod. + confFileMountPath: /tmp/cni-configuration + + # -- Configure the path to where the CNI configuration directory is mounted + # inside the agent pod. + hostConfDirMountPath: /host/etc/cni/net.d + + # -- Specifies the resources for the cni initContainer + resources: + requests: + cpu: 100m + memory: 10Mi + +# -- (string) Configure how frequently garbage collection should occur for the datapath +# connection tracking table. +# @default -- `"0s"` +conntrackGCInterval: "" + +# -- (string) Configure the maximum frequency for the garbage collection of the +# connection tracking table. Only affects the automatic computation for the frequency +# and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently +# clean up unused identities created from ToFQDN policies. +conntrackGCMaxInterval: "" + +# -- Configure container runtime specific integration. +# Deprecated in favor of bpf.autoMount.enabled. To be removed in 1.15. +containerRuntime: + # -- Enables specific integrations for container runtimes. + # Supported values: + # - crio + # - none + integration: none + +# -- (string) Configure timeout in which Cilium will exit if CRDs are not available +# @default -- `"5m"` +crdWaitTimeout: "" + +# -- Tail call hooks for custom eBPF programs. +customCalls: + # -- Enable tail call hooks for custom eBPF programs. + enabled: false + +daemon: + # -- Configure where Cilium runtime state should be stored. + runPath: "/var/run/cilium" + + # -- Configure a custom list of possible configuration override sources + # The default is "config-map:cilium-config,cilium-node-config". For supported + # values, see the help text for the build-config subcommand. + # Note that this value should be a comma-separated string. + configSources: ~ + + # -- allowedConfigOverrides is a list of config-map keys that can be overridden. + # That is to say, if this value is set, config sources (excepting the first one) can + # only override keys in this list. + # + # This takes precedence over blockedConfigOverrides. + # + # By default, all keys may be overridden. To disable overrides, set this to "none" or + # change the configSources variable. + allowedConfigOverrides: ~ + + # -- blockedConfigOverrides is a list of config-map keys that may not be overridden. + # In other words, if any of these keys appear in a configuration source excepting the + # first one, they will be ignored + # + # This is ignored if allowedConfigOverrides is set. + # + # By default, all keys may be overridden. + blockedConfigOverrides: ~ + +# -- Specify which network interfaces can run the eBPF datapath. This means +# that a packet sent from a pod to a destination outside the cluster will be +# masqueraded (to an output device IPv4 address), if the output device runs the +# program. When not specified, probing will automatically detect devices that have +# a non-local route. This should be used only when autodetection is not suitable. +# devices: "" + +# -- Enables experimental support for the detection of new and removed datapath +# devices. When devices change the eBPF datapath is reloaded and services updated. +# If "devices" is set then only those devices, or devices matching a wildcard will +# be considered. +enableRuntimeDeviceDetection: false + +# -- Chains to ignore when installing feeder rules. +# disableIptablesFeederRules: "" + +# -- Limit iptables-based egress masquerading to interface selector. +# egressMasqueradeInterfaces: "" + +# -- Enable setting identity mark for local traffic. +# enableIdentityMark: true + +# -- Enable Kubernetes EndpointSlice feature in Cilium if the cluster supports it. +# enableK8sEndpointSlice: true + +# -- Enable CiliumEndpointSlice feature. +enableCiliumEndpointSlice: false + +envoyConfig: + # -- Enable CiliumEnvoyConfig CRD + # CiliumEnvoyConfig CRD can also be implicitly enabled by other options. + enabled: false + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for CiliumEnvoyConfig CRDs. + create: true + + # -- The name of the secret namespace to which Cilium agents are given read access. + name: cilium-secrets + +ingressController: + # -- Enable cilium ingress controller + # This will automatically set enable-envoy-config as well. + enabled: false + + # -- Set cilium ingress controller to be the default ingress controller + # This will let cilium ingress controller route entries without ingress class set + default: false + + # -- Default ingress load balancer mode + # Supported values: shared, dedicated + # For granular control, use the following annotations on the ingress resource + # ingress.cilium.io/loadbalancer-mode: shared|dedicated, + loadbalancerMode: dedicated + + # -- Enforce https for host having matching TLS host in Ingress. + # Incoming traffic to http listener will return 308 http error code with respective location in header. + enforceHttps: true + + # -- Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. + enableProxyProtocol: false + + # -- IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service + ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] + + # -- Default secret namespace for ingresses without .spec.tls[].secretName set. + defaultSecretNamespace: + + # -- Default secret name for ingresses without .spec.tls[].secretName set. + defaultSecretName: + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. + secretsNamespace: + # -- Create secrets namespace for Ingress. + create: true + + # -- Name of Ingress secret namespace. + name: cilium-secrets + + # -- Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. + # If disabled, TLS secrets must be maintained externally. + sync: true + + # -- Load-balancer service in shared mode. + # This is a single load-balancer service for all Ingress resources. + service: + # -- Service name + name: cilium-ingress + # -- Labels to be added for the shared LB service + labels: {} + # -- Annotations to be added for the shared LB service + annotations: {} + # -- Service type for the shared LB service + type: LoadBalancer + # -- Configure a specific nodePort for insecure HTTP traffic on the shared LB service + insecureNodePort: ~ + # -- Configure a specific nodePort for secure HTTPS traffic on the shared LB service + secureNodePort : ~ + # -- Configure a specific loadBalancerClass on the shared LB service (requires Kubernetes 1.24+) + loadBalancerClass: ~ + # -- Configure a specific loadBalancerIP on the shared LB service + loadBalancerIP : ~ + # -- Configure if node port allocation is required for LB service + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation + allocateLoadBalancerNodePorts: ~ + +gatewayAPI: + # -- Enable support for Gateway API in cilium + # This will automatically set enable-envoy-config as well. + enabled: false + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. + secretsNamespace: + # -- Create secrets namespace for Gateway API. + create: true + + # -- Name of Gateway API secret namespace. + name: cilium-secrets + + # -- Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. + # If disabled, TLS secrets must be maintained externally. + sync: true + +# -- Enables the fallback compatibility solution for when the xt_socket kernel +# module is missing and it is needed for the datapath L7 redirection to work +# properly. See documentation for details on when this can be disabled: +# https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. +enableXTSocketFallback: true + +encryption: + # -- Enable transparent network encryption. + enabled: false + + # -- Encryption method. Can be either ipsec or wireguard. + type: ipsec + + # -- Enable encryption for pure node to node traffic. + # This option is only effective when encryption.type is set to "wireguard". + nodeEncryption: false + + # -- Configure the WireGuard Pod2Pod strict mode. + strictMode: + # -- Enable WireGuard Pod2Pod strict mode. + enabled: false + + # -- CIDR for the WireGuard Pod2Pod strict mode. + cidr: "" + + # -- Allow dynamic lookup of remote node identities. + # This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. + allowRemoteNodeIdentities: false + + ipsec: + # -- Name of the key file inside the Kubernetes secret configured via secretName. + keyFile: "" + + # -- Path to mount the secret inside the Cilium pod. + mountPath: "" + + # -- Name of the Kubernetes secret containing the encryption keys. + secretName: "" + + # -- The interface to use for encrypted traffic. + interface: "" + + # -- Enable the key watcher. If disabled, a restart of the agent will be + # necessary on key rotations. + keyWatcher: true + + # -- Maximum duration of the IPsec key rotation. The previous key will be + # removed after that delay. + keyRotationDuration: "5m" + + wireguard: + # -- Enables the fallback to the user-space implementation. + userspaceFallback: false + # -- Controls Wireguard PersistentKeepalive option. Set 0s to disable. + persistentKeepalive: 0s + + # -- Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. + # Name of the key file inside the Kubernetes secret configured via secretName. + # This option is only effective when encryption.type is set to ipsec. + keyFile: keys + + # -- Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. + # Path to mount the secret inside the Cilium pod. + # This option is only effective when encryption.type is set to ipsec. + mountPath: /etc/ipsec + + # -- Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. + # Name of the Kubernetes secret containing the encryption keys. + # This option is only effective when encryption.type is set to ipsec. + secretName: cilium-ipsec-keys + + # -- Deprecated in favor of encryption.ipsec.interface. To be removed in 1.15. + # The interface to use for encrypted traffic. + # This option is only effective when encryption.type is set to ipsec. + interface: "" + +endpointHealthChecking: + # -- Enable connectivity health checking between virtual endpoints. + enabled: true + +# -- Enable endpoint status. +# Status can be: policy, health, controllers, log and / or state. For 2 or more options use a space. +endpointStatus: + enabled: false + status: "" + +endpointRoutes: + # -- Enable use of per endpoint routes instead of routing via + # the cilium_host interface. + enabled: false + +k8sNetworkPolicy: + # -- Enable support for K8s NetworkPolicy + enabled: true + +eni: + # -- Enable Elastic Network Interface (ENI) integration. + enabled: false + # -- Update ENI Adapter limits from the EC2 API + updateEC2AdapterLimitViaAPI: true + # -- Release IPs not used from the ENI + awsReleaseExcessIPs: false + # -- Enable ENI prefix delegation + awsEnablePrefixDelegation: false + # -- EC2 API endpoint to use + ec2APIEndpoint: "" + # -- Tags to apply to the newly created ENIs + eniTags: {} + # -- Interval for garbage collection of unattached ENIs. Set to "0s" to disable. + # @default -- `"5m"` + gcInterval: "" + # -- Additional tags attached to ENIs created by Cilium. + # Dangling ENIs with this tag will be garbage collected + # @default -- `{"io.cilium/cilium-managed":"true,"io.cilium/cluster-name":""}` + gcTags: {} + # -- If using IAM role for Service Accounts will not try to + # inject identity values from cilium-aws kubernetes secret. + # Adds annotation to service account if managed by Helm. + # See https://github.com/aws/amazon-eks-pod-identity-webhook + iamRole: "" + # -- Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. + subnetIDsFilter: [] + # -- Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. + subnetTagsFilter: [] + # -- Filter via AWS EC2 Instance tags (k=v) which will dictate which AWS EC2 Instances + # are going to be used to create new ENIs + instanceTagsFilter: [] + +externalIPs: + # -- Enable ExternalIPs service support. + enabled: false + +# fragmentTracking enables IPv4 fragment tracking support in the datapath. +# fragmentTracking: true + +gke: + # -- Enable Google Kubernetes Engine integration + enabled: false + +# -- Enable connectivity health checking. +healthChecking: true + +# -- TCP port for the agent health API. This is not the port for cilium-health. +healthPort: 9879 + +# -- Configure the host firewall. +hostFirewall: + # -- Enables the enforcement of host policies in the eBPF datapath. + enabled: false + +hostPort: + # -- Enable hostPort service support. + enabled: false + +# -- Configure socket LB +socketLB: + # -- Enable socket LB + enabled: false + + # -- Disable socket lb for non-root ns. This is used to enable Istio routing rules. + # hostNamespaceOnly: false + +# -- Configure certificate generation for Hubble integration. +# If hubble.tls.auto.method=cronJob, these values are used +# for the Kubernetes CronJob which will be scheduled regularly to +# (re)generate any certificates not provided manually. +certgen: + image: + override: ~ + repository: "quay.io/cilium/certgen" + tag: "v0.1.9" + digest: "sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f" + useDigest: true + pullPolicy: "IfNotPresent" + # -- Seconds after which the completed job pod will be deleted + ttlSecondsAfterFinished: 1800 + # -- Labels to be added to hubble-certgen pods + podLabels: {} + # -- Annotations to be added to the hubble-certgen initial Job and CronJob + annotations: + job: {} + cronJob: {} + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- Additional certgen volumes. + extraVolumes: [] + + # -- Additional certgen volumeMounts. + extraVolumeMounts: [] + + # -- Affinity for certgen + affinity: {} + +hubble: + # -- Enable Hubble (true by default). + enabled: true + + # -- Annotations to be added to all top-level hubble objects (resources under templates/hubble) + annotations: {} + + # -- Buffer size of the channel Hubble uses to receive monitor events. If this + # value is not set, the queue size is set to the default monitor queue size. + # eventQueueSize: "" + + # -- Number of recent flows for Hubble to cache. Defaults to 4095. + # Possible values are: + # 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, + # 2047, 4095, 8191, 16383, 32767, 65535 + # eventBufferCapacity: "4095" + + # -- Hubble metrics configuration. + # See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics + # for more comprehensive documentation about Hubble metrics. + metrics: + # -- Configures the list of metrics to collect. If empty or null, metrics + # are disabled. + # Example: + # + # enabled: + # - dns:query;ignoreAAAA + # - drop + # - tcp + # - flow + # - icmp + # - http + # + # You can specify the list of metrics from the helm CLI: + # + # --set hubble.metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" + # + enabled: ~ + # -- Enables exporting hubble metrics in OpenMetrics format. + enableOpenMetrics: false + # -- Configure the port the hubble metric server listens on. + port: 9965 + # -- Annotations to be added to hubble-metrics service. + serviceAnnotations: {} + serviceMonitor: + # -- Create ServiceMonitor resources for Prometheus Operator. + # This requires the prometheus CRDs to be available. + # ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor hubble + labels: {} + # -- Annotations to add to ServiceMonitor hubble + annotations: {} + # -- jobLabel to add for ServiceMonitor hubble + jobLabel: "" + # -- Interval for scrape metrics. + interval: "10s" + # -- Relabeling configs for the ServiceMonitor hubble + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor hubble + metricRelabelings: ~ + # -- Grafana dashboards for hubble + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards + dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + + # -- Unix domain socket path to listen to when Hubble is enabled. + socketPath: /var/run/cilium/hubble.sock + + # -- Enables redacting sensitive information present in Layer 7 flows. + redact: + enabled: false + http: + # -- Enables redacting URL query (GET) parameters. + # Example: + # + # redact: + # enabled: true + # http: + # urlQuery: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.urlQuery="true" + urlQuery: false + # -- Enables redacting user info, e.g., password when basic auth is used. + # Example: + # + # redact: + # enabled: true + # http: + # userInfo: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.userInfo="true" + userInfo: true + headers: + # -- List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # allow: + # - traceparent + # - tracestate + # - Cache-Control + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" + allow: [] + # -- List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # deny: + # - Authorization + # - Proxy-Authorization + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" + deny: [] + kafka: + # -- Enables redacting Kafka's API key. + # Example: + # + # redact: + # enabled: true + # kafka: + # apiKey: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.kafka.apiKey="true" + apiKey: false + + # -- An additional address for Hubble to listen to. + # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that + # Hubble is listening on port 4244. + listenAddress: ":4244" + # -- Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. + preferIpv6: false + # -- (bool) Skip Hubble events with unknown cgroup ids + # @default -- `true` + skipUnknownCGroupIDs: ~ + + peerService: + # -- Service Port for the Peer service. + # If not set, it is dynamically assigned to port 443 if TLS is enabled and to + # port 80 if not. + # servicePort: 80 + # -- Target Port for the Peer service, must match the hubble.listenAddress' + # port. + targetPort: 4244 + # -- The cluster domain to use to query the Hubble Peer service. It should + # be the local cluster. + clusterDomain: cluster.local + # -- TLS configuration for Hubble + tls: + # -- Enable mutual TLS for listenAddress. Setting this value to false is + # highly discouraged as the Hubble API provides access to potentially + # sensitive network flow metadata and is exposed on the host network. + enabled: true + # -- Configure automatic TLS certificates generation. + auto: + # -- Auto-generate certificates. + # When set to true, automatically generate a CA and certificates to + # enable mTLS between Hubble server and Hubble Relay instances. If set to + # false, the certs for Hubble server need to be provided by setting + # appropriate values below. + enabled: true + # -- Set the method to auto-generate certificates. Supported values: + # - helm: This method uses Helm to generate all certificates. + # - cronJob: This method uses a Kubernetes CronJob the generate any + # certificates not provided by the user at installation + # time. + # - certmanager: This method use cert-manager to generate & rotate certificates. + method: helm + # -- Generated certificates validity duration in days. + certValidityDuration: 1095 + # -- Schedule for certificates regeneration (regardless of their expiration date). + # Only used if method is "cronJob". If nil, then no recurring job will be created. + # Instead, only the one-shot job is deployed to generate the certificates at + # installation time. + # + # Defaults to midnight of the first day of every fourth month. For syntax, see + # https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax + schedule: "0 0 1 */4 *" + + # [Example] + # certManagerIssuerRef: + # group: cert-manager.io + # kind: ClusterIssuer + # name: ca-issuer + # -- certmanager issuer used when hubble.tls.auto.method=certmanager. + certManagerIssuerRef: {} + + # -- base64 encoded PEM values for the Hubble server certificate and private key + server: + cert: "" + key: "" + # -- Extra DNS names added to certificate when it's auto generated + extraDnsNames: [] + # -- Extra IP addresses added to certificate when it's auto generated + extraIpAddresses: [] + + relay: + # -- Enable Hubble Relay (requires hubble.enabled=true) + enabled: false + + # -- Roll out Hubble Relay pods automatically when configmap is updated. + rollOutPods: false + + # -- Hubble-relay container image. + image: + override: ~ + repository: "quay.io/cilium/hubble-relay" + tag: "v1.15.3" + # hubble-relay-digest + digest: "sha256:b9c6431aa4f22242a5d0d750c621d9d04bdc25549e4fb1116bfec98dd87958a2" + useDigest: true + pullPolicy: "IfNotPresent" + + # -- Specifies the resources for the hubble-relay pods + resources: {} + + # -- Number of replicas run for the hubble-relay deployment. + replicas: 1 + + # -- Affinity for hubble-replay + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + + # -- Pod topology spread constraints for hubble-relay + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- Additional hubble-relay environment variables. + extraEnv: [] + + # -- Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) + annotations: {} + + # -- Annotations to be added to hubble-relay pods + podAnnotations: {} + + # -- Labels to be added to hubble-relay pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- The priority class to use for hubble-relay + priorityClassName: "" + + # -- Configure termination grace period for hubble relay Deployment. + terminationGracePeriodSeconds: 1 + + # -- hubble-relay update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- Additional hubble-relay volumes. + extraVolumes: [] + + # -- Additional hubble-relay volumeMounts. + extraVolumeMounts: [] + + # -- hubble-relay pod security context + podSecurityContext: + fsGroup: 65532 + + # -- hubble-relay container security context + securityContext: + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + + # -- hubble-relay service configuration. + service: + # --- The type of service used for Hubble Relay access, either ClusterIP or NodePort. + type: ClusterIP + # --- The port to use when the service type is set to NodePort. + nodePort: 31234 + + # -- Host to listen to. Specify an empty string to bind to all the interfaces. + listenHost: "" + + # -- Port to listen to. + listenPort: "4245" + + # -- TLS configuration for Hubble Relay + tls: + # -- base64 encoded PEM values for the hubble-relay client certificate and private key + # This keypair is presented to Hubble server instances for mTLS + # authentication and is required when hubble.tls.enabled is true. + # These values need to be set manually if hubble.tls.auto.enabled is false. + client: + cert: "" + key: "" + # -- base64 encoded PEM values for the hubble-relay server certificate and private key + server: + # When set to true, enable TLS on for Hubble Relay server + # (ie: for clients connecting to the Hubble Relay API). + enabled: false + # When set to true enforces mutual TLS between Hubble Relay server and its clients. + # False allow non-mutual TLS connections. + # This option has no effect when TLS is disabled. + mtls: false + # These values need to be set manually if hubble.tls.auto.enabled is false. + cert: "" + key: "" + # -- extra DNS names added to certificate when its auto gen + extraDnsNames: [] + # -- extra IP addresses added to certificate when its auto gen + extraIpAddresses: [] + # DNS name used by the backend to connect to the relay + # This is a simple workaround as the relay certificates are currently hardcoded to + # *.hubble-relay.cilium.io + # See https://github.com/cilium/cilium/pull/28709#discussion_r1371792546 + # For GKE Dataplane V2 this should be set to relay.kube-system.svc.cluster.local + relayName: "ui.hubble-relay.cilium.io" + + # -- Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). + dialTimeout: ~ + + # -- Backoff duration to retry connecting to the local hubble instance in case of failure (e.g. "30s"). + retryTimeout: ~ + + # -- Max number of flows that can be buffered for sorting before being sent to the + # client (per request) (e.g. 100). + sortBufferLenMax: ~ + + # -- When the per-request flows sort buffer is not full, a flow is drained every + # time this timeout is reached (only affects requests in follow-mode) (e.g. "1s"). + sortBufferDrainTimeout: ~ + + # -- Port to use for the k8s service backed by hubble-relay pods. + # If not set, it is dynamically assigned to port 443 if TLS is enabled and to + # port 80 if not. + # servicePort: 80 + + # -- Enable prometheus metrics for hubble-relay on the configured port at + # /metrics + prometheus: + enabled: false + port: 9966 + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor hubble-relay + labels: {} + # -- Annotations to add to ServiceMonitor hubble-relay + annotations: {} + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor hubble-relay + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor hubble-relay + metricRelabelings: ~ + + gops: + # -- Enable gops for hubble-relay + enabled: true + # -- Configure gops listen port for hubble-relay + port: 9893 + + pprof: + # -- Enable pprof for hubble-relay + enabled: false + # -- Configure pprof listen address for hubble-relay + address: localhost + # -- Configure pprof listen port for hubble-relay + port: 6062 + + ui: + # -- Whether to enable the Hubble UI. + enabled: false + + standalone: + # -- When true, it will allow installing the Hubble UI only, without checking dependencies. + # It is useful if a cluster already has cilium and Hubble relay installed and you just + # want Hubble UI to be deployed. + # When installed via helm, installing UI should be done via `helm upgrade` and when installed via the cilium cli, then `cilium hubble enable --ui` + enabled: false + + tls: + # -- When deploying Hubble UI in standalone, with tls enabled for Hubble relay, it is required + # to provide a volume for mounting the client certificates. + certsVolume: {} + # projected: + # defaultMode: 0400 + # sources: + # - secret: + # name: hubble-ui-client-certs + # items: + # - key: tls.crt + # path: client.crt + # - key: tls.key + # path: client.key + # - key: ca.crt + # path: hubble-relay-ca.crt + + # -- Roll out Hubble-ui pods automatically when configmap is updated. + rollOutPods: false + + tls: + # -- base64 encoded PEM values used to connect to hubble-relay + # This keypair is presented to Hubble Relay instances for mTLS + # authentication and is required when hubble.relay.tls.server.enabled is true. + # These values need to be set manually if hubble.tls.auto.enabled is false. + client: + cert: "" + key: "" + + backend: + # -- Hubble-ui backend image. + image: + override: ~ + repository: "quay.io/cilium/hubble-ui-backend" + tag: "v0.13.0" + digest: "sha256:1e7657d997c5a48253bb8dc91ecee75b63018d16ff5e5797e5af367336bc8803" + useDigest: true + pullPolicy: "IfNotPresent" + + # -- Hubble-ui backend security context. + securityContext: {} + + # -- Additional hubble-ui backend environment variables. + extraEnv: [] + + # -- Additional hubble-ui backend volumes. + extraVolumes: [] + + # -- Additional hubble-ui backend volumeMounts. + extraVolumeMounts: [] + + livenessProbe: + # -- Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + + readinessProbe: + # -- Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + + # -- Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. + resources: {} + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + + frontend: + # -- Hubble-ui frontend image. + image: + override: ~ + repository: "quay.io/cilium/hubble-ui" + tag: "v0.13.0" + digest: "sha256:7d663dc16538dd6e29061abd1047013a645e6e69c115e008bee9ea9fef9a6666" + useDigest: true + pullPolicy: "IfNotPresent" + + # -- Hubble-ui frontend security context. + securityContext: {} + + # -- Additional hubble-ui frontend environment variables. + extraEnv: [] + + # -- Additional hubble-ui frontend volumes. + extraVolumes: [] + + # -- Additional hubble-ui frontend volumeMounts. + extraVolumeMounts: [] + + # -- Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. + resources: {} + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + server: + # -- Controls server listener for ipv6 + ipv6: + enabled: true + + # -- The number of replicas of Hubble UI to deploy. + replicas: 1 + + # -- Annotations to be added to all top-level hubble-ui objects (resources under templates/hubble-ui) + annotations: {} + + # -- Annotations to be added to hubble-ui pods + podAnnotations: {} + + # -- Labels to be added to hubble-ui pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- Affinity for hubble-ui + affinity: {} + + # -- Pod topology spread constraints for hubble-ui + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- The priority class to use for hubble-ui + priorityClassName: "" + + # -- hubble-ui update strategy. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- Security context to be added to Hubble UI pods + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + + # -- hubble-ui service configuration. + service: + # -- Annotations to be added for the Hubble UI service + annotations: {} + # --- The type of service used for Hubble UI access, either ClusterIP or NodePort. + type: ClusterIP + # --- The port to use when the service type is set to NodePort. + nodePort: 31235 + + # -- Defines base url prefix for all hubble-ui http requests. + # It needs to be changed in case if ingress for hubble-ui is configured under some sub-path. + # Trailing `/` is required for custom path, ex. `/service-map/` + baseUrl: "/" + + # -- hubble-ui ingress configuration. + ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + className: "" + hosts: + - chart-example.local + labels: {} + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + # -- Hubble flows export. + export: + # --- Defines max file size of output file before it gets rotated. + fileMaxSizeMb: 10 + # --- Defines max number of backup/rotated files. + fileMaxBackups: 5 + # --- Static exporter configuration. + # Static exporter is bound to agent lifecycle. + static: + enabled: false + filePath: /var/run/cilium/hubble/events.log + fieldMask: [] + # - time + # - source + # - destination + # - verdict + allowList: [] + # - '{"verdict":["DROPPED","ERROR"]}' + denyList: [] + # - '{"source_pod":["kube-system/"]}' + # - '{"destination_pod":["kube-system/"]}' + # --- Dynamic exporters configuration. + # Dynamic exporters may be reconfigured without a need of agent restarts. + dynamic: + enabled: false + config: + # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. + configMapName: cilium-flowlog-config + # ---- True if helm installer should create config map. + # Switch to false if you want to self maintain the file content. + createConfigMap: true + # ---- Exporters configuration in YAML format. + content: + - name: all + fieldMask: [] + includeFilters: [] + excludeFilters: [] + filePath: "/var/run/cilium/hubble/events.log" + #- name: "test002" + # filePath: "/var/log/network/flow-log/pa/test002.log" + # fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"] + # includeFilters: + # - source_pod: ["default/"] + # event_type: + # - type: 1 + # - destination_pod: ["frontend/nginx-975996d4c-7hhgt"] + # excludeFilters: [] + # end: "2023-10-09T23:59:59-07:00" + +# -- Method to use for identity allocation (`crd` or `kvstore`). +identityAllocationMode: "crd" + +# -- (string) Time to wait before using new identity on endpoint identity change. +# @default -- `"5s"` +identityChangeGracePeriod: "" + +# -- Install Iptables rules to skip netfilter connection tracking on all pod +# traffic. This option is only effective when Cilium is running in direct +# routing and full KPR mode. Moreover, this option cannot be enabled when Cilium +# is running in a managed Kubernetes environment or in a chained CNI setup. +installNoConntrackIptablesRules: false + +ipam: + # -- Configure IP Address Management mode. + # ref: https://docs.cilium.io/en/stable/network/concepts/ipam/ + mode: "cluster-pool" + # -- Maximum rate at which the CiliumNode custom resource is updated. + ciliumNodeUpdateRate: "15s" + operator: + # -- IPv4 CIDR list range to delegate to individual nodes for IPAM. + clusterPoolIPv4PodCIDRList: ["10.0.0.0/8"] + # -- IPv4 CIDR mask size to delegate to individual nodes for IPAM. + clusterPoolIPv4MaskSize: 24 + # -- IPv6 CIDR list range to delegate to individual nodes for IPAM. + clusterPoolIPv6PodCIDRList: ["fd00::/104"] + # -- IPv6 CIDR mask size to delegate to individual nodes for IPAM. + clusterPoolIPv6MaskSize: 120 + # -- IP pools to auto-create in multi-pool IPAM mode. + autoCreateCiliumPodIPPools: {} + # default: + # ipv4: + # cidrs: + # - 10.10.0.0/8 + # maskSize: 24 + # other: + # ipv6: + # cidrs: + # - fd00:100::/80 + # maskSize: 96 + # -- The maximum burst size when rate limiting access to external APIs. + # Also known as the token bucket capacity. + # @default -- `20` + externalAPILimitBurstSize: ~ + # -- The maximum queries per second when rate limiting access to + # external APIs. Also known as the bucket refill rate, which is used to + # refill the bucket up to the burst size capacity. + # @default -- `4.0` + externalAPILimitQPS: ~ + +# -- The api-rate-limit option can be used to overwrite individual settings of the default configuration for rate limiting calls to the Cilium Agent API +apiRateLimit: ~ + +# -- Configure the eBPF-based ip-masq-agent +ipMasqAgent: + enabled: false +# the config of nonMasqueradeCIDRs +# config: + # nonMasqueradeCIDRs: [] + # masqLinkLocal: false + # masqLinkLocalIPv6: false + +# iptablesLockTimeout defines the iptables "--wait" option when invoked from Cilium. +# iptablesLockTimeout: "5s" + +ipv4: + # -- Enable IPv4 support. + enabled: true + +ipv6: + # -- Enable IPv6 support. + enabled: false + +# -- Configure Kubernetes specific configuration +k8s: {} + # -- requireIPv4PodCIDR enables waiting for Kubernetes to provide the PodCIDR + # range via the Kubernetes node resource + # requireIPv4PodCIDR: false + + # -- requireIPv6PodCIDR enables waiting for Kubernetes to provide the PodCIDR + # range via the Kubernetes node resource + # requireIPv6PodCIDR: false + +# -- Keep the deprecated selector labels when deploying Cilium DaemonSet. +keepDeprecatedLabels: false + +# -- Keep the deprecated probes when deploying Cilium DaemonSet +keepDeprecatedProbes: false + +startupProbe: + # -- failure threshold of startup probe. + # 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) + failureThreshold: 105 + # -- interval between checks of the startup probe + periodSeconds: 2 +livenessProbe: + # -- failure threshold of liveness probe + failureThreshold: 10 + # -- interval between checks of the liveness probe + periodSeconds: 30 +readinessProbe: + # -- failure threshold of readiness probe + failureThreshold: 3 + # -- interval between checks of the readiness probe + periodSeconds: 30 + +# -- Configure the kube-proxy replacement in Cilium BPF datapath +# Valid options are "true", "false", "disabled" (deprecated), "partial" (deprecated), "strict" (deprecated). +# ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ +#kubeProxyReplacement: "false" + +# -- healthz server bind address for the kube-proxy replacement. +# To enable set the value to '0.0.0.0:10256' for all ipv4 +# addresses and this '[::]:10256' for all ipv6 addresses. +# By default it is disabled. +kubeProxyReplacementHealthzBindAddr: "" + +l2NeighDiscovery: + # -- Enable L2 neighbor discovery in the agent + enabled: true + # -- Override the agent's default neighbor resolution refresh period. + refreshPeriod: "30s" + +# -- Enable Layer 7 network policy. +l7Proxy: true + +# -- Enable Local Redirect Policy. +localRedirectPolicy: false + +# To include or exclude matched resources from cilium identity evaluation +# labels: "" + +# logOptions allows you to define logging options. eg: +# logOptions: +# format: json + +# -- Enables periodic logging of system load +logSystemLoad: false + +# -- Configure maglev consistent hashing +maglev: {} + # -- tableSize is the size (parameter M) for the backend table of one + # service entry + # tableSize: + + # -- hashSeed is the cluster-wide base64 encoded seed for the hashing + # hashSeed: + +# -- Enables masquerading of IPv4 traffic leaving the node from endpoints. +enableIPv4Masquerade: true + +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +enableIPv6Masquerade: true + +# -- Enables masquerading to the source of the route for traffic leaving the node from endpoints. +enableMasqueradeRouteSource: false + +# -- Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods +enableIPv4BIGTCP: false + +# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods +enableIPv6BIGTCP: false + +egressGateway: + # -- Enables egress gateway to redirect and SNAT the traffic that leaves the + # cluster. + enabled: false + # -- Deprecated without a replacement necessary. + installRoutes: false + # -- Time between triggers of egress gateway state reconciliations + reconciliationTriggerInterval: 1s + # -- Maximum number of entries in egress gateway policy map + # maxPolicyEntries: 16384 + +vtep: +# -- Enables VXLAN Tunnel Endpoint (VTEP) Integration (beta) to allow +# Cilium-managed pods to talk to third party VTEP devices over Cilium tunnel. + enabled: false + +# -- A space separated list of VTEP device endpoint IPs, for example "1.1.1.1 1.1.2.1" + endpoint: "" +# -- A space separated list of VTEP device CIDRs, for example "1.1.1.0/24 1.1.2.0/24" + cidr: "" +# -- VTEP CIDRs Mask that applies to all VTEP CIDRs, for example "255.255.255.0" + mask: "" +# -- A space separated list of VTEP device MAC addresses (VTEP MAC), for example "x:x:x:x:x:x y:y:y:y:y:y:y" + mac: "" + +# -- (string) Allows to explicitly specify the IPv4 CIDR for native routing. +# When specified, Cilium assumes networking for this CIDR is preconfigured and +# hands traffic destined for that range to the Linux network stack without +# applying any SNAT. +# Generally speaking, specifying a native routing CIDR implies that Cilium can +# depend on the underlying networking stack to route packets to their +# destination. To offer a concrete example, if Cilium is configured to use +# direct routing and the Kubernetes CIDR is included in the native routing CIDR, +# the user must configure the routes to reach pods, either manually or by +# setting the auto-direct-node-routes flag. +ipv4NativeRoutingCIDR: "" + +# -- (string) Allows to explicitly specify the IPv6 CIDR for native routing. +# When specified, Cilium assumes networking for this CIDR is preconfigured and +# hands traffic destined for that range to the Linux network stack without +# applying any SNAT. +# Generally speaking, specifying a native routing CIDR implies that Cilium can +# depend on the underlying networking stack to route packets to their +# destination. To offer a concrete example, if Cilium is configured to use +# direct routing and the Kubernetes CIDR is included in the native routing CIDR, +# the user must configure the routes to reach pods, either manually or by +# setting the auto-direct-node-routes flag. +ipv6NativeRoutingCIDR: "" + +# -- cilium-monitor sidecar. +monitor: + # -- Enable the cilium-monitor sidecar. + enabled: false + +# -- Configure service load balancing +loadBalancer: + # -- standalone enables the standalone L4LB which does not connect to + # kube-apiserver. + # standalone: false + + # -- algorithm is the name of the load balancing algorithm for backend + # selection e.g. random or maglev + # algorithm: random + + # -- mode is the operation mode of load balancing for remote backends + # e.g. snat, dsr, hybrid + # mode: snat + + # -- acceleration is the option to accelerate service handling via XDP + # Applicable values can be: disabled (do not use XDP), native (XDP BPF + # program is run directly out of the networking driver's early receive + # path), or best-effort (use native mode XDP acceleration on devices + # that support it). + acceleration: disabled + + # -- dsrDispatch configures whether IP option or IPIP encapsulation is + # used to pass a service IP and port to remote backend + # dsrDispatch: opt + + # -- serviceTopology enables K8s Topology Aware Hints -based service + # endpoints filtering + # serviceTopology: false + + # -- L7 LoadBalancer + l7: + # -- Enable L7 service load balancing via envoy proxy. + # The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, + # will be forwarded to the local backend proxy to be load balanced to the service endpoints. + # Please refer to docs for supported annotations for more configuration. + # + # Applicable values: + # - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. + # - disabled: Disable L7 load balancing by way of service annotation. + backend: disabled + # -- List of ports from service to be automatically redirected to above backend. + # Any service exposing one of these ports will be automatically redirected. + # Fine-grained control can be achieved by using the service annotation. + ports: [] + # -- Default LB algorithm + # The default LB algorithm to be used for services, which can be overridden by the + # service annotation (e.g. service.cilium.io/lb-l7-algorithm) + # Applicable values: round_robin, least_request, random + algorithm: round_robin + +# -- Configure N-S k8s service loadbalancing +nodePort: + # -- Enable the Cilium NodePort service implementation. + enabled: false + + # -- Port range to use for NodePort services. + # range: "30000,32767" + + # -- Set to true to prevent applications binding to service ports. + bindProtection: true + + # -- Append NodePort range to ip_local_reserved_ports if clash with ephemeral + # ports is detected. + autoProtectPortRange: true + + # -- Enable healthcheck nodePort server for NodePort services + enableHealthCheck: true + + # -- Enable access of the healthcheck nodePort on the LoadBalancerIP. Needs + # EnableHealthCheck to be enabled + enableHealthCheckLoadBalancerIP: false + +# policyAuditMode: false + +# -- The agent can be put into one of the three policy enforcement modes: +# default, always and never. +# ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes +policyEnforcementMode: "default" + +# -- policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. +# The possible value is "nodes". +policyCIDRMatchMode: + +pprof: + # -- Enable pprof for cilium-agent + enabled: false + # -- Configure pprof listen address for cilium-agent + address: localhost + # -- Configure pprof listen port for cilium-agent + port: 6060 + +# -- Configure prometheus metrics on the configured port at /metrics +prometheus: + enabled: false + port: 9962 + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor cilium-agent + labels: {} + # -- Annotations to add to ServiceMonitor cilium-agent + annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-agent + jobLabel: "" + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor cilium-agent + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor cilium-agent + metricRelabelings: ~ + # -- Set to `true` and helm will not check for monitoring.coreos.com/v1 CRDs before deploying + trustCRDsExist: false + + # -- Metrics that should be enabled or disabled from the default metric list. + # The list is expected to be separated by a space. (+metric_foo to enable + # metric_foo , -metric_bar to disable metric_bar). + # ref: https://docs.cilium.io/en/stable/observability/metrics/ + metrics: ~ + + # --- Enable controller group metrics for monitoring specific Cilium + # subsystems. The list is a list of controller group names. The special + # values of "all" and "none" are supported. The set of controller + # group names is not guaranteed to be stable between Cilium versions. + controllerGroupMetrics: + - write-cni-file + - sync-host-ips + - sync-lb-maps-with-k8s-services + +# -- Grafana dashboards for cilium-agent +# grafana can import dashboards based on the label and value +# ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards +dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + +# -- Configure Istio proxy options. +proxy: + + prometheus: + # -- Deprecated in favor of envoy.prometheus.enabled + enabled: true + # -- Deprecated in favor of envoy.prometheus.port + port: ~ + # -- Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecarImageRegex: "cilium/istio_proxy" + +# Configure Cilium Envoy options. +envoy: + # -- Enable Envoy Proxy in standalone DaemonSet. + enabled: false + + log: + # -- The format string to use for laying out the log message metadata of Envoy. + format: "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v" + # -- Path to a separate Envoy log file, if any. Defaults to /dev/stdout. + path: "" + + # -- Time in seconds after which a TCP connection attempt times out + connectTimeoutSeconds: 2 + # -- ProxyMaxRequestsPerConnection specifies the max_requests_per_connection setting for Envoy + maxRequestsPerConnection: 0 + # -- Set Envoy HTTP option max_connection_duration seconds. Default 0 (disable) + maxConnectionDurationSeconds: 0 + # -- Set Envoy upstream HTTP idle connection timeout seconds. + # Does not apply to connections with pending requests. Default 60s + idleTimeoutDurationSeconds: 60 + + # -- Envoy container image. + image: + override: ~ + repository: "quay.io/cilium/cilium-envoy" + tag: "v1.27.3-99c1c8f42c8de70fc8f6dd594f4a425cd38b6688" + pullPolicy: "IfNotPresent" + digest: "sha256:877ead12d08d4c04a9f67f86d3c6e542aeb7bf97e1e401aee74de456f496ac30" + useDigest: true + + # -- Additional containers added to the cilium Envoy DaemonSet. + extraContainers: [] + + # -- Additional envoy container arguments. + extraArgs: [] + + # -- Additional envoy container environment variables. + extraEnv: [] + + # -- Additional envoy hostPath mounts. + extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + + # -- Additional envoy volumes. + extraVolumes: [] + + # -- Additional envoy volumeMounts. + extraVolumeMounts: [] + + # -- Configure termination grace period for cilium-envoy DaemonSet. + terminationGracePeriodSeconds: 1 + + # -- TCP port for the health API. + healthPort: 9878 + + # -- cilium-envoy update strategy + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + # -- Roll out cilium envoy pods automatically when configmap is updated. + rollOutPods: false + + # -- Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) + annotations: {} + + # -- Security Context for cilium-envoy pods. + podSecurityContext: {} + + # -- Annotations to be added to envoy pods + podAnnotations: {} + + # -- Labels to be added to envoy pods + podLabels: {} + + # -- Envoy resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + startupProbe: + # -- failure threshold of startup probe. + # 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) + failureThreshold: 105 + # -- interval between checks of the startup probe + periodSeconds: 2 + livenessProbe: + # -- failure threshold of liveness probe + failureThreshold: 10 + # -- interval between checks of the liveness probe + periodSeconds: 30 + readinessProbe: + # -- failure threshold of readiness probe + failureThreshold: 3 + # -- interval between checks of the readiness probe + periodSeconds: 30 + + securityContext: + # -- User to run the pod with + # runAsUser: 0 + # -- Run the pod with elevated privileges + privileged: false + # -- SELinux options for the `cilium-envoy` container + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + # -- Capabilities for the `cilium-envoy` container + envoy: + # Used since cilium proxy uses setting IPPROTO_IP/IP_TRANSPARENT + - NET_ADMIN + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + + # -- Affinity for cilium-envoy. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium-envoy + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + # -- Node selector for cilium-envoy. + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for envoy scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- The priority class to use for cilium-envoy. + priorityClassName: ~ + + # -- DNS policy for Cilium envoy pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: ~ + + # -- Configure Cilium Envoy Prometheus options. + # Note that some of these apply to either cilium-agent or cilium-envoy. + prometheus: + # -- Enable prometheus metrics for cilium-envoy + enabled: true + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + # Note that this setting applies to both cilium-envoy _and_ cilium-agent + # with Envoy enabled. + enabled: false + # -- Labels to add to ServiceMonitor cilium-envoy + labels: {} + # -- Annotations to add to ServiceMonitor cilium-envoy + annotations: {} + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. + metricRelabelings: ~ + # -- Serve prometheus metrics for cilium-envoy on the configured port + port: "9964" + +# -- Enable use of the remote node identity. +# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# Deprecated without replacement in 1.15. To be removed in 1.16. +remoteNodeIdentity: true + +# -- Enable resource quotas for priority classes used in the cluster. +resourceQuotas: + enabled: false + cilium: + hard: + # 5k nodes * 2 DaemonSets (Cilium and cilium node init) + pods: "10k" + operator: + hard: + # 15 "clusterwide" Cilium Operator pods for HA + pods: "15" + +# Need to document default +################## +#sessionAffinity: false + +# -- Do not run Cilium agent when running with clean mode. Useful to completely +# uninstall Cilium as it will stop Cilium from starting and create artifacts +# in the node. +sleepAfterInit: false + +# -- Enable check of service source ranges (currently, only for LoadBalancer). +svcSourceRangeCheck: true + +# -- Synchronize Kubernetes nodes to kvstore and perform CNP GC. +synchronizeK8sNodes: true + +# -- Configure TLS configuration in the agent. +tls: + # -- This configures how the Cilium agent loads the secrets used TLS-aware CiliumNetworkPolicies + # (namely the secrets referenced by terminatingTLS and originatingTLS). + # Possible values: + # - local + # - k8s + secretsBackend: local + + # -- Base64 encoded PEM values for the CA certificate and private key. + # This can be used as common CA to generate certificates used by hubble and clustermesh components. + # It is neither required nor used when cert-manager is used to generate the certificates. + ca: + # -- Optional CA cert. If it is provided, it will be used by cilium to + # generate all other certificates. Otherwise, an ephemeral CA is generated. + cert: "" + + # -- Optional CA private key. If it is provided, it will be used by cilium to + # generate all other certificates. Otherwise, an ephemeral CA is generated. + key: "" + + # -- Generated certificates validity duration in days. This will be used for auto generated CA. + certValidityDuration: 1095 + + # -- Configure the CA trust bundle used for the validation of the certificates + # leveraged by hubble and clustermesh. When enabled, it overrides the content of the + # 'ca.crt' field of the respective certificates, allowing for CA rotation with no down-time. + caBundle: + # -- Enable the use of the CA trust bundle. + enabled: false + + # -- Name of the ConfigMap containing the CA trust bundle. + name: cilium-root-ca.crt + + # -- Entry of the ConfigMap containing the CA trust bundle. + key: ca.crt + + # -- Use a Secret instead of a ConfigMap. + useSecret: false + + # If uncommented, creates the ConfigMap and fills it with the specified content. + # Otherwise, the ConfigMap is assumed to be already present in .Release.Namespace. + # + # content: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + +# -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. +# Possible values: +# - "" +# - vxlan +# - geneve +# @default -- `"vxlan"` +tunnelProtocol: "" + +# -- Enable native-routing mode or tunneling mode. +# Possible values: +# - "" +# - native +# - tunnel +# @default -- `"tunnel"` +routingMode: "" + +# -- Configure VXLAN and Geneve tunnel port. +# @default -- Port 8472 for VXLAN, Port 6081 for Geneve +tunnelPort: 0 + +# -- Configure what the response should be to traffic for a service without backends. +# "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". +# Possible values: +# - reject (default) +# - drop +serviceNoBackendResponse: reject + +# -- Configure the underlying network MTU to overwrite auto-detected MTU. +MTU: 0 + +# -- Disable the usage of CiliumEndpoint CRD. +disableEndpointCRD: false + +wellKnownIdentities: + # -- Enable the use of well-known identities. + enabled: false + +etcd: + # -- Enable etcd mode for the agent. + enabled: false + + # -- cilium-etcd-operator image. + image: + override: ~ + repository: "quay.io/cilium/cilium-etcd-operator" + tag: "v2.0.7" + digest: "sha256:04b8327f7f992693c2cb483b999041ed8f92efc8e14f2a5f3ab95574a65ea2dc" + useDigest: true + pullPolicy: "IfNotPresent" + + # -- The priority class to use for cilium-etcd-operator + priorityClassName: "" + + # -- Additional cilium-etcd-operator container arguments. + extraArgs: [] + + # -- Additional cilium-etcd-operator volumes. + extraVolumes: [] + + # -- Additional cilium-etcd-operator volumeMounts. + extraVolumeMounts: [] + + # -- Node tolerations for cilium-etcd-operator scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Pod topology spread constraints for cilium-etcd-operator + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for cilium-etcd-operator pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Annotations to be added to all top-level etcd-operator objects (resources under templates/etcd-operator) + annotations: {} + + # -- Security context to be added to cilium-etcd-operator pods + podSecurityContext: {} + + # -- Annotations to be added to cilium-etcd-operator pods + podAnnotations: {} + + # -- Labels to be added to cilium-etcd-operator pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- cilium-etcd-operator resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + # -- Security context to be added to cilium-etcd-operator pods + securityContext: {} + # runAsUser: 0 + + # -- cilium-etcd-operator update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + + # -- If etcd is behind a k8s service set this option to true so that Cilium + # does the service translation automatically without requiring a DNS to be + # running. + k8sService: false + + # -- Cluster domain for cilium-etcd-operator. + clusterDomain: cluster.local + + # -- List of etcd endpoints (not needed when using managed=true). + endpoints: + - https://CHANGE-ME:2379 + + # -- Enable use of TLS/SSL for connectivity to etcd. (auto-enabled if + # managed=true) + ssl: false + +operator: + # -- Enable the cilium-operator component (required). + enabled: true + + # -- Roll out cilium-operator pods automatically when configmap is updated. + rollOutPods: false + + # -- cilium-operator image. + image: + override: ~ + repository: "quay.io/cilium/operator" + tag: "v1.15.3" + # operator-generic-digest + genericDigest: "sha256:c97f23161906b82f5c81a2d825b0646a5aa1dfb4adf1d49cbb87815079e69d61" + # operator-azure-digest + azureDigest: "sha256:b85a2671a74903c6e9a45e884654bb970b5b8d6a6e20371811a6cc0ad92b2f87" + # operator-aws-digest + awsDigest: "sha256:2b05dc6b88037a5ce05e4030ef616b1f7be9e65083e35abd36a1b66953fd0b6a" + # operator-alibabacloud-digest + alibabacloudDigest: "sha256:59d5c0c5782163d38151dd06bae0118144f6c080598901a632c628b1143ccd10" + useDigest: true + pullPolicy: "IfNotPresent" + suffix: "" + + # -- Number of replicas to run for the cilium-operator deployment + replicas: 2 + + # -- The priority class to use for cilium-operator + priorityClassName: "" + + # -- DNS policy for Cilium operator pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: "" + + # -- cilium-operator update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + + # -- Affinity for cilium-operator + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator + + # -- Pod topology spread constraints for cilium-operator + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for cilium-operator pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for cilium-operator scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Additional cilium-operator container arguments. + extraArgs: [] + + # -- Additional cilium-operator environment variables. + extraEnv: [] + + # -- Additional cilium-operator hostPath mounts. + extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + + # -- Additional cilium-operator volumes. + extraVolumes: [] + + # -- Additional cilium-operator volumeMounts. + extraVolumeMounts: [] + + # -- Annotations to be added to all top-level cilium-operator objects (resources under templates/cilium-operator) + annotations: {} + + # -- Security context to be added to cilium-operator pods + podSecurityContext: {} + + # -- Annotations to be added to cilium-operator pods + podAnnotations: {} + + # -- Labels to be added to cilium-operator pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- cilium-operator resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 1000m + # memory: 1Gi + # requests: + # cpu: 100m + # memory: 128Mi + + # -- Security context to be added to cilium-operator pods + securityContext: {} + # runAsUser: 0 + + # -- Interval for endpoint garbage collection. + endpointGCInterval: "5m0s" + + # -- Interval for cilium node garbage collection. + nodeGCInterval: "5m0s" + + # -- Skip CNP node status clean up at operator startup. + skipCNPStatusStartupClean: false + + # -- Interval for identity garbage collection. + identityGCInterval: "15m0s" + + # -- Timeout for identity heartbeats. + identityHeartbeatTimeout: "30m0s" + + pprof: + # -- Enable pprof for cilium-operator + enabled: false + # -- Configure pprof listen address for cilium-operator + address: localhost + # -- Configure pprof listen port for cilium-operator + port: 6061 + + # -- Enable prometheus metrics for cilium-operator on the configured port at + # /metrics + prometheus: + enabled: true + port: 9963 + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor cilium-operator + labels: {} + # -- Annotations to add to ServiceMonitor cilium-operator + annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-operator + jobLabel: "" + # -- Interval for scrape metrics. + interval: "10s" + # -- Relabeling configs for the ServiceMonitor cilium-operator + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor cilium-operator + metricRelabelings: ~ + + # -- Grafana dashboards for cilium-operator + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards + dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + + # -- Skip CRDs creation for cilium-operator + skipCRDCreation: false + + # -- Remove Cilium node taint from Kubernetes nodes that have a healthy Cilium + # pod running. + removeNodeTaints: true + + # -- Taint nodes where Cilium is scheduled but not running. This prevents pods + # from being scheduled to nodes where Cilium is not the default CNI provider. + # @default -- same as removeNodeTaints + setNodeTaints: ~ + + # -- Set Node condition NetworkUnavailable to 'false' with the reason + # 'CiliumIsUp' for nodes that have a healthy Cilium pod. + setNodeNetworkStatus: true + + unmanagedPodWatcher: + # -- Restart any pod that are not managed by Cilium. + restart: true + # -- Interval, in seconds, to check if there are any pods that are not + # managed by Cilium. + intervalSeconds: 15 + +nodeinit: + # -- Enable the node initialization DaemonSet + enabled: false + + # -- node-init image. + image: + override: ~ + repository: "quay.io/cilium/startup-script" + tag: "62093c5c233ea914bfa26a10ba41f8780d9b737f" + pullPolicy: "IfNotPresent" + + # -- The priority class to use for the nodeinit pod. + priorityClassName: "" + + # -- node-init update strategy + updateStrategy: + type: RollingUpdate + + # -- Additional nodeinit environment variables. + extraEnv: [] + + # -- Additional nodeinit volumes. + extraVolumes: [] + + # -- Additional nodeinit volumeMounts. + extraVolumeMounts: [] + + # -- Affinity for cilium-nodeinit + affinity: {} + + # -- Node labels for nodeinit pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for nodeinit scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Annotations to be added to all top-level nodeinit objects (resources under templates/cilium-nodeinit) + annotations: {} + + # -- Annotations to be added to node-init pods. + podAnnotations: {} + + # -- Labels to be added to node-init pods. + podLabels: {} + + # -- nodeinit resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + requests: + cpu: 100m + memory: 100Mi + + # -- Security context to be added to nodeinit pods. + securityContext: + privileged: false + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + add: + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # Used for nsenter + - NET_ADMIN + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + + # -- bootstrapFile is the location of the file where the bootstrap timestamp is + # written by the node-init DaemonSet + bootstrapFile: "/tmp/cilium-bootstrap.d/cilium-bootstrap-time" + + # -- startup offers way to customize startup nodeinit script (pre and post position) + startup: + preScript: "" + postScript: "" + # -- prestop offers way to customize prestop nodeinit script (pre and post position) + prestop: + preScript: "" + postScript: "" + +preflight: + # -- Enable Cilium pre-flight resources (required for upgrade) + enabled: false + + # -- Cilium pre-flight image. + image: + override: ~ + repository: "quay.io/cilium/cilium" + tag: "v1.15.3" + # cilium-digest + digest: "sha256:da74ab61d1bc665c1c088dff41d5be388d252ca5800f30c7d88844e6b5e440b0" + useDigest: true + pullPolicy: "IfNotPresent" + + # -- The priority class to use for the preflight pod. + priorityClassName: "" + + # -- preflight update strategy + updateStrategy: + type: RollingUpdate + + # -- Additional preflight environment variables. + extraEnv: [] + + # -- Additional preflight volumes. + extraVolumes: [] + + # -- Additional preflight volumeMounts. + extraVolumeMounts: [] + + # -- Affinity for cilium-preflight + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + + # -- Node labels for preflight pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for preflight scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) + annotations: {} + + # -- Security context to be added to preflight pods. + podSecurityContext: {} + + # -- Annotations to be added to preflight pods + podAnnotations: {} + + # -- Labels to be added to the preflight pod. + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- preflight resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + # -- Security context to be added to preflight pods + securityContext: {} + # runAsUser: 0 + + # -- Path to write the `--tofqdns-pre-cache` file to. + tofqdnsPreCache: "" + + # -- Configure termination grace period for preflight Deployment and DaemonSet. + terminationGracePeriodSeconds: 1 + + # -- By default we should always validate the installed CNPs before upgrading + # Cilium. This will make sure the user will have the policies deployed in the + # cluster with the right schema. + validateCNPs: true + +# -- Explicitly enable or disable priority class. +# .Capabilities.KubeVersion is unsettable in `helm template` calls, +# it depends on k8s libraries version that Helm was compiled against. +# This option allows to explicitly disable setting the priority class, which +# is useful for rendering charts for gke clusters in advance. +enableCriticalPriorityClass: true + +# disableEnvoyVersionCheck removes the check for Envoy, which can be useful +# on AArch64 as the images do not currently ship a version of Envoy. +#disableEnvoyVersionCheck: false + +clustermesh: + # -- Deploy clustermesh-apiserver for clustermesh + useAPIServer: false + # -- The maximum number of clusters to support in a ClusterMesh. This value + # cannot be changed on running clusters, and all clusters in a ClusterMesh + # must be configured with the same value. Values > 255 will decrease the + # maximum allocatable cluster-local identities. + # Supported values are 255 and 511. + maxConnectedClusters: 255 + + # -- Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) + annotations: {} + + # -- Clustermesh explicit configuration. + config: + # -- Enable the Clustermesh explicit configuration. + enabled: false + # -- Default dns domain for the Clustermesh API servers + # This is used in the case cluster addresses are not provided + # and IPs are used. + domain: mesh.cilium.io + # -- List of clusters to be peered in the mesh. + clusters: [] + # clusters: + # # -- Name of the cluster + # - name: cluster1 + # # -- Address of the cluster, use this if you created DNS records for + # # the cluster Clustermesh API server. + # address: cluster1.mesh.cilium.io + # # -- Port of the cluster Clustermesh API server. + # port: 2379 + # # -- IPs of the cluster Clustermesh API server, use multiple ones when + # # you have multiple IPs to access the Clustermesh API server. + # ips: + # - 172.18.255.201 + # # -- base64 encoded PEM values for the cluster client certificate, private key and certificate authority. + # # These fields can (and should) be omitted in case the CA is shared across clusters. In that case, the + # # "remote" private key and certificate available in the local cluster are automatically used instead. + # tls: + # cert: "" + # key: "" + # caCert: "" + + apiserver: + # -- Clustermesh API server image. + image: + override: ~ + repository: "quay.io/cilium/clustermesh-apiserver" + tag: "v1.15.3" + # clustermesh-apiserver-digest + digest: "sha256:da4573f8fe4415bdb786c4fdcbc3b518e5a485f930cd4292416eb80800cbd7fc" + useDigest: true + pullPolicy: "IfNotPresent" + + etcd: + # The etcd binary is included in the clustermesh API server image, so the same image from above is reused. + # Independent override isn't supported, because clustermesh-apiserver is tested against the etcd version it is + # built with. + + # -- Specifies the resources for etcd container in the apiserver + resources: {} + # requests: + # cpu: 200m + # memory: 256Mi + # limits: + # cpu: 1000m + # memory: 256Mi + + # -- Security context to be added to clustermesh-apiserver etcd containers + securityContext: {} + + # -- lifecycle setting for the etcd container + lifecycle: {} + + init: + # -- Specifies the resources for etcd init container in the apiserver + resources: {} + # requests: + # cpu: 100m + # memory: 100Mi + # limits: + # cpu: 100m + # memory: 100Mi + + # -- Additional arguments to `clustermesh-apiserver etcdinit`. + extraArgs: [] + + # -- Additional environment variables to `clustermesh-apiserver etcdinit`. + extraEnv: [] + + kvstoremesh: + # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved + # from the remote clusters in the local etcd instance. + enabled: false + + # -- Additional KVStoreMesh arguments. + extraArgs: [] + + # -- Additional KVStoreMesh environment variables. + extraEnv: [] + + # -- Resource requests and limits for the KVStoreMesh container + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Additional KVStoreMesh volumeMounts. + extraVolumeMounts: [] + + # -- KVStoreMesh Security context + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + # -- lifecycle setting for the KVStoreMesh container + lifecycle: {} + + service: + # -- The type of service used for apiserver access. + type: NodePort + # -- Optional port to use as the node port for apiserver access. + # + # WARNING: make sure to configure a different NodePort in each cluster if + # kube-proxy replacement is enabled, as Cilium is currently affected by a known + # bug (#24692) when NodePorts are handled by the KPR implementation. If a service + # with the same NodePort exists both in the local and the remote cluster, all + # traffic originating from inside the cluster and targeting the corresponding + # NodePort will be redirected to a local backend, regardless of whether the + # destination node belongs to the local or the remote cluster. + nodePort: 32379 + # -- Optional loadBalancer IP address to use with type LoadBalancer. + # loadBalancerIP: + + # -- Annotations for the clustermesh-apiserver + # For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" + # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 + annotations: {} + + # -- The externalTrafficPolicy of service used for apiserver access. + externalTrafficPolicy: + + # -- The internalTrafficPolicy of service used for apiserver access. + internalTrafficPolicy: + + # -- Number of replicas run for the clustermesh-apiserver deployment. + replicas: 1 + + # -- lifecycle setting for the apiserver container + lifecycle: {} + + # -- terminationGracePeriodSeconds for the clustermesh-apiserver deployment + terminationGracePeriodSeconds: 30 + + # -- Additional clustermesh-apiserver arguments. + extraArgs: [] + + # -- Additional clustermesh-apiserver environment variables. + extraEnv: [] + + # -- Additional clustermesh-apiserver volumes. + extraVolumes: [] + + # -- Additional clustermesh-apiserver volumeMounts. + extraVolumeMounts: [] + + # -- Security context to be added to clustermesh-apiserver containers + securityContext: {} + + # -- Security context to be added to clustermesh-apiserver pods + podSecurityContext: {} + + # -- Annotations to be added to clustermesh-apiserver pods + podAnnotations: {} + + # -- Labels to be added to clustermesh-apiserver pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- Resource requests and limits for the clustermesh-apiserver container of the clustermesh-apiserver deployment, such as + # resources: + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + # -- Resource requests and limits for the clustermesh-apiserver + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Affinity for clustermesh.apiserver + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: clustermesh-apiserver + + # -- Pod topology spread constraints for clustermesh-apiserver + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- clustermesh-apiserver update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- The priority class to use for clustermesh-apiserver + priorityClassName: "" + + tls: + # -- Configure the clustermesh authentication mode. + # Supported values: + # - legacy: All clusters access remote clustermesh instances with the same + # username (i.e., remote). The "remote" certificate must be + # generated with CN=remote if provided manually. + # - migration: Intermediate mode required to upgrade from legacy to cluster + # (and vice versa) with no disruption. Specifically, it enables + # the creation of the per-cluster usernames, while still using + # the common one for authentication. The "remote" certificate must + # be generated with CN=remote if provided manually (same as legacy). + # - cluster: Each cluster accesses remote etcd instances with a username + # depending on the local cluster name (i.e., remote-). + # The "remote" certificate must be generated with CN=remote- + # if provided manually. Cluster mode is meaningful only when the same + # CA is shared across all clusters part of the mesh. + authMode: legacy + + # -- Configure automatic TLS certificates generation. + # A Kubernetes CronJob is used the generate any + # certificates not provided by the user at installation + # time. + auto: + # -- When set to true, automatically generate a CA and certificates to + # enable mTLS between clustermesh-apiserver and external workload instances. + # If set to false, the certs to be provided by setting appropriate values below. + enabled: true + # Sets the method to auto-generate certificates. Supported values: + # - helm: This method uses Helm to generate all certificates. + # - cronJob: This method uses a Kubernetes CronJob the generate any + # certificates not provided by the user at installation + # time. + # - certmanager: This method use cert-manager to generate & rotate certificates. + method: helm + # -- Generated certificates validity duration in days. + certValidityDuration: 1095 + # -- Schedule for certificates regeneration (regardless of their expiration date). + # Only used if method is "cronJob". If nil, then no recurring job will be created. + # Instead, only the one-shot job is deployed to generate the certificates at + # installation time. + # + # Due to the out-of-band distribution of client certs to external workloads the + # CA is (re)regenerated only if it is not provided as a helm value and the k8s + # secret is manually deleted. + # + # Defaults to none. Commented syntax gives midnight of the first day of every + # fourth month. For syntax, see + # https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax + # schedule: "0 0 1 */4 *" + + # [Example] + # certManagerIssuerRef: + # group: cert-manager.io + # kind: ClusterIssuer + # name: ca-issuer + # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. + certManagerIssuerRef: {} + # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. + # Used if 'auto' is not enabled. + server: + cert: "" + key: "" + # -- Extra DNS names added to certificate when it's auto generated + extraDnsNames: [] + # -- Extra IP addresses added to certificate when it's auto generated + extraIpAddresses: [] + # -- base64 encoded PEM values for the clustermesh-apiserver admin certificate and private key. + # Used if 'auto' is not enabled. + admin: + cert: "" + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver client certificate and private key. + # Used if 'auto' is not enabled. + client: + cert: "" + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver remote cluster certificate and private key. + # Used if 'auto' is not enabled. + remote: + cert: "" + key: "" + + # clustermesh-apiserver Prometheus metrics configuration + metrics: + # -- Enables exporting apiserver metrics in OpenMetrics format. + enabled: true + # -- Configure the port the apiserver metric server listens on. + port: 9962 + + kvstoremesh: + # -- Enables exporting KVStoreMesh metrics in OpenMetrics format. + enabled: true + # -- Configure the port the KVStoreMesh metric server listens on. + port: 9964 + + etcd: + # -- Enables exporting etcd metrics in OpenMetrics format. + enabled: true + # -- Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. + mode: basic + # -- Configure the port the etcd metric server listens on. + port: 9963 + + serviceMonitor: + # -- Enable service monitor. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor clustermesh-apiserver + labels: {} + # -- Annotations to add to ServiceMonitor clustermesh-apiserver + annotations: {} + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + + # -- Interval for scrape metrics (apiserver metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + metricRelabelings: ~ + + kvstoremesh: + # -- Interval for scrape metrics (KVStoreMesh metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + metricRelabelings: ~ + + etcd: + # -- Interval for scrape metrics (etcd metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + metricRelabelings: ~ + +# -- Configure external workloads support +externalWorkloads: + # -- Enable support for external workloads, such as VMs (false by default). + enabled: false + +# -- Configure cgroup related configuration +cgroup: + autoMount: + # -- Enable auto mount of cgroup2 filesystem. + # When `autoMount` is enabled, cgroup2 filesystem is mounted at + # `cgroup.hostRoot` path on the underlying host and inside the cilium agent pod. + # If users disable `autoMount`, it's expected that users have mounted + # cgroup2 filesystem at the specified `cgroup.hostRoot` volume, and then the + # volume will be mounted inside the cilium agent pod at the same path. + enabled: true + # -- Init Container Cgroup Automount resource limits & requests + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # -- Configure cgroup root where cgroup2 filesystem is mounted on the host (see also: `cgroup.autoMount`) + hostRoot: /run/cilium/cgroupv2 + +# -- Configure whether to enable auto detect of terminating state for endpoints +# in order to support graceful termination. +enableK8sTerminatingEndpoint: true + +# -- Configure whether to unload DNS policy rules on graceful shutdown +# dnsPolicyUnloadOnShutdown: false + +# -- Configure the key of the taint indicating that Cilium is not ready on the node. +# When set to a value starting with `ignore-taint.cluster-autoscaler.kubernetes.io/`, the Cluster Autoscaler will ignore the taint on its decisions, allowing the cluster to scale up. +agentNotReadyTaintKey: "node.cilium.io/agent-not-ready" + +dnsProxy: + # -- DNS response code for rejecting DNS requests, available options are '[nameError refused]'. + dnsRejectResponseCode: refused + # -- Allow the DNS proxy to compress responses to endpoints that are larger than 512 Bytes or the EDNS0 option, if present. + enableDnsCompression: true + # -- Maximum number of IPs to maintain per FQDN name for each endpoint. + endpointMaxIpPerHostname: 50 + # -- Time during which idle but previously active connections with expired DNS lookups are still considered alive. + idleConnectionGracePeriod: 0s + # -- Maximum number of IPs to retain for expired DNS lookups with still-active connections. + maxDeferredConnectionDeletes: 10000 + # -- The minimum time, in seconds, to use DNS data for toFQDNs policies. If + # the upstream DNS server returns a DNS record with a shorter TTL, Cilium + # overwrites the TTL with this value. Setting this value to zero means that + # Cilium will honor the TTLs returned by the upstream DNS server. + minTtl: 0 + # -- DNS cache data at this path is preloaded on agent startup. + preCache: "" + # -- Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. + proxyPort: 0 + # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. + proxyResponseMaxDelay: 100ms + # -- DNS proxy operation mode (true/false, or unset to use version dependent defaults) + # enableTransparentMode: true + +# -- SCTP Configuration Values +sctp: + # -- Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. + enabled: false + +# Configuration for types of authentication for Cilium (beta) +authentication: + # -- Enable authentication processing and garbage collection. + # Note that if disabled, policy enforcement will still block requests that require authentication. + # But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. + enabled: true + # -- Buffer size of the channel Cilium uses to receive authentication events from the signal map. + queueSize: 1024 + # -- Buffer size of the channel Cilium uses to receive certificate expiration events from auth handlers. + rotatedIdentitiesQueueSize: 1024 + # -- Interval for garbage collection of auth map entries. + gcInterval: "5m0s" + # Configuration for Cilium's service-to-service mutual authentication using TLS handshakes. + # Note that this is not full mTLS support without also enabling encryption of some form. + # Current encryption options are Wireguard or IPSec, configured in encryption block above. + mutual: + # -- Port on the agent where mutual authentication handshakes between agents will be performed + port: 4250 + # -- Timeout for connecting to the remote node TCP socket + connectTimeout: 5s + # Settings for SPIRE + spire: + # -- Enable SPIRE integration (beta) + enabled: false + # -- Annotations to be added to all top-level spire objects (resources under templates/spire) + annotations: {} + # Settings to control the SPIRE installation and configuration + install: + # -- Enable SPIRE installation. + # This will only take effect only if authentication.mutual.spire.enabled is true + enabled: true + # -- SPIRE namespace to install into + namespace: cilium-spire + # -- SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. + existingNamespace: false + # -- init container image of SPIRE agent and server + initImage: + override: ~ + repository: "docker.io/library/busybox" + tag: "1.36.1" + digest: "sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b" + useDigest: true + pullPolicy: "IfNotPresent" + # SPIRE agent configuration + agent: + # -- SPIRE agent image + image: + override: ~ + repository: "ghcr.io/spiffe/spire-agent" + tag: "1.8.5" + digest: "sha256:99405637647968245ff9fe215f8bd2bd0ea9807be9725f8bf19fe1b21471e52b" + useDigest: true + pullPolicy: "IfNotPresent" + # -- SPIRE agent service account + serviceAccount: + create: true + name: spire-agent + # -- SPIRE agent annotations + annotations: {} + # -- SPIRE agent labels + labels: {} + # -- SPIRE Workload Attestor kubelet verification. + skipKubeletVerification: true + # -- SPIRE agent tolerations configuration + # By default it follows the same tolerations as the agent itself + # to allow the Cilium agent on this node to connect to SPIRE. + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" + # -- SPIRE agent affinity configuration + affinity: {} + # -- SPIRE agent nodeSelector configuration + # ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: {} + # -- Security context to be added to spire agent pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire agent containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} + server: + # -- SPIRE server image + image: + override: ~ + repository: "ghcr.io/spiffe/spire-server" + tag: "1.8.5" + digest: "sha256:28269265882048dcf0fed32fe47663cd98613727210b8d1a55618826f9bf5428" + useDigest: true + pullPolicy: "IfNotPresent" + # -- SPIRE server service account + serviceAccount: + create: true + name: spire-server + # -- SPIRE server init containers + initContainers: [] + # -- SPIRE server annotations + annotations: {} + # -- SPIRE server labels + labels: {} + # SPIRE server service configuration + service: + # -- Service type for the SPIRE server service + type: ClusterIP + # -- Annotations to be added to the SPIRE server service + annotations: {} + # -- Labels to be added to the SPIRE server service + labels: {} + # -- SPIRE server affinity configuration + affinity: {} + # -- SPIRE server nodeSelector configuration + # ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: {} + # -- SPIRE server tolerations configuration + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # SPIRE server datastorage configuration + dataStorage: + # -- Enable SPIRE server data storage + enabled: true + # -- Size of the SPIRE server data storage + size: 1Gi + # -- Access mode of the SPIRE server data storage + accessMode: ReadWriteOnce + # -- StorageClass of the SPIRE server data storage + storageClass: null + # -- Security context to be added to spire server pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire server containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} + # SPIRE CA configuration + ca: + # -- SPIRE CA key type + # AWS requires the use of RSA. EC cryptography is not supported + keyType: "rsa-4096" + # -- SPIRE CA Subject + subject: + country: "US" + organization: "SPIRE" + commonName: "Cilium SPIRE CA" + # -- SPIRE server address used by Cilium Operator + # + # If k8s Service DNS along with port number is used (e.g. ..svc(.*): format), + # Cilium Operator will resolve its address by looking up the clusterIP from Service resource. + # + # Example values: 10.0.0.1:8081, spire-server.cilium-spire.svc:8081 + serverAddress: ~ + # -- SPIFFE trust domain to use for fetching certificates + trustDomain: spiffe.cilium + # -- SPIRE socket path where the SPIRE delegated api agent is listening + adminSocketPath: /run/spire/sockets/admin.sock + # -- SPIRE socket path where the SPIRE workload agent is listening. + # Applies to both the Cilium Agent and Operator + agentSocketPath: /run/spire/sockets/agent/agent.sock + # -- SPIRE connection timeout + connectionTimeout: 30s diff --git a/magnum_cluster_api/charts/cilium/values.yaml.tmpl b/magnum_cluster_api/charts/cilium/values.yaml.tmpl new file mode 100644 index 00000000..c6df8efd --- /dev/null +++ b/magnum_cluster_api/charts/cilium/values.yaml.tmpl @@ -0,0 +1,3513 @@ + +# upgradeCompatibility helps users upgrading to ensure that the configMap for +# Cilium will not change critical values to ensure continued operation +# This flag is not required for new installations. +# For example: 1.7, 1.8, 1.9 +# upgradeCompatibility: '1.8' + +debug: + # -- Enable debug logging + enabled: false + # -- Configure verbosity levels for debug logging + # This option is used to enable debug messages for operations related to such + # sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is + # for enabling debug messages emitted per request, message and connection. + # Multiple values can be set via a space-separated string (e.g. "datapath envoy"). + # + # Applicable values: + # - flow + # - kvstore + # - envoy + # - datapath + # - policy + verbose: ~ + +rbac: + # -- Enable creation of Resource-Based Access Control configuration. + create: true + +# -- Configure image pull secrets for pulling container images +imagePullSecrets: +# - name: "image-pull-secret" + +# -- (string) Kubernetes config path +# @default -- `"~/.kube/config"` +kubeConfigPath: "" +# -- (string) Kubernetes service host +k8sServiceHost: "" +# -- (string) Kubernetes service port +k8sServicePort: "" + +# -- Configure the client side rate limit for the agent and operator +# +# If the amount of requests to the Kubernetes API server exceeds the configured +# rate limit, the agent and operator will start to throttle requests by delaying +# them until there is budget or the request times out. +k8sClientRateLimit: + # -- (int) The sustained request rate in requests per second. + # @default -- 5 for k8s up to 1.26. 10 for k8s version 1.27+ + qps: + # -- (int) The burst request rate in requests per second. + # The rate limiter will allow short bursts with a higher rate. + # @default -- 10 for k8s up to 1.26. 20 for k8s version 1.27+ + burst: + +cluster: + # -- Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. + name: default + # -- (int) Unique ID of the cluster. Must be unique across all connected + # clusters and in the range of 1 to 255. Only required for Cluster Mesh, + # may be 0 if Cluster Mesh is not used. + id: 0 + +# -- Define serviceAccount names for components. +# @default -- Component's fully qualified name. +serviceAccounts: + cilium: + create: true + name: cilium + automount: true + annotations: {} + nodeinit: + create: true + # -- Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. + # Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by + # this issue. Name and automount can be configured, if enabled is set to true. + # Otherwise, they are ignored. Enabled can be removed once the issue is fixed. + # Cilium-nodeinit DS must also be fixed. + enabled: false + name: cilium-nodeinit + automount: true + annotations: {} + envoy: + create: true + name: cilium-envoy + automount: true + annotations: {} + etcd: + create: true + name: cilium-etcd-operator + automount: true + annotations: {} + operator: + create: true + name: cilium-operator + automount: true + annotations: {} + preflight: + create: true + name: cilium-pre-flight + automount: true + annotations: {} + relay: + create: true + name: hubble-relay + automount: false + annotations: {} + ui: + create: true + name: hubble-ui + automount: true + annotations: {} + clustermeshApiserver: + create: true + name: clustermesh-apiserver + automount: true + annotations: {} + # -- Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob + clustermeshcertgen: + create: true + name: clustermesh-apiserver-generate-certs + automount: true + annotations: {} + # -- Hubblecertgen is used if hubble.tls.auto.method=cronJob + hubblecertgen: + create: true + name: hubble-generate-certs + automount: true + annotations: {} + +# -- Configure termination grace period for cilium-agent DaemonSet. +terminationGracePeriodSeconds: 1 + +# -- Install the cilium agent resources. +agent: true + +# -- Agent container name. +name: cilium + +# -- Roll out cilium agent pods automatically when configmap is updated. +rollOutCiliumPods: false + +# -- Agent container image. +image: + override: ~ + repository: "${CILIUM_REPO}" + tag: "${CILIUM_VERSION}" + pullPolicy: "${PULL_POLICY}" + # cilium-digest + digest: ${CILIUM_DIGEST} + useDigest: ${USE_DIGESTS} + +# -- Affinity for cilium-agent. +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + +# -- Node selector for cilium-agent. +nodeSelector: + kubernetes.io/os: linux + +# -- Node tolerations for agent scheduling to nodes with taints +# ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ +tolerations: +- operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + +# -- The priority class to use for cilium-agent. +priorityClassName: "" + +# -- DNS policy for Cilium agent pods. +# Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy +dnsPolicy: "" + +# -- Additional containers added to the cilium DaemonSet. +extraContainers: [] + +# -- Additional agent container arguments. +extraArgs: [] + +# -- Additional agent container environment variables. +extraEnv: [] + +# -- Additional agent hostPath mounts. +extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + +# -- Additional agent volumes. +extraVolumes: [] + +# -- Additional agent volumeMounts. +extraVolumeMounts: [] + +# -- extraConfig allows you to specify additional configuration parameters to be +# included in the cilium-config configmap. +extraConfig: {} +# my-config-a: "1234" +# my-config-b: |- +# test 1 +# test 2 +# test 3 + +# -- Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) +annotations: {} + +# -- Security Context for cilium-agent pods. +podSecurityContext: {} + +# -- Annotations to be added to agent pods +podAnnotations: {} + +# -- Labels to be added to agent pods +podLabels: {} + +# -- Agent resource limits & requests +# ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + +# -- resources & limits for the agent init containers +initResources: {} + +securityContext: + # -- User to run the pod with + # runAsUser: 0 + # -- Run the pod with elevated privileges + privileged: false + # -- SELinux options for the `cilium-agent` and init containers + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + # -- Capabilities for the `cilium-agent` container + ciliumAgent: + # Use to set socket permission + - CHOWN + # Used to terminate envoy child process + - KILL + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used since cilium creates raw sockets, etc... + - NET_RAW + # Used since cilium monitor uses mmap + - IPC_LOCK + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + # Allow discretionary access control (e.g. required for package installation) + - DAC_OVERRIDE + # Allow to set Access Control Lists (ACLs) on arbitrary files (e.g. required for package installation) + - FOWNER + # Allow to execute program that changes GID (e.g. required for package installation) + - SETGID + # Allow to execute program that changes UID (e.g. required for package installation) + - SETUID + # -- Capabilities for the `mount-cgroup` init container + mountCgroup: + # Only used for 'mount' cgroup + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # -- capabilities for the `apply-sysctl-overwrites` init container + applySysctlOverwrites: + # Required in order to access host's /etc/sysctl.d dir + - SYS_ADMIN + # Used for nsenter + - SYS_CHROOT + - SYS_PTRACE + # -- Capabilities for the `clean-cilium-state` init container + cleanCiliumState: + # Most of the capabilities here are the same ones used in the + # cilium-agent's container because this container can be used to + # uninstall all Cilium resources, and therefore it is likely that + # will need the same capabilities. + # Used since cilium modifies routing tables, etc... + - NET_ADMIN + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC + - SYS_RESOURCE + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + +# -- Cilium agent update strategy +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + +# Configuration Values for cilium-agent + +aksbyocni: + # -- Enable AKS BYOCNI integration. + # Note that this is incompatible with AKS clusters not created in BYOCNI mode: + # use Azure integration (`azure.enabled`) instead. + enabled: false + +# -- Enable installation of PodCIDR routes between worker +# nodes if worker nodes share a common L2 network segment. +autoDirectNodeRoutes: false + +# -- Annotate k8s node upon initialization with Cilium's metadata. +annotateK8sNode: false + +azure: + # -- Enable Azure integration. + # Note that this is incompatible with AKS clusters created in BYOCNI mode: use + # AKS BYOCNI integration (`aksbyocni.enabled`) instead. + enabled: false + # usePrimaryAddress: false + # resourceGroup: group1 + # subscriptionID: 00000000-0000-0000-0000-000000000000 + # tenantID: 00000000-0000-0000-0000-000000000000 + # clientID: 00000000-0000-0000-0000-000000000000 + # clientSecret: 00000000-0000-0000-0000-000000000000 + # userAssignedIdentityID: 00000000-0000-0000-0000-000000000000 + +alibabacloud: + # -- Enable AlibabaCloud ENI integration + enabled: false + +# -- Enable bandwidth manager to optimize TCP and UDP workloads and allow +# for rate-limiting traffic from individual Pods with EDT (Earliest Departure +# Time) through the "kubernetes.io/egress-bandwidth" Pod annotation. +bandwidthManager: + # -- Enable bandwidth manager infrastructure (also prerequirement for BBR) + enabled: false + # -- Activate BBR TCP congestion control for Pods + bbr: false + +# -- Configure standalone NAT46/NAT64 gateway +nat46x64Gateway: + # -- Enable RFC8215-prefixed translation + enabled: false + +# -- EnableHighScaleIPcache enables the special ipcache mode for high scale +# clusters. The ipcache content will be reduced to the strict minimum and +# traffic will be encapsulated to carry security identities. +highScaleIPcache: + # -- Enable the high scale mode for the ipcache. + enabled: false + +# -- Configure L2 announcements +l2announcements: + # -- Enable L2 announcements + enabled: false + # -- If a lease is not renewed for X duration, the current leader is considered dead, a new leader is picked + # leaseDuration: 15s + # -- The interval at which the leader will renew the lease + # leaseRenewDeadline: 5s + # -- The timeout between retries if renewal fails + # leaseRetryPeriod: 2s + +# -- Configure L2 pod announcements +l2podAnnouncements: + # -- Enable L2 pod announcements + enabled: false + # -- Interface used for sending Gratuitous ARP pod announcements + interface: "eth0" + +# -- Configure BGP +bgp: + # -- Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside + # cilium-agent and cilium-operator + enabled: false + announce: + # -- Enable allocation and announcement of service LoadBalancer IPs + loadbalancerIP: false + # -- Enable announcement of node pod CIDR + podCIDR: false + +# -- This feature set enables virtual BGP routers to be created via +# CiliumBGPPeeringPolicy CRDs. +bgpControlPlane: + # -- Enables the BGP control plane. + enabled: false + # -- SecretsNamespace is the namespace which BGP support will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for BGP secrets. + create: false + # -- The name of the secret namespace to which Cilium agents are given read access + name: kube-system + +pmtuDiscovery: + # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to + # the client. + enabled: false + +bpf: + autoMount: + # -- Enable automatic mount of BPF filesystem + # When `autoMount` is enabled, the BPF filesystem is mounted at + # `bpf.root` path on the underlying host and inside the cilium agent pod. + # If users disable `autoMount`, it's expected that users have mounted + # bpffs filesystem at the specified `bpf.root` volume, and then the + # volume will be mounted inside the cilium agent pod at the same path. + enabled: true + # -- Configure the mount point for the BPF filesystem + root: /sys/fs/bpf + + # -- Enables pre-allocation of eBPF map values. This increases + # memory usage but can reduce latency. + preallocateMaps: false + + # -- (int) Configure the maximum number of entries in auth map. + # @default -- `524288` + authMapMax: ~ + + # -- (int) Configure the maximum number of entries in the TCP connection tracking + # table. + # @default -- `524288` + ctTcpMax: ~ + + # -- (int) Configure the maximum number of entries for the non-TCP connection + # tracking table. + # @default -- `262144` + ctAnyMax: ~ + + # -- Configure the maximum number of service entries in the + # load balancer maps. + lbMapMax: 65536 + + # -- (int) Configure the maximum number of entries for the NAT table. + # @default -- `524288` + natMax: ~ + + # -- (int) Configure the maximum number of entries for the neighbor table. + # @default -- `524288` + neighMax: ~ + + # -- Configure the maximum number of entries in endpoint policy map (per endpoint). + policyMapMax: 16384 + + # -- (float64) Configure auto-sizing for all BPF maps based on available memory. + # ref: https://docs.cilium.io/en/stable/network/ebpf/maps/ + # @default -- `0.0025` + mapDynamicSizeRatio: ~ + + # -- Configure the level of aggregation for monitor notifications. + # Valid options are none, low, medium, maximum. + monitorAggregation: medium + + # -- Configure the typical time between monitor notifications for + # active connections. + monitorInterval: "5s" + + # -- Configure which TCP flags trigger notifications when seen for the + # first time in a connection. + monitorFlags: "all" + + # -- Allow cluster external access to ClusterIP services. + lbExternalClusterIP: false + + # -- (bool) Enable native IP masquerade support in eBPF + # @default -- `false` + masquerade: ~ + + # -- (bool) Configure whether direct routing mode should route traffic via + # host stack (true) or directly and more efficiently out of BPF (false) if + # the kernel supports it. The latter has the implication that it will also + # bypass netfilter in the host namespace. + # @default -- `false` + hostLegacyRouting: ~ + + # -- (bool) Configure the eBPF-based TPROXY to reduce reliance on iptables rules + # for implementing Layer 7 policy. + # @default -- `false` + tproxy: ~ + + # -- (list) Configure explicitly allowed VLAN id's for bpf logic bypass. + # [0] will allow all VLAN id's without any filtering. + # @default -- `[]` + vlanBypass: ~ + +# -- Enable BPF clock source probing for more efficient tick retrieval. +bpfClockProbe: false + +# -- Clean all eBPF datapath state from the initContainer of the cilium-agent +# DaemonSet. +# +# WARNING: Use with care! +cleanBpfState: false + +# -- Clean all local Cilium state from the initContainer of the cilium-agent +# DaemonSet. Implies cleanBpfState: true. +# +# WARNING: Use with care! +cleanState: false + +# -- Wait for KUBE-PROXY-CANARY iptables rule to appear in "wait-for-kube-proxy" +# init container before launching cilium-agent. +# More context can be found in the commit message of below PR +# https://github.com/cilium/cilium/pull/20123 +waitForKubeProxy: false + +cni: + # -- Install the CNI configuration and binary files into the filesystem. + install: true + + # -- Remove the CNI configuration and binary files on agent shutdown. Enable this + # if you're removing Cilium from the cluster. Disable this to prevent the CNI + # configuration file from being removed during agent upgrade, which can cause + # nodes to go unmanageable. + uninstall: false + + # -- Configure chaining on top of other CNI plugins. Possible values: + # - none + # - aws-cni + # - flannel + # - generic-veth + # - portmap + chainingMode: ~ + + # -- A CNI network name in to which the Cilium plugin should be added as a chained plugin. + # This will cause the agent to watch for a CNI network with this network name. When it is + # found, this will be used as the basis for Cilium's CNI configuration file. If this is + # set, it assumes a chaining mode of generic-veth. As a special case, a chaining mode + # of aws-cni implies a chainingTarget of aws-cni. + chainingTarget: ~ + + # -- Make Cilium take ownership over the `/etc/cni/net.d` directory on the + # node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. + # This ensures no Pods can be scheduled using other CNI plugins during Cilium + # agent downtime. + exclusive: true + + # -- Configure the log file for CNI logging with retention policy of 7 days. + # Disable CNI file logging by setting this field to empty explicitly. + logFile: /var/run/cilium/cilium-cni.log + + # -- Skip writing of the CNI configuration. This can be used if + # writing of the CNI configuration is performed by external automation. + customConf: false + + # -- Configure the path to the CNI configuration directory on the host. + confPath: /etc/cni/net.d + + # -- Configure the path to the CNI binary directory on the host. + binPath: /opt/cni/bin + + # -- Specify the path to a CNI config to read from on agent start. + # This can be useful if you want to manage your CNI + # configuration outside of a Kubernetes environment. This parameter is + # mutually exclusive with the 'cni.configMap' parameter. The agent will + # write this to 05-cilium.conflist on startup. + # readCniConf: /host/etc/cni/net.d/05-sample.conflist.input + + # -- When defined, configMap will mount the provided value as ConfigMap and + # interpret the cniConf variable as CNI configuration file and write it + # when the agent starts up + # configMap: cni-configuration + + # -- Configure the key in the CNI ConfigMap to read the contents of + # the CNI configuration from. + configMapKey: cni-config + + # -- Configure the path to where to mount the ConfigMap inside the agent pod. + confFileMountPath: /tmp/cni-configuration + + # -- Configure the path to where the CNI configuration directory is mounted + # inside the agent pod. + hostConfDirMountPath: /host/etc/cni/net.d + + # -- Specifies the resources for the cni initContainer + resources: + requests: + cpu: 100m + memory: 10Mi + +# -- (string) Configure how frequently garbage collection should occur for the datapath +# connection tracking table. +# @default -- `"0s"` +conntrackGCInterval: "" + +# -- (string) Configure the maximum frequency for the garbage collection of the +# connection tracking table. Only affects the automatic computation for the frequency +# and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently +# clean up unused identities created from ToFQDN policies. +conntrackGCMaxInterval: "" + +# -- Configure container runtime specific integration. +# Deprecated in favor of bpf.autoMount.enabled. To be removed in 1.15. +containerRuntime: + # -- Enables specific integrations for container runtimes. + # Supported values: + # - crio + # - none + integration: none + +# -- (string) Configure timeout in which Cilium will exit if CRDs are not available +# @default -- `"5m"` +crdWaitTimeout: "" + +# -- Tail call hooks for custom eBPF programs. +customCalls: + # -- Enable tail call hooks for custom eBPF programs. + enabled: false + +daemon: + # -- Configure where Cilium runtime state should be stored. + runPath: "/var/run/cilium" + + # -- Configure a custom list of possible configuration override sources + # The default is "config-map:cilium-config,cilium-node-config". For supported + # values, see the help text for the build-config subcommand. + # Note that this value should be a comma-separated string. + configSources: ~ + + # -- allowedConfigOverrides is a list of config-map keys that can be overridden. + # That is to say, if this value is set, config sources (excepting the first one) can + # only override keys in this list. + # + # This takes precedence over blockedConfigOverrides. + # + # By default, all keys may be overridden. To disable overrides, set this to "none" or + # change the configSources variable. + allowedConfigOverrides: ~ + + # -- blockedConfigOverrides is a list of config-map keys that may not be overridden. + # In other words, if any of these keys appear in a configuration source excepting the + # first one, they will be ignored + # + # This is ignored if allowedConfigOverrides is set. + # + # By default, all keys may be overridden. + blockedConfigOverrides: ~ + +# -- Specify which network interfaces can run the eBPF datapath. This means +# that a packet sent from a pod to a destination outside the cluster will be +# masqueraded (to an output device IPv4 address), if the output device runs the +# program. When not specified, probing will automatically detect devices that have +# a non-local route. This should be used only when autodetection is not suitable. +# devices: "" + +# -- Enables experimental support for the detection of new and removed datapath +# devices. When devices change the eBPF datapath is reloaded and services updated. +# If "devices" is set then only those devices, or devices matching a wildcard will +# be considered. +enableRuntimeDeviceDetection: false + +# -- Chains to ignore when installing feeder rules. +# disableIptablesFeederRules: "" + +# -- Limit iptables-based egress masquerading to interface selector. +# egressMasqueradeInterfaces: "" + +# -- Enable setting identity mark for local traffic. +# enableIdentityMark: true + +# -- Enable Kubernetes EndpointSlice feature in Cilium if the cluster supports it. +# enableK8sEndpointSlice: true + +# -- Enable CiliumEndpointSlice feature. +enableCiliumEndpointSlice: false + +envoyConfig: + # -- Enable CiliumEnvoyConfig CRD + # CiliumEnvoyConfig CRD can also be implicitly enabled by other options. + enabled: false + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for CiliumEnvoyConfig CRDs. + create: true + + # -- The name of the secret namespace to which Cilium agents are given read access. + name: cilium-secrets + +ingressController: + # -- Enable cilium ingress controller + # This will automatically set enable-envoy-config as well. + enabled: false + + # -- Set cilium ingress controller to be the default ingress controller + # This will let cilium ingress controller route entries without ingress class set + default: false + + # -- Default ingress load balancer mode + # Supported values: shared, dedicated + # For granular control, use the following annotations on the ingress resource + # ingress.cilium.io/loadbalancer-mode: shared|dedicated, + loadbalancerMode: dedicated + + # -- Enforce https for host having matching TLS host in Ingress. + # Incoming traffic to http listener will return 308 http error code with respective location in header. + enforceHttps: true + + # -- Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. + enableProxyProtocol: false + + # -- IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service + ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] + + # -- Default secret namespace for ingresses without .spec.tls[].secretName set. + defaultSecretNamespace: + + # -- Default secret name for ingresses without .spec.tls[].secretName set. + defaultSecretName: + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. + secretsNamespace: + # -- Create secrets namespace for Ingress. + create: true + + # -- Name of Ingress secret namespace. + name: cilium-secrets + + # -- Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. + # If disabled, TLS secrets must be maintained externally. + sync: true + + # -- Load-balancer service in shared mode. + # This is a single load-balancer service for all Ingress resources. + service: + # -- Service name + name: cilium-ingress + # -- Labels to be added for the shared LB service + labels: {} + # -- Annotations to be added for the shared LB service + annotations: {} + # -- Service type for the shared LB service + type: LoadBalancer + # -- Configure a specific nodePort for insecure HTTP traffic on the shared LB service + insecureNodePort: ~ + # -- Configure a specific nodePort for secure HTTPS traffic on the shared LB service + secureNodePort : ~ + # -- Configure a specific loadBalancerClass on the shared LB service (requires Kubernetes 1.24+) + loadBalancerClass: ~ + # -- Configure a specific loadBalancerIP on the shared LB service + loadBalancerIP : ~ + # -- Configure if node port allocation is required for LB service + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation + allocateLoadBalancerNodePorts: ~ + +gatewayAPI: + # -- Enable support for Gateway API in cilium + # This will automatically set enable-envoy-config as well. + enabled: false + + # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. + secretsNamespace: + # -- Create secrets namespace for Gateway API. + create: true + + # -- Name of Gateway API secret namespace. + name: cilium-secrets + + # -- Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. + # If disabled, TLS secrets must be maintained externally. + sync: true + +# -- Enables the fallback compatibility solution for when the xt_socket kernel +# module is missing and it is needed for the datapath L7 redirection to work +# properly. See documentation for details on when this can be disabled: +# https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. +enableXTSocketFallback: true + +encryption: + # -- Enable transparent network encryption. + enabled: false + + # -- Encryption method. Can be either ipsec or wireguard. + type: ipsec + + # -- Enable encryption for pure node to node traffic. + # This option is only effective when encryption.type is set to "wireguard". + nodeEncryption: false + + # -- Configure the WireGuard Pod2Pod strict mode. + strictMode: + # -- Enable WireGuard Pod2Pod strict mode. + enabled: false + + # -- CIDR for the WireGuard Pod2Pod strict mode. + cidr: "" + + # -- Allow dynamic lookup of remote node identities. + # This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. + allowRemoteNodeIdentities: false + + ipsec: + # -- Name of the key file inside the Kubernetes secret configured via secretName. + keyFile: "" + + # -- Path to mount the secret inside the Cilium pod. + mountPath: "" + + # -- Name of the Kubernetes secret containing the encryption keys. + secretName: "" + + # -- The interface to use for encrypted traffic. + interface: "" + + # -- Enable the key watcher. If disabled, a restart of the agent will be + # necessary on key rotations. + keyWatcher: true + + # -- Maximum duration of the IPsec key rotation. The previous key will be + # removed after that delay. + keyRotationDuration: "5m" + + wireguard: + # -- Enables the fallback to the user-space implementation. + userspaceFallback: false + # -- Controls Wireguard PersistentKeepalive option. Set 0s to disable. + persistentKeepalive: 0s + + # -- Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. + # Name of the key file inside the Kubernetes secret configured via secretName. + # This option is only effective when encryption.type is set to ipsec. + keyFile: keys + + # -- Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. + # Path to mount the secret inside the Cilium pod. + # This option is only effective when encryption.type is set to ipsec. + mountPath: /etc/ipsec + + # -- Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. + # Name of the Kubernetes secret containing the encryption keys. + # This option is only effective when encryption.type is set to ipsec. + secretName: cilium-ipsec-keys + + # -- Deprecated in favor of encryption.ipsec.interface. To be removed in 1.15. + # The interface to use for encrypted traffic. + # This option is only effective when encryption.type is set to ipsec. + interface: "" + +endpointHealthChecking: + # -- Enable connectivity health checking between virtual endpoints. + enabled: true + +# -- Enable endpoint status. +# Status can be: policy, health, controllers, log and / or state. For 2 or more options use a space. +endpointStatus: + enabled: false + status: "" + +endpointRoutes: + # -- Enable use of per endpoint routes instead of routing via + # the cilium_host interface. + enabled: false + +k8sNetworkPolicy: + # -- Enable support for K8s NetworkPolicy + enabled: true + +eni: + # -- Enable Elastic Network Interface (ENI) integration. + enabled: false + # -- Update ENI Adapter limits from the EC2 API + updateEC2AdapterLimitViaAPI: true + # -- Release IPs not used from the ENI + awsReleaseExcessIPs: false + # -- Enable ENI prefix delegation + awsEnablePrefixDelegation: false + # -- EC2 API endpoint to use + ec2APIEndpoint: "" + # -- Tags to apply to the newly created ENIs + eniTags: {} + # -- Interval for garbage collection of unattached ENIs. Set to "0s" to disable. + # @default -- `"5m"` + gcInterval: "" + # -- Additional tags attached to ENIs created by Cilium. + # Dangling ENIs with this tag will be garbage collected + # @default -- `{"io.cilium/cilium-managed":"true,"io.cilium/cluster-name":""}` + gcTags: {} + # -- If using IAM role for Service Accounts will not try to + # inject identity values from cilium-aws kubernetes secret. + # Adds annotation to service account if managed by Helm. + # See https://github.com/aws/amazon-eks-pod-identity-webhook + iamRole: "" + # -- Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. + subnetIDsFilter: [] + # -- Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. + subnetTagsFilter: [] + # -- Filter via AWS EC2 Instance tags (k=v) which will dictate which AWS EC2 Instances + # are going to be used to create new ENIs + instanceTagsFilter: [] + +externalIPs: + # -- Enable ExternalIPs service support. + enabled: false + +# fragmentTracking enables IPv4 fragment tracking support in the datapath. +# fragmentTracking: true + +gke: + # -- Enable Google Kubernetes Engine integration + enabled: false + +# -- Enable connectivity health checking. +healthChecking: true + +# -- TCP port for the agent health API. This is not the port for cilium-health. +healthPort: 9879 + +# -- Configure the host firewall. +hostFirewall: + # -- Enables the enforcement of host policies in the eBPF datapath. + enabled: false + +hostPort: + # -- Enable hostPort service support. + enabled: false + +# -- Configure socket LB +socketLB: + # -- Enable socket LB + enabled: false + + # -- Disable socket lb for non-root ns. This is used to enable Istio routing rules. + # hostNamespaceOnly: false + +# -- Configure certificate generation for Hubble integration. +# If hubble.tls.auto.method=cronJob, these values are used +# for the Kubernetes CronJob which will be scheduled regularly to +# (re)generate any certificates not provided manually. +certgen: + image: + override: ~ + repository: "${CERTGEN_REPO}" + tag: "${CERTGEN_VERSION}" + digest: "${CERTGEN_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + # -- Seconds after which the completed job pod will be deleted + ttlSecondsAfterFinished: 1800 + # -- Labels to be added to hubble-certgen pods + podLabels: {} + # -- Annotations to be added to the hubble-certgen initial Job and CronJob + annotations: + job: {} + cronJob: {} + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- Additional certgen volumes. + extraVolumes: [] + + # -- Additional certgen volumeMounts. + extraVolumeMounts: [] + + # -- Affinity for certgen + affinity: {} + +hubble: + # -- Enable Hubble (true by default). + enabled: true + + # -- Annotations to be added to all top-level hubble objects (resources under templates/hubble) + annotations: {} + + # -- Buffer size of the channel Hubble uses to receive monitor events. If this + # value is not set, the queue size is set to the default monitor queue size. + # eventQueueSize: "" + + # -- Number of recent flows for Hubble to cache. Defaults to 4095. + # Possible values are: + # 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, + # 2047, 4095, 8191, 16383, 32767, 65535 + # eventBufferCapacity: "4095" + + # -- Hubble metrics configuration. + # See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics + # for more comprehensive documentation about Hubble metrics. + metrics: + # -- Configures the list of metrics to collect. If empty or null, metrics + # are disabled. + # Example: + # + # enabled: + # - dns:query;ignoreAAAA + # - drop + # - tcp + # - flow + # - icmp + # - http + # + # You can specify the list of metrics from the helm CLI: + # + # --set hubble.metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" + # + enabled: ~ + # -- Enables exporting hubble metrics in OpenMetrics format. + enableOpenMetrics: false + # -- Configure the port the hubble metric server listens on. + port: 9965 + # -- Annotations to be added to hubble-metrics service. + serviceAnnotations: {} + serviceMonitor: + # -- Create ServiceMonitor resources for Prometheus Operator. + # This requires the prometheus CRDs to be available. + # ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor hubble + labels: {} + # -- Annotations to add to ServiceMonitor hubble + annotations: {} + # -- jobLabel to add for ServiceMonitor hubble + jobLabel: "" + # -- Interval for scrape metrics. + interval: "10s" + # -- Relabeling configs for the ServiceMonitor hubble + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor hubble + metricRelabelings: ~ + # -- Grafana dashboards for hubble + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards + dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + + # -- Unix domain socket path to listen to when Hubble is enabled. + socketPath: /var/run/cilium/hubble.sock + + # -- Enables redacting sensitive information present in Layer 7 flows. + redact: + enabled: false + http: + # -- Enables redacting URL query (GET) parameters. + # Example: + # + # redact: + # enabled: true + # http: + # urlQuery: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.urlQuery="true" + urlQuery: false + # -- Enables redacting user info, e.g., password when basic auth is used. + # Example: + # + # redact: + # enabled: true + # http: + # userInfo: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.userInfo="true" + userInfo: true + headers: + # -- List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # allow: + # - traceparent + # - tracestate + # - Cache-Control + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" + allow: [] + # -- List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # deny: + # - Authorization + # - Proxy-Authorization + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" + deny: [] + kafka: + # -- Enables redacting Kafka's API key. + # Example: + # + # redact: + # enabled: true + # kafka: + # apiKey: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.kafka.apiKey="true" + apiKey: false + + # -- An additional address for Hubble to listen to. + # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that + # Hubble is listening on port 4244. + listenAddress: ":4244" + # -- Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. + preferIpv6: false + # -- (bool) Skip Hubble events with unknown cgroup ids + # @default -- `true` + skipUnknownCGroupIDs: ~ + + peerService: + # -- Service Port for the Peer service. + # If not set, it is dynamically assigned to port 443 if TLS is enabled and to + # port 80 if not. + # servicePort: 80 + # -- Target Port for the Peer service, must match the hubble.listenAddress' + # port. + targetPort: 4244 + # -- The cluster domain to use to query the Hubble Peer service. It should + # be the local cluster. + clusterDomain: cluster.local + # -- TLS configuration for Hubble + tls: + # -- Enable mutual TLS for listenAddress. Setting this value to false is + # highly discouraged as the Hubble API provides access to potentially + # sensitive network flow metadata and is exposed on the host network. + enabled: true + # -- Configure automatic TLS certificates generation. + auto: + # -- Auto-generate certificates. + # When set to true, automatically generate a CA and certificates to + # enable mTLS between Hubble server and Hubble Relay instances. If set to + # false, the certs for Hubble server need to be provided by setting + # appropriate values below. + enabled: true + # -- Set the method to auto-generate certificates. Supported values: + # - helm: This method uses Helm to generate all certificates. + # - cronJob: This method uses a Kubernetes CronJob the generate any + # certificates not provided by the user at installation + # time. + # - certmanager: This method use cert-manager to generate & rotate certificates. + method: helm + # -- Generated certificates validity duration in days. + certValidityDuration: 1095 + # -- Schedule for certificates regeneration (regardless of their expiration date). + # Only used if method is "cronJob". If nil, then no recurring job will be created. + # Instead, only the one-shot job is deployed to generate the certificates at + # installation time. + # + # Defaults to midnight of the first day of every fourth month. For syntax, see + # https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax + schedule: "0 0 1 */4 *" + + # [Example] + # certManagerIssuerRef: + # group: cert-manager.io + # kind: ClusterIssuer + # name: ca-issuer + # -- certmanager issuer used when hubble.tls.auto.method=certmanager. + certManagerIssuerRef: {} + + # -- base64 encoded PEM values for the Hubble server certificate and private key + server: + cert: "" + key: "" + # -- Extra DNS names added to certificate when it's auto generated + extraDnsNames: [] + # -- Extra IP addresses added to certificate when it's auto generated + extraIpAddresses: [] + + relay: + # -- Enable Hubble Relay (requires hubble.enabled=true) + enabled: false + + # -- Roll out Hubble Relay pods automatically when configmap is updated. + rollOutPods: false + + # -- Hubble-relay container image. + image: + override: ~ + repository: "${HUBBLE_RELAY_REPO}" + tag: "${CILIUM_VERSION}" + # hubble-relay-digest + digest: ${HUBBLE_RELAY_DIGEST} + useDigest: ${USE_DIGESTS} + pullPolicy: "${PULL_POLICY}" + + # -- Specifies the resources for the hubble-relay pods + resources: {} + + # -- Number of replicas run for the hubble-relay deployment. + replicas: 1 + + # -- Affinity for hubble-replay + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + + # -- Pod topology spread constraints for hubble-relay + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- Additional hubble-relay environment variables. + extraEnv: [] + + # -- Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) + annotations: {} + + # -- Annotations to be added to hubble-relay pods + podAnnotations: {} + + # -- Labels to be added to hubble-relay pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- The priority class to use for hubble-relay + priorityClassName: "" + + # -- Configure termination grace period for hubble relay Deployment. + terminationGracePeriodSeconds: 1 + + # -- hubble-relay update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- Additional hubble-relay volumes. + extraVolumes: [] + + # -- Additional hubble-relay volumeMounts. + extraVolumeMounts: [] + + # -- hubble-relay pod security context + podSecurityContext: + fsGroup: 65532 + + # -- hubble-relay container security context + securityContext: + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + + # -- hubble-relay service configuration. + service: + # --- The type of service used for Hubble Relay access, either ClusterIP or NodePort. + type: ClusterIP + # --- The port to use when the service type is set to NodePort. + nodePort: 31234 + + # -- Host to listen to. Specify an empty string to bind to all the interfaces. + listenHost: "" + + # -- Port to listen to. + listenPort: "4245" + + # -- TLS configuration for Hubble Relay + tls: + # -- base64 encoded PEM values for the hubble-relay client certificate and private key + # This keypair is presented to Hubble server instances for mTLS + # authentication and is required when hubble.tls.enabled is true. + # These values need to be set manually if hubble.tls.auto.enabled is false. + client: + cert: "" + key: "" + # -- base64 encoded PEM values for the hubble-relay server certificate and private key + server: + # When set to true, enable TLS on for Hubble Relay server + # (ie: for clients connecting to the Hubble Relay API). + enabled: false + # When set to true enforces mutual TLS between Hubble Relay server and its clients. + # False allow non-mutual TLS connections. + # This option has no effect when TLS is disabled. + mtls: false + # These values need to be set manually if hubble.tls.auto.enabled is false. + cert: "" + key: "" + # -- extra DNS names added to certificate when its auto gen + extraDnsNames: [] + # -- extra IP addresses added to certificate when its auto gen + extraIpAddresses: [] + # DNS name used by the backend to connect to the relay + # This is a simple workaround as the relay certificates are currently hardcoded to + # *.hubble-relay.cilium.io + # See https://github.com/cilium/cilium/pull/28709#discussion_r1371792546 + # For GKE Dataplane V2 this should be set to relay.kube-system.svc.cluster.local + relayName: "ui.hubble-relay.cilium.io" + + # -- Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). + dialTimeout: ~ + + # -- Backoff duration to retry connecting to the local hubble instance in case of failure (e.g. "30s"). + retryTimeout: ~ + + # -- Max number of flows that can be buffered for sorting before being sent to the + # client (per request) (e.g. 100). + sortBufferLenMax: ~ + + # -- When the per-request flows sort buffer is not full, a flow is drained every + # time this timeout is reached (only affects requests in follow-mode) (e.g. "1s"). + sortBufferDrainTimeout: ~ + + # -- Port to use for the k8s service backed by hubble-relay pods. + # If not set, it is dynamically assigned to port 443 if TLS is enabled and to + # port 80 if not. + # servicePort: 80 + + # -- Enable prometheus metrics for hubble-relay on the configured port at + # /metrics + prometheus: + enabled: false + port: 9966 + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor hubble-relay + labels: {} + # -- Annotations to add to ServiceMonitor hubble-relay + annotations: {} + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor hubble-relay + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor hubble-relay + metricRelabelings: ~ + + gops: + # -- Enable gops for hubble-relay + enabled: true + # -- Configure gops listen port for hubble-relay + port: 9893 + + pprof: + # -- Enable pprof for hubble-relay + enabled: false + # -- Configure pprof listen address for hubble-relay + address: localhost + # -- Configure pprof listen port for hubble-relay + port: 6062 + + ui: + # -- Whether to enable the Hubble UI. + enabled: false + + standalone: + # -- When true, it will allow installing the Hubble UI only, without checking dependencies. + # It is useful if a cluster already has cilium and Hubble relay installed and you just + # want Hubble UI to be deployed. + # When installed via helm, installing UI should be done via `helm upgrade` and when installed via the cilium cli, then `cilium hubble enable --ui` + enabled: false + + tls: + # -- When deploying Hubble UI in standalone, with tls enabled for Hubble relay, it is required + # to provide a volume for mounting the client certificates. + certsVolume: {} + # projected: + # defaultMode: 0400 + # sources: + # - secret: + # name: hubble-ui-client-certs + # items: + # - key: tls.crt + # path: client.crt + # - key: tls.key + # path: client.key + # - key: ca.crt + # path: hubble-relay-ca.crt + + # -- Roll out Hubble-ui pods automatically when configmap is updated. + rollOutPods: false + + tls: + # -- base64 encoded PEM values used to connect to hubble-relay + # This keypair is presented to Hubble Relay instances for mTLS + # authentication and is required when hubble.relay.tls.server.enabled is true. + # These values need to be set manually if hubble.tls.auto.enabled is false. + client: + cert: "" + key: "" + + backend: + # -- Hubble-ui backend image. + image: + override: ~ + repository: "${HUBBLE_UI_BACKEND_REPO}" + tag: "${HUBBLE_UI_BACKEND_VERSION}" + digest: "${HUBBLE_UI_BACKEND_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + + # -- Hubble-ui backend security context. + securityContext: {} + + # -- Additional hubble-ui backend environment variables. + extraEnv: [] + + # -- Additional hubble-ui backend volumes. + extraVolumes: [] + + # -- Additional hubble-ui backend volumeMounts. + extraVolumeMounts: [] + + livenessProbe: + # -- Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + + readinessProbe: + # -- Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + + # -- Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. + resources: {} + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + + frontend: + # -- Hubble-ui frontend image. + image: + override: ~ + repository: "${HUBBLE_UI_FRONTEND_REPO}" + tag: "${HUBBLE_UI_FRONTEND_VERSION}" + digest: "${HUBBLE_UI_FRONTEND_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + + # -- Hubble-ui frontend security context. + securityContext: {} + + # -- Additional hubble-ui frontend environment variables. + extraEnv: [] + + # -- Additional hubble-ui frontend volumes. + extraVolumes: [] + + # -- Additional hubble-ui frontend volumeMounts. + extraVolumeMounts: [] + + # -- Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. + resources: {} + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + server: + # -- Controls server listener for ipv6 + ipv6: + enabled: true + + # -- The number of replicas of Hubble UI to deploy. + replicas: 1 + + # -- Annotations to be added to all top-level hubble-ui objects (resources under templates/hubble-ui) + annotations: {} + + # -- Annotations to be added to hubble-ui pods + podAnnotations: {} + + # -- Labels to be added to hubble-ui pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- Affinity for hubble-ui + affinity: {} + + # -- Pod topology spread constraints for hubble-ui + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- The priority class to use for hubble-ui + priorityClassName: "" + + # -- hubble-ui update strategy. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- Security context to be added to Hubble UI pods + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + + # -- hubble-ui service configuration. + service: + # -- Annotations to be added for the Hubble UI service + annotations: {} + # --- The type of service used for Hubble UI access, either ClusterIP or NodePort. + type: ClusterIP + # --- The port to use when the service type is set to NodePort. + nodePort: 31235 + + # -- Defines base url prefix for all hubble-ui http requests. + # It needs to be changed in case if ingress for hubble-ui is configured under some sub-path. + # Trailing `/` is required for custom path, ex. `/service-map/` + baseUrl: "/" + + # -- hubble-ui ingress configuration. + ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + className: "" + hosts: + - chart-example.local + labels: {} + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + # -- Hubble flows export. + export: + # --- Defines max file size of output file before it gets rotated. + fileMaxSizeMb: 10 + # --- Defines max number of backup/rotated files. + fileMaxBackups: 5 + # --- Static exporter configuration. + # Static exporter is bound to agent lifecycle. + static: + enabled: false + filePath: /var/run/cilium/hubble/events.log + fieldMask: [] + # - time + # - source + # - destination + # - verdict + allowList: [] + # - '{"verdict":["DROPPED","ERROR"]}' + denyList: [] + # - '{"source_pod":["kube-system/"]}' + # - '{"destination_pod":["kube-system/"]}' + # --- Dynamic exporters configuration. + # Dynamic exporters may be reconfigured without a need of agent restarts. + dynamic: + enabled: false + config: + # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. + configMapName: cilium-flowlog-config + # ---- True if helm installer should create config map. + # Switch to false if you want to self maintain the file content. + createConfigMap: true + # ---- Exporters configuration in YAML format. + content: + - name: all + fieldMask: [] + includeFilters: [] + excludeFilters: [] + filePath: "/var/run/cilium/hubble/events.log" + #- name: "test002" + # filePath: "/var/log/network/flow-log/pa/test002.log" + # fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"] + # includeFilters: + # - source_pod: ["default/"] + # event_type: + # - type: 1 + # - destination_pod: ["frontend/nginx-975996d4c-7hhgt"] + # excludeFilters: [] + # end: "2023-10-09T23:59:59-07:00" + +# -- Method to use for identity allocation (`crd` or `kvstore`). +identityAllocationMode: "crd" + +# -- (string) Time to wait before using new identity on endpoint identity change. +# @default -- `"5s"` +identityChangeGracePeriod: "" + +# -- Install Iptables rules to skip netfilter connection tracking on all pod +# traffic. This option is only effective when Cilium is running in direct +# routing and full KPR mode. Moreover, this option cannot be enabled when Cilium +# is running in a managed Kubernetes environment or in a chained CNI setup. +installNoConntrackIptablesRules: false + +ipam: + # -- Configure IP Address Management mode. + # ref: https://docs.cilium.io/en/stable/network/concepts/ipam/ + mode: "cluster-pool" + # -- Maximum rate at which the CiliumNode custom resource is updated. + ciliumNodeUpdateRate: "15s" + operator: + # -- IPv4 CIDR list range to delegate to individual nodes for IPAM. + clusterPoolIPv4PodCIDRList: ["10.0.0.0/8"] + # -- IPv4 CIDR mask size to delegate to individual nodes for IPAM. + clusterPoolIPv4MaskSize: 24 + # -- IPv6 CIDR list range to delegate to individual nodes for IPAM. + clusterPoolIPv6PodCIDRList: ["fd00::/104"] + # -- IPv6 CIDR mask size to delegate to individual nodes for IPAM. + clusterPoolIPv6MaskSize: 120 + # -- IP pools to auto-create in multi-pool IPAM mode. + autoCreateCiliumPodIPPools: {} + # default: + # ipv4: + # cidrs: + # - 10.10.0.0/8 + # maskSize: 24 + # other: + # ipv6: + # cidrs: + # - fd00:100::/80 + # maskSize: 96 + # -- The maximum burst size when rate limiting access to external APIs. + # Also known as the token bucket capacity. + # @default -- `20` + externalAPILimitBurstSize: ~ + # -- The maximum queries per second when rate limiting access to + # external APIs. Also known as the bucket refill rate, which is used to + # refill the bucket up to the burst size capacity. + # @default -- `4.0` + externalAPILimitQPS: ~ + +# -- The api-rate-limit option can be used to overwrite individual settings of the default configuration for rate limiting calls to the Cilium Agent API +apiRateLimit: ~ + +# -- Configure the eBPF-based ip-masq-agent +ipMasqAgent: + enabled: false +# the config of nonMasqueradeCIDRs +# config: + # nonMasqueradeCIDRs: [] + # masqLinkLocal: false + # masqLinkLocalIPv6: false + +# iptablesLockTimeout defines the iptables "--wait" option when invoked from Cilium. +# iptablesLockTimeout: "5s" + +ipv4: + # -- Enable IPv4 support. + enabled: true + +ipv6: + # -- Enable IPv6 support. + enabled: false + +# -- Configure Kubernetes specific configuration +k8s: {} + # -- requireIPv4PodCIDR enables waiting for Kubernetes to provide the PodCIDR + # range via the Kubernetes node resource + # requireIPv4PodCIDR: false + + # -- requireIPv6PodCIDR enables waiting for Kubernetes to provide the PodCIDR + # range via the Kubernetes node resource + # requireIPv6PodCIDR: false + +# -- Keep the deprecated selector labels when deploying Cilium DaemonSet. +keepDeprecatedLabels: false + +# -- Keep the deprecated probes when deploying Cilium DaemonSet +keepDeprecatedProbes: false + +startupProbe: + # -- failure threshold of startup probe. + # 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) + failureThreshold: 105 + # -- interval between checks of the startup probe + periodSeconds: 2 +livenessProbe: + # -- failure threshold of liveness probe + failureThreshold: 10 + # -- interval between checks of the liveness probe + periodSeconds: 30 +readinessProbe: + # -- failure threshold of readiness probe + failureThreshold: 3 + # -- interval between checks of the readiness probe + periodSeconds: 30 + +# -- Configure the kube-proxy replacement in Cilium BPF datapath +# Valid options are "true", "false", "disabled" (deprecated), "partial" (deprecated), "strict" (deprecated). +# ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ +#kubeProxyReplacement: "false" + +# -- healthz server bind address for the kube-proxy replacement. +# To enable set the value to '0.0.0.0:10256' for all ipv4 +# addresses and this '[::]:10256' for all ipv6 addresses. +# By default it is disabled. +kubeProxyReplacementHealthzBindAddr: "" + +l2NeighDiscovery: + # -- Enable L2 neighbor discovery in the agent + enabled: true + # -- Override the agent's default neighbor resolution refresh period. + refreshPeriod: "30s" + +# -- Enable Layer 7 network policy. +l7Proxy: true + +# -- Enable Local Redirect Policy. +localRedirectPolicy: false + +# To include or exclude matched resources from cilium identity evaluation +# labels: "" + +# logOptions allows you to define logging options. eg: +# logOptions: +# format: json + +# -- Enables periodic logging of system load +logSystemLoad: false + +# -- Configure maglev consistent hashing +maglev: {} + # -- tableSize is the size (parameter M) for the backend table of one + # service entry + # tableSize: + + # -- hashSeed is the cluster-wide base64 encoded seed for the hashing + # hashSeed: + +# -- Enables masquerading of IPv4 traffic leaving the node from endpoints. +enableIPv4Masquerade: true + +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +enableIPv6Masquerade: true + +# -- Enables masquerading to the source of the route for traffic leaving the node from endpoints. +enableMasqueradeRouteSource: false + +# -- Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods +enableIPv4BIGTCP: false + +# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods +enableIPv6BIGTCP: false + +egressGateway: + # -- Enables egress gateway to redirect and SNAT the traffic that leaves the + # cluster. + enabled: false + # -- Deprecated without a replacement necessary. + installRoutes: false + # -- Time between triggers of egress gateway state reconciliations + reconciliationTriggerInterval: 1s + # -- Maximum number of entries in egress gateway policy map + # maxPolicyEntries: 16384 + +vtep: +# -- Enables VXLAN Tunnel Endpoint (VTEP) Integration (beta) to allow +# Cilium-managed pods to talk to third party VTEP devices over Cilium tunnel. + enabled: false + +# -- A space separated list of VTEP device endpoint IPs, for example "1.1.1.1 1.1.2.1" + endpoint: "" +# -- A space separated list of VTEP device CIDRs, for example "1.1.1.0/24 1.1.2.0/24" + cidr: "" +# -- VTEP CIDRs Mask that applies to all VTEP CIDRs, for example "255.255.255.0" + mask: "" +# -- A space separated list of VTEP device MAC addresses (VTEP MAC), for example "x:x:x:x:x:x y:y:y:y:y:y:y" + mac: "" + +# -- (string) Allows to explicitly specify the IPv4 CIDR for native routing. +# When specified, Cilium assumes networking for this CIDR is preconfigured and +# hands traffic destined for that range to the Linux network stack without +# applying any SNAT. +# Generally speaking, specifying a native routing CIDR implies that Cilium can +# depend on the underlying networking stack to route packets to their +# destination. To offer a concrete example, if Cilium is configured to use +# direct routing and the Kubernetes CIDR is included in the native routing CIDR, +# the user must configure the routes to reach pods, either manually or by +# setting the auto-direct-node-routes flag. +ipv4NativeRoutingCIDR: "" + +# -- (string) Allows to explicitly specify the IPv6 CIDR for native routing. +# When specified, Cilium assumes networking for this CIDR is preconfigured and +# hands traffic destined for that range to the Linux network stack without +# applying any SNAT. +# Generally speaking, specifying a native routing CIDR implies that Cilium can +# depend on the underlying networking stack to route packets to their +# destination. To offer a concrete example, if Cilium is configured to use +# direct routing and the Kubernetes CIDR is included in the native routing CIDR, +# the user must configure the routes to reach pods, either manually or by +# setting the auto-direct-node-routes flag. +ipv6NativeRoutingCIDR: "" + +# -- cilium-monitor sidecar. +monitor: + # -- Enable the cilium-monitor sidecar. + enabled: false + +# -- Configure service load balancing +loadBalancer: + # -- standalone enables the standalone L4LB which does not connect to + # kube-apiserver. + # standalone: false + + # -- algorithm is the name of the load balancing algorithm for backend + # selection e.g. random or maglev + # algorithm: random + + # -- mode is the operation mode of load balancing for remote backends + # e.g. snat, dsr, hybrid + # mode: snat + + # -- acceleration is the option to accelerate service handling via XDP + # Applicable values can be: disabled (do not use XDP), native (XDP BPF + # program is run directly out of the networking driver's early receive + # path), or best-effort (use native mode XDP acceleration on devices + # that support it). + acceleration: disabled + + # -- dsrDispatch configures whether IP option or IPIP encapsulation is + # used to pass a service IP and port to remote backend + # dsrDispatch: opt + + # -- serviceTopology enables K8s Topology Aware Hints -based service + # endpoints filtering + # serviceTopology: false + + # -- L7 LoadBalancer + l7: + # -- Enable L7 service load balancing via envoy proxy. + # The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, + # will be forwarded to the local backend proxy to be load balanced to the service endpoints. + # Please refer to docs for supported annotations for more configuration. + # + # Applicable values: + # - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. + # - disabled: Disable L7 load balancing by way of service annotation. + backend: disabled + # -- List of ports from service to be automatically redirected to above backend. + # Any service exposing one of these ports will be automatically redirected. + # Fine-grained control can be achieved by using the service annotation. + ports: [] + # -- Default LB algorithm + # The default LB algorithm to be used for services, which can be overridden by the + # service annotation (e.g. service.cilium.io/lb-l7-algorithm) + # Applicable values: round_robin, least_request, random + algorithm: round_robin + +# -- Configure N-S k8s service loadbalancing +nodePort: + # -- Enable the Cilium NodePort service implementation. + enabled: false + + # -- Port range to use for NodePort services. + # range: "30000,32767" + + # -- Set to true to prevent applications binding to service ports. + bindProtection: true + + # -- Append NodePort range to ip_local_reserved_ports if clash with ephemeral + # ports is detected. + autoProtectPortRange: true + + # -- Enable healthcheck nodePort server for NodePort services + enableHealthCheck: true + + # -- Enable access of the healthcheck nodePort on the LoadBalancerIP. Needs + # EnableHealthCheck to be enabled + enableHealthCheckLoadBalancerIP: false + +# policyAuditMode: false + +# -- The agent can be put into one of the three policy enforcement modes: +# default, always and never. +# ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes +policyEnforcementMode: "default" + +# -- policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. +# The possible value is "nodes". +policyCIDRMatchMode: + +pprof: + # -- Enable pprof for cilium-agent + enabled: false + # -- Configure pprof listen address for cilium-agent + address: localhost + # -- Configure pprof listen port for cilium-agent + port: 6060 + +# -- Configure prometheus metrics on the configured port at /metrics +prometheus: + enabled: false + port: 9962 + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor cilium-agent + labels: {} + # -- Annotations to add to ServiceMonitor cilium-agent + annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-agent + jobLabel: "" + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor cilium-agent + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor cilium-agent + metricRelabelings: ~ + # -- Set to `true` and helm will not check for monitoring.coreos.com/v1 CRDs before deploying + trustCRDsExist: false + + # -- Metrics that should be enabled or disabled from the default metric list. + # The list is expected to be separated by a space. (+metric_foo to enable + # metric_foo , -metric_bar to disable metric_bar). + # ref: https://docs.cilium.io/en/stable/observability/metrics/ + metrics: ~ + + # --- Enable controller group metrics for monitoring specific Cilium + # subsystems. The list is a list of controller group names. The special + # values of "all" and "none" are supported. The set of controller + # group names is not guaranteed to be stable between Cilium versions. + controllerGroupMetrics: + - write-cni-file + - sync-host-ips + - sync-lb-maps-with-k8s-services + +# -- Grafana dashboards for cilium-agent +# grafana can import dashboards based on the label and value +# ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards +dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + +# -- Configure Istio proxy options. +proxy: + + prometheus: + # -- Deprecated in favor of envoy.prometheus.enabled + enabled: true + # -- Deprecated in favor of envoy.prometheus.port + port: ~ + # -- Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecarImageRegex: "cilium/istio_proxy" + +# Configure Cilium Envoy options. +envoy: + # -- Enable Envoy Proxy in standalone DaemonSet. + enabled: false + + log: + # -- The format string to use for laying out the log message metadata of Envoy. + format: "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v" + # -- Path to a separate Envoy log file, if any. Defaults to /dev/stdout. + path: "" + + # -- Time in seconds after which a TCP connection attempt times out + connectTimeoutSeconds: 2 + # -- ProxyMaxRequestsPerConnection specifies the max_requests_per_connection setting for Envoy + maxRequestsPerConnection: 0 + # -- Set Envoy HTTP option max_connection_duration seconds. Default 0 (disable) + maxConnectionDurationSeconds: 0 + # -- Set Envoy upstream HTTP idle connection timeout seconds. + # Does not apply to connections with pending requests. Default 60s + idleTimeoutDurationSeconds: 60 + + # -- Envoy container image. + image: + override: ~ + repository: "${CILIUM_ENVOY_REPO}" + tag: "${CILIUM_ENVOY_VERSION}" + pullPolicy: "${PULL_POLICY}" + digest: "${CILIUM_ENVOY_DIGEST}" + useDigest: true + + # -- Additional containers added to the cilium Envoy DaemonSet. + extraContainers: [] + + # -- Additional envoy container arguments. + extraArgs: [] + + # -- Additional envoy container environment variables. + extraEnv: [] + + # -- Additional envoy hostPath mounts. + extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + + # -- Additional envoy volumes. + extraVolumes: [] + + # -- Additional envoy volumeMounts. + extraVolumeMounts: [] + + # -- Configure termination grace period for cilium-envoy DaemonSet. + terminationGracePeriodSeconds: 1 + + # -- TCP port for the health API. + healthPort: 9878 + + # -- cilium-envoy update strategy + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + # -- Roll out cilium envoy pods automatically when configmap is updated. + rollOutPods: false + + # -- Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) + annotations: {} + + # -- Security Context for cilium-envoy pods. + podSecurityContext: {} + + # -- Annotations to be added to envoy pods + podAnnotations: {} + + # -- Labels to be added to envoy pods + podLabels: {} + + # -- Envoy resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + startupProbe: + # -- failure threshold of startup probe. + # 105 x 2s translates to the old behaviour of the readiness probe (120s delay + 30 x 3s) + failureThreshold: 105 + # -- interval between checks of the startup probe + periodSeconds: 2 + livenessProbe: + # -- failure threshold of liveness probe + failureThreshold: 10 + # -- interval between checks of the liveness probe + periodSeconds: 30 + readinessProbe: + # -- failure threshold of readiness probe + failureThreshold: 3 + # -- interval between checks of the readiness probe + periodSeconds: 30 + + securityContext: + # -- User to run the pod with + # runAsUser: 0 + # -- Run the pod with elevated privileges + privileged: false + # -- SELinux options for the `cilium-envoy` container + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + # -- Capabilities for the `cilium-envoy` container + envoy: + # Used since cilium proxy uses setting IPPROTO_IP/IP_TRANSPARENT + - NET_ADMIN + # We need it for now but might not need it for >= 5.11 specially + # for the 'SYS_RESOURCE'. + # In >= 5.8 there's already BPF and PERMON capabilities + - SYS_ADMIN + # Both PERFMON and BPF requires kernel 5.8, container runtime + # cri-o >= v1.22.0 or containerd >= v1.5.0. + # If available, SYS_ADMIN can be removed. + #- PERFMON + #- BPF + + # -- Affinity for cilium-envoy. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium-envoy + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + # -- Node selector for cilium-envoy. + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for envoy scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- The priority class to use for cilium-envoy. + priorityClassName: ~ + + # -- DNS policy for Cilium envoy pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: ~ + + # -- Configure Cilium Envoy Prometheus options. + # Note that some of these apply to either cilium-agent or cilium-envoy. + prometheus: + # -- Enable prometheus metrics for cilium-envoy + enabled: true + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + # Note that this setting applies to both cilium-envoy _and_ cilium-agent + # with Envoy enabled. + enabled: false + # -- Labels to add to ServiceMonitor cilium-envoy + labels: {} + # -- Annotations to add to ServiceMonitor cilium-envoy + annotations: {} + # -- Interval for scrape metrics. + interval: "10s" + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + # -- Relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + replacement: ${1} + # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. + metricRelabelings: ~ + # -- Serve prometheus metrics for cilium-envoy on the configured port + port: "9964" + +# -- Enable use of the remote node identity. +# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# Deprecated without replacement in 1.15. To be removed in 1.16. +remoteNodeIdentity: true + +# -- Enable resource quotas for priority classes used in the cluster. +resourceQuotas: + enabled: false + cilium: + hard: + # 5k nodes * 2 DaemonSets (Cilium and cilium node init) + pods: "10k" + operator: + hard: + # 15 "clusterwide" Cilium Operator pods for HA + pods: "15" + +# Need to document default +################## +#sessionAffinity: false + +# -- Do not run Cilium agent when running with clean mode. Useful to completely +# uninstall Cilium as it will stop Cilium from starting and create artifacts +# in the node. +sleepAfterInit: false + +# -- Enable check of service source ranges (currently, only for LoadBalancer). +svcSourceRangeCheck: true + +# -- Synchronize Kubernetes nodes to kvstore and perform CNP GC. +synchronizeK8sNodes: true + +# -- Configure TLS configuration in the agent. +tls: + # -- This configures how the Cilium agent loads the secrets used TLS-aware CiliumNetworkPolicies + # (namely the secrets referenced by terminatingTLS and originatingTLS). + # Possible values: + # - local + # - k8s + secretsBackend: local + + # -- Base64 encoded PEM values for the CA certificate and private key. + # This can be used as common CA to generate certificates used by hubble and clustermesh components. + # It is neither required nor used when cert-manager is used to generate the certificates. + ca: + # -- Optional CA cert. If it is provided, it will be used by cilium to + # generate all other certificates. Otherwise, an ephemeral CA is generated. + cert: "" + + # -- Optional CA private key. If it is provided, it will be used by cilium to + # generate all other certificates. Otherwise, an ephemeral CA is generated. + key: "" + + # -- Generated certificates validity duration in days. This will be used for auto generated CA. + certValidityDuration: 1095 + + # -- Configure the CA trust bundle used for the validation of the certificates + # leveraged by hubble and clustermesh. When enabled, it overrides the content of the + # 'ca.crt' field of the respective certificates, allowing for CA rotation with no down-time. + caBundle: + # -- Enable the use of the CA trust bundle. + enabled: false + + # -- Name of the ConfigMap containing the CA trust bundle. + name: cilium-root-ca.crt + + # -- Entry of the ConfigMap containing the CA trust bundle. + key: ca.crt + + # -- Use a Secret instead of a ConfigMap. + useSecret: false + + # If uncommented, creates the ConfigMap and fills it with the specified content. + # Otherwise, the ConfigMap is assumed to be already present in .Release.Namespace. + # + # content: | + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + # -----BEGIN CERTIFICATE----- + # ... + # -----END CERTIFICATE----- + +# -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. +# Possible values: +# - "" +# - vxlan +# - geneve +# @default -- `"vxlan"` +tunnelProtocol: "" + +# -- Enable native-routing mode or tunneling mode. +# Possible values: +# - "" +# - native +# - tunnel +# @default -- `"tunnel"` +routingMode: "" + +# -- Configure VXLAN and Geneve tunnel port. +# @default -- Port 8472 for VXLAN, Port 6081 for Geneve +tunnelPort: 0 + +# -- Configure what the response should be to traffic for a service without backends. +# "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". +# Possible values: +# - reject (default) +# - drop +serviceNoBackendResponse: reject + +# -- Configure the underlying network MTU to overwrite auto-detected MTU. +MTU: 0 + +# -- Disable the usage of CiliumEndpoint CRD. +disableEndpointCRD: false + +wellKnownIdentities: + # -- Enable the use of well-known identities. + enabled: false + +etcd: + # -- Enable etcd mode for the agent. + enabled: false + + # -- cilium-etcd-operator image. + image: + override: ~ + repository: "${CILIUM_ETCD_OPERATOR_REPO}" + tag: "${CILIUM_ETCD_OPERATOR_VERSION}" + digest: "${CILIUM_ETCD_OPERATOR_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + + # -- The priority class to use for cilium-etcd-operator + priorityClassName: "" + + # -- Additional cilium-etcd-operator container arguments. + extraArgs: [] + + # -- Additional cilium-etcd-operator volumes. + extraVolumes: [] + + # -- Additional cilium-etcd-operator volumeMounts. + extraVolumeMounts: [] + + # -- Node tolerations for cilium-etcd-operator scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Pod topology spread constraints for cilium-etcd-operator + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for cilium-etcd-operator pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Annotations to be added to all top-level etcd-operator objects (resources under templates/etcd-operator) + annotations: {} + + # -- Security context to be added to cilium-etcd-operator pods + podSecurityContext: {} + + # -- Annotations to be added to cilium-etcd-operator pods + podAnnotations: {} + + # -- Labels to be added to cilium-etcd-operator pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- cilium-etcd-operator resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + # -- Security context to be added to cilium-etcd-operator pods + securityContext: {} + # runAsUser: 0 + + # -- cilium-etcd-operator update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + + # -- If etcd is behind a k8s service set this option to true so that Cilium + # does the service translation automatically without requiring a DNS to be + # running. + k8sService: false + + # -- Cluster domain for cilium-etcd-operator. + clusterDomain: cluster.local + + # -- List of etcd endpoints (not needed when using managed=true). + endpoints: + - https://CHANGE-ME:2379 + + # -- Enable use of TLS/SSL for connectivity to etcd. (auto-enabled if + # managed=true) + ssl: false + +operator: + # -- Enable the cilium-operator component (required). + enabled: true + + # -- Roll out cilium-operator pods automatically when configmap is updated. + rollOutPods: false + + # -- cilium-operator image. + image: + override: ~ + repository: "${CILIUM_OPERATOR_BASE_REPO}" + tag: "${CILIUM_VERSION}" + # operator-generic-digest + genericDigest: ${OPERATOR_GENERIC_DIGEST} + # operator-azure-digest + azureDigest: ${OPERATOR_AZURE_DIGEST} + # operator-aws-digest + awsDigest: ${OPERATOR_AWS_DIGEST} + # operator-alibabacloud-digest + alibabacloudDigest: ${OPERATOR_ALIBABACLOUD_DIGEST} + useDigest: ${USE_DIGESTS} + pullPolicy: "${PULL_POLICY}" + suffix: "${CILIUM_OPERATOR_SUFFIX}" + + # -- Number of replicas to run for the cilium-operator deployment + replicas: 2 + + # -- The priority class to use for cilium-operator + priorityClassName: "" + + # -- DNS policy for Cilium operator pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: "" + + # -- cilium-operator update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + + # -- Affinity for cilium-operator + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator + + # -- Pod topology spread constraints for cilium-operator + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for cilium-operator pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for cilium-operator scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Additional cilium-operator container arguments. + extraArgs: [] + + # -- Additional cilium-operator environment variables. + extraEnv: [] + + # -- Additional cilium-operator hostPath mounts. + extraHostPathMounts: [] + # - name: host-mnt-data + # mountPath: /host/mnt/data + # hostPath: /mnt/data + # hostPathType: Directory + # readOnly: true + # mountPropagation: HostToContainer + + # -- Additional cilium-operator volumes. + extraVolumes: [] + + # -- Additional cilium-operator volumeMounts. + extraVolumeMounts: [] + + # -- Annotations to be added to all top-level cilium-operator objects (resources under templates/cilium-operator) + annotations: {} + + # -- Security context to be added to cilium-operator pods + podSecurityContext: {} + + # -- Annotations to be added to cilium-operator pods + podAnnotations: {} + + # -- Labels to be added to cilium-operator pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- cilium-operator resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 1000m + # memory: 1Gi + # requests: + # cpu: 100m + # memory: 128Mi + + # -- Security context to be added to cilium-operator pods + securityContext: {} + # runAsUser: 0 + + # -- Interval for endpoint garbage collection. + endpointGCInterval: "5m0s" + + # -- Interval for cilium node garbage collection. + nodeGCInterval: "5m0s" + + # -- Skip CNP node status clean up at operator startup. + skipCNPStatusStartupClean: false + + # -- Interval for identity garbage collection. + identityGCInterval: "15m0s" + + # -- Timeout for identity heartbeats. + identityHeartbeatTimeout: "30m0s" + + pprof: + # -- Enable pprof for cilium-operator + enabled: false + # -- Configure pprof listen address for cilium-operator + address: localhost + # -- Configure pprof listen port for cilium-operator + port: 6061 + + # -- Enable prometheus metrics for cilium-operator on the configured port at + # /metrics + prometheus: + enabled: true + port: 9963 + serviceMonitor: + # -- Enable service monitors. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor cilium-operator + labels: {} + # -- Annotations to add to ServiceMonitor cilium-operator + annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-operator + jobLabel: "" + # -- Interval for scrape metrics. + interval: "10s" + # -- Relabeling configs for the ServiceMonitor cilium-operator + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor cilium-operator + metricRelabelings: ~ + + # -- Grafana dashboards for cilium-operator + # grafana can import dashboards based on the label and value + # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards + dashboards: + enabled: false + label: grafana_dashboard + namespace: ~ + labelValue: "1" + annotations: {} + + # -- Skip CRDs creation for cilium-operator + skipCRDCreation: false + + # -- Remove Cilium node taint from Kubernetes nodes that have a healthy Cilium + # pod running. + removeNodeTaints: true + + # -- Taint nodes where Cilium is scheduled but not running. This prevents pods + # from being scheduled to nodes where Cilium is not the default CNI provider. + # @default -- same as removeNodeTaints + setNodeTaints: ~ + + # -- Set Node condition NetworkUnavailable to 'false' with the reason + # 'CiliumIsUp' for nodes that have a healthy Cilium pod. + setNodeNetworkStatus: true + + unmanagedPodWatcher: + # -- Restart any pod that are not managed by Cilium. + restart: true + # -- Interval, in seconds, to check if there are any pods that are not + # managed by Cilium. + intervalSeconds: 15 + +nodeinit: + # -- Enable the node initialization DaemonSet + enabled: false + + # -- node-init image. + image: + override: ~ + repository: "${CILIUM_NODEINIT_REPO}" + tag: "${CILIUM_NODEINIT_VERSION}" + pullPolicy: "${PULL_POLICY}" + + # -- The priority class to use for the nodeinit pod. + priorityClassName: "" + + # -- node-init update strategy + updateStrategy: + type: RollingUpdate + + # -- Additional nodeinit environment variables. + extraEnv: [] + + # -- Additional nodeinit volumes. + extraVolumes: [] + + # -- Additional nodeinit volumeMounts. + extraVolumeMounts: [] + + # -- Affinity for cilium-nodeinit + affinity: {} + + # -- Node labels for nodeinit pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for nodeinit scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - operator: Exists + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Annotations to be added to all top-level nodeinit objects (resources under templates/cilium-nodeinit) + annotations: {} + + # -- Annotations to be added to node-init pods. + podAnnotations: {} + + # -- Labels to be added to node-init pods. + podLabels: {} + + # -- nodeinit resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + requests: + cpu: 100m + memory: 100Mi + + # -- Security context to be added to nodeinit pods. + securityContext: + privileged: false + seLinuxOptions: + level: 's0' + # Running with spc_t since we have removed the privileged mode. + # Users can change it to a different type as long as they have the + # type available on the system. + type: 'spc_t' + capabilities: + add: + # Used in iptables. Consider removing once we are iptables-free + - SYS_MODULE + # Used for nsenter + - NET_ADMIN + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + + # -- bootstrapFile is the location of the file where the bootstrap timestamp is + # written by the node-init DaemonSet + bootstrapFile: "/tmp/cilium-bootstrap.d/cilium-bootstrap-time" + + # -- startup offers way to customize startup nodeinit script (pre and post position) + startup: + preScript: "" + postScript: "" + # -- prestop offers way to customize prestop nodeinit script (pre and post position) + prestop: + preScript: "" + postScript: "" + +preflight: + # -- Enable Cilium pre-flight resources (required for upgrade) + enabled: false + + # -- Cilium pre-flight image. + image: + override: ~ + repository: "${CILIUM_REPO}" + tag: "${CILIUM_VERSION}" + # cilium-digest + digest: ${CILIUM_DIGEST} + useDigest: ${USE_DIGESTS} + pullPolicy: "${PULL_POLICY}" + + # -- The priority class to use for the preflight pod. + priorityClassName: "" + + # -- preflight update strategy + updateStrategy: + type: RollingUpdate + + # -- Additional preflight environment variables. + extraEnv: [] + + # -- Additional preflight volumes. + extraVolumes: [] + + # -- Additional preflight volumeMounts. + extraVolumeMounts: [] + + # -- Affinity for cilium-preflight + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + + # -- Node labels for preflight pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for preflight scheduling to nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + # -- Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) + annotations: {} + + # -- Security context to be added to preflight pods. + podSecurityContext: {} + + # -- Annotations to be added to preflight pods + podAnnotations: {} + + # -- Labels to be added to the preflight pod. + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- preflight resource limits & requests + # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: {} + # limits: + # cpu: 4000m + # memory: 4Gi + # requests: + # cpu: 100m + # memory: 512Mi + + # -- Security context to be added to preflight pods + securityContext: {} + # runAsUser: 0 + + # -- Path to write the `--tofqdns-pre-cache` file to. + tofqdnsPreCache: "" + + # -- Configure termination grace period for preflight Deployment and DaemonSet. + terminationGracePeriodSeconds: 1 + + # -- By default we should always validate the installed CNPs before upgrading + # Cilium. This will make sure the user will have the policies deployed in the + # cluster with the right schema. + validateCNPs: true + +# -- Explicitly enable or disable priority class. +# .Capabilities.KubeVersion is unsettable in `helm template` calls, +# it depends on k8s libraries version that Helm was compiled against. +# This option allows to explicitly disable setting the priority class, which +# is useful for rendering charts for gke clusters in advance. +enableCriticalPriorityClass: true + +# disableEnvoyVersionCheck removes the check for Envoy, which can be useful +# on AArch64 as the images do not currently ship a version of Envoy. +#disableEnvoyVersionCheck: false + +clustermesh: + # -- Deploy clustermesh-apiserver for clustermesh + useAPIServer: false + # -- The maximum number of clusters to support in a ClusterMesh. This value + # cannot be changed on running clusters, and all clusters in a ClusterMesh + # must be configured with the same value. Values > 255 will decrease the + # maximum allocatable cluster-local identities. + # Supported values are 255 and 511. + maxConnectedClusters: 255 + + # -- Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) + annotations: {} + + # -- Clustermesh explicit configuration. + config: + # -- Enable the Clustermesh explicit configuration. + enabled: false + # -- Default dns domain for the Clustermesh API servers + # This is used in the case cluster addresses are not provided + # and IPs are used. + domain: mesh.cilium.io + # -- List of clusters to be peered in the mesh. + clusters: [] + # clusters: + # # -- Name of the cluster + # - name: cluster1 + # # -- Address of the cluster, use this if you created DNS records for + # # the cluster Clustermesh API server. + # address: cluster1.mesh.cilium.io + # # -- Port of the cluster Clustermesh API server. + # port: 2379 + # # -- IPs of the cluster Clustermesh API server, use multiple ones when + # # you have multiple IPs to access the Clustermesh API server. + # ips: + # - 172.18.255.201 + # # -- base64 encoded PEM values for the cluster client certificate, private key and certificate authority. + # # These fields can (and should) be omitted in case the CA is shared across clusters. In that case, the + # # "remote" private key and certificate available in the local cluster are automatically used instead. + # tls: + # cert: "" + # key: "" + # caCert: "" + + apiserver: + # -- Clustermesh API server image. + image: + override: ~ + repository: "${CLUSTERMESH_APISERVER_REPO}" + tag: "${CILIUM_VERSION}" + # clustermesh-apiserver-digest + digest: ${CLUSTERMESH_APISERVER_DIGEST} + useDigest: ${USE_DIGESTS} + pullPolicy: "${PULL_POLICY}" + + etcd: + # The etcd binary is included in the clustermesh API server image, so the same image from above is reused. + # Independent override isn't supported, because clustermesh-apiserver is tested against the etcd version it is + # built with. + + # -- Specifies the resources for etcd container in the apiserver + resources: {} + # requests: + # cpu: 200m + # memory: 256Mi + # limits: + # cpu: 1000m + # memory: 256Mi + + # -- Security context to be added to clustermesh-apiserver etcd containers + securityContext: {} + + # -- lifecycle setting for the etcd container + lifecycle: {} + + init: + # -- Specifies the resources for etcd init container in the apiserver + resources: {} + # requests: + # cpu: 100m + # memory: 100Mi + # limits: + # cpu: 100m + # memory: 100Mi + + # -- Additional arguments to `clustermesh-apiserver etcdinit`. + extraArgs: [] + + # -- Additional environment variables to `clustermesh-apiserver etcdinit`. + extraEnv: [] + + kvstoremesh: + # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved + # from the remote clusters in the local etcd instance. + enabled: false + + # -- Additional KVStoreMesh arguments. + extraArgs: [] + + # -- Additional KVStoreMesh environment variables. + extraEnv: [] + + # -- Resource requests and limits for the KVStoreMesh container + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Additional KVStoreMesh volumeMounts. + extraVolumeMounts: [] + + # -- KVStoreMesh Security context + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + # -- lifecycle setting for the KVStoreMesh container + lifecycle: {} + + service: + # -- The type of service used for apiserver access. + type: NodePort + # -- Optional port to use as the node port for apiserver access. + # + # WARNING: make sure to configure a different NodePort in each cluster if + # kube-proxy replacement is enabled, as Cilium is currently affected by a known + # bug (#24692) when NodePorts are handled by the KPR implementation. If a service + # with the same NodePort exists both in the local and the remote cluster, all + # traffic originating from inside the cluster and targeting the corresponding + # NodePort will be redirected to a local backend, regardless of whether the + # destination node belongs to the local or the remote cluster. + nodePort: 32379 + # -- Optional loadBalancer IP address to use with type LoadBalancer. + # loadBalancerIP: + + # -- Annotations for the clustermesh-apiserver + # For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" + # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 + annotations: {} + + # -- The externalTrafficPolicy of service used for apiserver access. + externalTrafficPolicy: + + # -- The internalTrafficPolicy of service used for apiserver access. + internalTrafficPolicy: + + # -- Number of replicas run for the clustermesh-apiserver deployment. + replicas: 1 + + # -- lifecycle setting for the apiserver container + lifecycle: {} + + # -- terminationGracePeriodSeconds for the clustermesh-apiserver deployment + terminationGracePeriodSeconds: 30 + + # -- Additional clustermesh-apiserver arguments. + extraArgs: [] + + # -- Additional clustermesh-apiserver environment variables. + extraEnv: [] + + # -- Additional clustermesh-apiserver volumes. + extraVolumes: [] + + # -- Additional clustermesh-apiserver volumeMounts. + extraVolumeMounts: [] + + # -- Security context to be added to clustermesh-apiserver containers + securityContext: {} + + # -- Security context to be added to clustermesh-apiserver pods + podSecurityContext: {} + + # -- Annotations to be added to clustermesh-apiserver pods + podAnnotations: {} + + # -- Labels to be added to clustermesh-apiserver pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- Resource requests and limits for the clustermesh-apiserver container of the clustermesh-apiserver deployment, such as + # resources: + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + # -- Resource requests and limits for the clustermesh-apiserver + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Affinity for clustermesh.apiserver + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: clustermesh-apiserver + + # -- Pod topology spread constraints for clustermesh-apiserver + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + + # -- clustermesh-apiserver update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- The priority class to use for clustermesh-apiserver + priorityClassName: "" + + tls: + # -- Configure the clustermesh authentication mode. + # Supported values: + # - legacy: All clusters access remote clustermesh instances with the same + # username (i.e., remote). The "remote" certificate must be + # generated with CN=remote if provided manually. + # - migration: Intermediate mode required to upgrade from legacy to cluster + # (and vice versa) with no disruption. Specifically, it enables + # the creation of the per-cluster usernames, while still using + # the common one for authentication. The "remote" certificate must + # be generated with CN=remote if provided manually (same as legacy). + # - cluster: Each cluster accesses remote etcd instances with a username + # depending on the local cluster name (i.e., remote-). + # The "remote" certificate must be generated with CN=remote- + # if provided manually. Cluster mode is meaningful only when the same + # CA is shared across all clusters part of the mesh. + authMode: legacy + + # -- Configure automatic TLS certificates generation. + # A Kubernetes CronJob is used the generate any + # certificates not provided by the user at installation + # time. + auto: + # -- When set to true, automatically generate a CA and certificates to + # enable mTLS between clustermesh-apiserver and external workload instances. + # If set to false, the certs to be provided by setting appropriate values below. + enabled: true + # Sets the method to auto-generate certificates. Supported values: + # - helm: This method uses Helm to generate all certificates. + # - cronJob: This method uses a Kubernetes CronJob the generate any + # certificates not provided by the user at installation + # time. + # - certmanager: This method use cert-manager to generate & rotate certificates. + method: helm + # -- Generated certificates validity duration in days. + certValidityDuration: 1095 + # -- Schedule for certificates regeneration (regardless of their expiration date). + # Only used if method is "cronJob". If nil, then no recurring job will be created. + # Instead, only the one-shot job is deployed to generate the certificates at + # installation time. + # + # Due to the out-of-band distribution of client certs to external workloads the + # CA is (re)regenerated only if it is not provided as a helm value and the k8s + # secret is manually deleted. + # + # Defaults to none. Commented syntax gives midnight of the first day of every + # fourth month. For syntax, see + # https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax + # schedule: "0 0 1 */4 *" + + # [Example] + # certManagerIssuerRef: + # group: cert-manager.io + # kind: ClusterIssuer + # name: ca-issuer + # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. + certManagerIssuerRef: {} + # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. + # Used if 'auto' is not enabled. + server: + cert: "" + key: "" + # -- Extra DNS names added to certificate when it's auto generated + extraDnsNames: [] + # -- Extra IP addresses added to certificate when it's auto generated + extraIpAddresses: [] + # -- base64 encoded PEM values for the clustermesh-apiserver admin certificate and private key. + # Used if 'auto' is not enabled. + admin: + cert: "" + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver client certificate and private key. + # Used if 'auto' is not enabled. + client: + cert: "" + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver remote cluster certificate and private key. + # Used if 'auto' is not enabled. + remote: + cert: "" + key: "" + + # clustermesh-apiserver Prometheus metrics configuration + metrics: + # -- Enables exporting apiserver metrics in OpenMetrics format. + enabled: true + # -- Configure the port the apiserver metric server listens on. + port: 9962 + + kvstoremesh: + # -- Enables exporting KVStoreMesh metrics in OpenMetrics format. + enabled: true + # -- Configure the port the KVStoreMesh metric server listens on. + port: 9964 + + etcd: + # -- Enables exporting etcd metrics in OpenMetrics format. + enabled: true + # -- Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. + mode: basic + # -- Configure the port the etcd metric server listens on. + port: 9963 + + serviceMonitor: + # -- Enable service monitor. + # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + enabled: false + # -- Labels to add to ServiceMonitor clustermesh-apiserver + labels: {} + # -- Annotations to add to ServiceMonitor clustermesh-apiserver + annotations: {} + # -- Specify the Kubernetes namespace where Prometheus expects to find + # service monitors configured. + # namespace: "" + + # -- Interval for scrape metrics (apiserver metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (apiserver metrics) + metricRelabelings: ~ + + kvstoremesh: + # -- Interval for scrape metrics (KVStoreMesh metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (KVStoreMesh metrics) + metricRelabelings: ~ + + etcd: + # -- Interval for scrape metrics (etcd metrics) + interval: "10s" + # -- Relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + relabelings: ~ + # -- Metrics relabeling configs for the ServiceMonitor clustermesh-apiserver (etcd metrics) + metricRelabelings: ~ + +# -- Configure external workloads support +externalWorkloads: + # -- Enable support for external workloads, such as VMs (false by default). + enabled: false + +# -- Configure cgroup related configuration +cgroup: + autoMount: + # -- Enable auto mount of cgroup2 filesystem. + # When `autoMount` is enabled, cgroup2 filesystem is mounted at + # `cgroup.hostRoot` path on the underlying host and inside the cilium agent pod. + # If users disable `autoMount`, it's expected that users have mounted + # cgroup2 filesystem at the specified `cgroup.hostRoot` volume, and then the + # volume will be mounted inside the cilium agent pod at the same path. + enabled: true + # -- Init Container Cgroup Automount resource limits & requests + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # -- Configure cgroup root where cgroup2 filesystem is mounted on the host (see also: `cgroup.autoMount`) + hostRoot: /run/cilium/cgroupv2 + +# -- Configure whether to enable auto detect of terminating state for endpoints +# in order to support graceful termination. +enableK8sTerminatingEndpoint: true + +# -- Configure whether to unload DNS policy rules on graceful shutdown +# dnsPolicyUnloadOnShutdown: false + +# -- Configure the key of the taint indicating that Cilium is not ready on the node. +# When set to a value starting with `ignore-taint.cluster-autoscaler.kubernetes.io/`, the Cluster Autoscaler will ignore the taint on its decisions, allowing the cluster to scale up. +agentNotReadyTaintKey: "node.cilium.io/agent-not-ready" + +dnsProxy: + # -- DNS response code for rejecting DNS requests, available options are '[nameError refused]'. + dnsRejectResponseCode: refused + # -- Allow the DNS proxy to compress responses to endpoints that are larger than 512 Bytes or the EDNS0 option, if present. + enableDnsCompression: true + # -- Maximum number of IPs to maintain per FQDN name for each endpoint. + endpointMaxIpPerHostname: 50 + # -- Time during which idle but previously active connections with expired DNS lookups are still considered alive. + idleConnectionGracePeriod: 0s + # -- Maximum number of IPs to retain for expired DNS lookups with still-active connections. + maxDeferredConnectionDeletes: 10000 + # -- The minimum time, in seconds, to use DNS data for toFQDNs policies. If + # the upstream DNS server returns a DNS record with a shorter TTL, Cilium + # overwrites the TTL with this value. Setting this value to zero means that + # Cilium will honor the TTLs returned by the upstream DNS server. + minTtl: 0 + # -- DNS cache data at this path is preloaded on agent startup. + preCache: "" + # -- Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. + proxyPort: 0 + # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. + proxyResponseMaxDelay: 100ms + # -- DNS proxy operation mode (true/false, or unset to use version dependent defaults) + # enableTransparentMode: true + +# -- SCTP Configuration Values +sctp: + # -- Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. + enabled: false + +# Configuration for types of authentication for Cilium (beta) +authentication: + # -- Enable authentication processing and garbage collection. + # Note that if disabled, policy enforcement will still block requests that require authentication. + # But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. + enabled: true + # -- Buffer size of the channel Cilium uses to receive authentication events from the signal map. + queueSize: 1024 + # -- Buffer size of the channel Cilium uses to receive certificate expiration events from auth handlers. + rotatedIdentitiesQueueSize: 1024 + # -- Interval for garbage collection of auth map entries. + gcInterval: "5m0s" + # Configuration for Cilium's service-to-service mutual authentication using TLS handshakes. + # Note that this is not full mTLS support without also enabling encryption of some form. + # Current encryption options are Wireguard or IPSec, configured in encryption block above. + mutual: + # -- Port on the agent where mutual authentication handshakes between agents will be performed + port: 4250 + # -- Timeout for connecting to the remote node TCP socket + connectTimeout: 5s + # Settings for SPIRE + spire: + # -- Enable SPIRE integration (beta) + enabled: false + # -- Annotations to be added to all top-level spire objects (resources under templates/spire) + annotations: {} + # Settings to control the SPIRE installation and configuration + install: + # -- Enable SPIRE installation. + # This will only take effect only if authentication.mutual.spire.enabled is true + enabled: true + # -- SPIRE namespace to install into + namespace: cilium-spire + # -- SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. + existingNamespace: false + # -- init container image of SPIRE agent and server + initImage: + override: ~ + repository: "${SPIRE_INIT_REPO}" + tag: "${SPIRE_INIT_VERSION}" + digest: "${SPIRE_INIT_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + # SPIRE agent configuration + agent: + # -- SPIRE agent image + image: + override: ~ + repository: "${SPIRE_AGENT_REPO}" + tag: "${SPIRE_AGENT_VERSION}" + digest: "${SPIRE_AGENT_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + # -- SPIRE agent service account + serviceAccount: + create: true + name: spire-agent + # -- SPIRE agent annotations + annotations: {} + # -- SPIRE agent labels + labels: {} + # -- SPIRE Workload Attestor kubelet verification. + skipKubeletVerification: true + # -- SPIRE agent tolerations configuration + # By default it follows the same tolerations as the agent itself + # to allow the Cilium agent on this node to connect to SPIRE. + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" + # -- SPIRE agent affinity configuration + affinity: {} + # -- SPIRE agent nodeSelector configuration + # ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: {} + # -- Security context to be added to spire agent pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire agent containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} + server: + # -- SPIRE server image + image: + override: ~ + repository: "${SPIRE_SERVER_REPO}" + tag: "${SPIRE_SERVER_VERSION}" + digest: "${SPIRE_SERVER_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" + # -- SPIRE server service account + serviceAccount: + create: true + name: spire-server + # -- SPIRE server init containers + initContainers: [] + # -- SPIRE server annotations + annotations: {} + # -- SPIRE server labels + labels: {} + # SPIRE server service configuration + service: + # -- Service type for the SPIRE server service + type: ClusterIP + # -- Annotations to be added to the SPIRE server service + annotations: {} + # -- Labels to be added to the SPIRE server service + labels: {} + # -- SPIRE server affinity configuration + affinity: {} + # -- SPIRE server nodeSelector configuration + # ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: {} + # -- SPIRE server tolerations configuration + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # SPIRE server datastorage configuration + dataStorage: + # -- Enable SPIRE server data storage + enabled: true + # -- Size of the SPIRE server data storage + size: 1Gi + # -- Access mode of the SPIRE server data storage + accessMode: ReadWriteOnce + # -- StorageClass of the SPIRE server data storage + storageClass: null + # -- Security context to be added to spire server pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire server containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} + # SPIRE CA configuration + ca: + # -- SPIRE CA key type + # AWS requires the use of RSA. EC cryptography is not supported + keyType: "rsa-4096" + # -- SPIRE CA Subject + subject: + country: "US" + organization: "SPIRE" + commonName: "Cilium SPIRE CA" + # -- SPIRE server address used by Cilium Operator + # + # If k8s Service DNS along with port number is used (e.g. ..svc(.*): format), + # Cilium Operator will resolve its address by looking up the clusterIP from Service resource. + # + # Example values: 10.0.0.1:8081, spire-server.cilium-spire.svc:8081 + serverAddress: ~ + # -- SPIFFE trust domain to use for fetching certificates + trustDomain: spiffe.cilium + # -- SPIRE socket path where the SPIRE delegated api agent is listening + adminSocketPath: /run/spire/sockets/admin.sock + # -- SPIRE socket path where the SPIRE workload agent is listening. + # Applies to both the Cilium Agent and Operator + agentSocketPath: /run/spire/sockets/agent/agent.sock + # -- SPIRE connection timeout + connectionTimeout: 30s diff --git a/magnum_cluster_api/charts/cluster-autoscaler/.helmignore b/magnum_cluster_api/charts/cluster-autoscaler/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/magnum_cluster_api/charts/cluster-autoscaler/Chart.yaml b/magnum_cluster_api/charts/cluster-autoscaler/Chart.yaml new file mode 100644 index 00000000..78bc69e0 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +appVersion: 1.27.2 +description: Scales Kubernetes worker nodes within autoscaling groups. +home: https://github.com/kubernetes/autoscaler +icon: https://github.com/kubernetes/kubernetes/raw/master/logo/logo.png +maintainers: +- email: guyjtempleton@googlemail.com + name: gjtempleton +name: cluster-autoscaler +sources: +- https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler +type: application +version: 9.29.1 diff --git a/magnum_cluster_api/charts/cluster-autoscaler/README.md b/magnum_cluster_api/charts/cluster-autoscaler/README.md new file mode 100644 index 00000000..ff3aba2e --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/README.md @@ -0,0 +1,437 @@ +# cluster-autoscaler + +Scales Kubernetes worker nodes within autoscaling groups. + +## TL;DR + +```console +$ helm repo add autoscaler https://kubernetes.github.io/autoscaler + +# Method 1 - Using Autodiscovery +$ helm install my-release autoscaler/cluster-autoscaler \ + --set 'autoDiscovery.clusterName'= + +# Method 2 - Specifying groups manually +$ helm install my-release autoscaler/cluster-autoscaler \ + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" +``` + +## Introduction + +This chart bootstraps a cluster-autoscaler deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Helm 3+ +- Kubernetes 1.8+ + - [Older versions](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#releases) may work by overriding the `image`. Cluster autoscaler internally simulates the scheduler and bugs between mismatched versions may be subtle. +- Azure AKS specific Prerequisites: + - Kubernetes 1.10+ with RBAC-enabled. + +## Previous Helm Chart + +The previous `cluster-autoscaler` Helm chart hosted at [helm/charts](https://github.com/helm/charts) has been moved to this repository in accordance with the [Deprecation timeline](https://github.com/helm/charts#deprecation-timeline). Note that a few things have changed between this version and the old version: + +- This repository **only** supports Helm chart installations using Helm 3+ since the `apiVersion` on the charts has been marked as `v2`. +- Previous versions of the Helm chart have not been migrated + +## Migration from 1.X to 9.X+ versions of this Chart + +**TL;DR:** +You should choose to use versions >=9.0.0 of the `cluster-autoscaler` chart published from this repository; previous versions, and the `cluster-autoscaler-chart` with versioning 1.X.X published from this repository are deprecated. + +
+ Previous versions of this chart - further details +On initial migration of this chart from the `helm/charts` repository this chart was renamed from `cluster-autoscaler` to `cluster-autoscaler-chart` due to technical limitations. This affected all `1.X` releases of the chart, version 2.0.0 of this chart exists only to mark the [`cluster-autoscaler-chart` chart](https://artifacthub.io/packages/helm/cluster-autoscaler/cluster-autoscaler-chart) as deprecated. + +Releases of the chart from `9.0.0` onwards return the naming of the chart to `cluster-autoscaler` and return to following the versioning established by the chart's previous location at . + +To migrate from a 1.X release of the chart to a `9.0.0` or later release, you should first uninstall your `1.X` install of the `cluster-autoscaler-chart` chart, before performing the installation of the new `cluster-autoscaler` chart. +
+ +## Migration from 9.0 to 9.1 + +Starting from `9.1.0` the `envFromConfigMap` value is expected to contain the name of a ConfigMap that is used as ref for `envFrom`, similar to `envFromSecret`. If you want to keep the previous behaviour of `envFromConfigMap` you must rename it to `extraEnvConfigMaps`. + +## Installing the Chart + +**By default, no deployment is created and nothing will autoscale**. + +You must provide some minimal configuration, either to specify instance groups or enable auto-discovery. It is not recommended to do both. + +Either: + +- Set `autoDiscovery.clusterName` and provide additional autodiscovery options if necessary **or** +- Set static node group configurations for one or more node groups (using `autoscalingGroups` or `autoscalingGroupsnamePrefix`). + +To create a valid configuration, follow instructions for your cloud provider: + +- [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +- [GCE](#gce) +- [Azure AKS](#azure-aks) +- [OpenStack Magnum](#openstack-magnum) + +### AWS - Using auto-discovery of tagged instance groups + +Auto-discovery finds ASGs tags as below and automatically manages them based on the min and max size specified in the ASG. `cloudProvider=aws` only. + +- Tag the ASGs with keys to match `.Values.autoDiscovery.tags`, by default: `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` +- Verify the [IAM Permissions](#aws---iam) +- Set `autoDiscovery.clusterName=` +- Set `awsRegion=` +- Set (option) `awsAccessKeyID=` and `awsSecretAccessKey=` if you want to [use AWS credentials directly instead of an instance role](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= +``` + +Alternatively with your own AWS credentials + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= \ + --set awsAccessKeyID= \ + --set awsSecretAccessKey= +``` + +#### Specifying groups manually + +Without autodiscovery, specify an array of elements each containing ASG name, min size, max size. The sizes specified here will be applied to the ASG, assuming IAM permissions are correctly configured. + +- Verify the [IAM Permissions](#aws---iam) +- Either provide a yaml file setting `autoscalingGroups` (see values.yaml) or use `--set` e.g.: + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" +``` + +#### Auto-discovery + +For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/`. + +The value of the tag does not matter, only the key. + +An example kops spec excerpt: + +```yaml +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + name: my.cluster.internal +spec: + additionalPolicies: + node: | + [ + {"Effect":"Allow","Action":["autoscaling:DescribeAutoScalingGroups","autoscaling:DescribeAutoScalingInstances","autoscaling:DescribeLaunchConfigurations","autoscaling:DescribeTags","autoscaling:SetDesiredCapacity","autoscaling:TerminateInstanceInAutoScalingGroup"],"Resource":"*"} + ] + ... +--- +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: my.cluster.internal + name: my-instances +spec: + cloudLabels: + k8s.io/cluster-autoscaler/enabled: "" + k8s.io/cluster-autoscaler/my.cluster.internal: "" + image: kops.io/k8s-1.8-debian-jessie-amd64-hvm-ebs-2018-01-14 + machineType: r4.large + maxSize: 4 + minSize: 0 +``` + +In this example you would need to `--set autoDiscovery.clusterName=my.cluster.internal` when installing. + +It is not recommended to try to mix this with setting `autoscalingGroups`. + +See [autoscaler AWS documentation](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup) for a more discussion of the setup. + +### GCE + +The following parameters are required: + +- `autoDiscovery.clusterName=any-name` +- `cloud-provider=gce` +- `autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefix[0].minSize=1` + +To use Managed Instance Group (MIG) auto-discovery, provide a YAML file setting `autoscalingGroupsnamePrefix` (see values.yaml) or use `--set` when installing the Chart - e.g. + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ + --set autoDiscovery.clusterName= \ + --set cloudProvider=gce +``` + +Note that `your-ig-prefix` should be a _prefix_ matching one or more MIGs, and _not_ the full name of the MIG. For example, to match multiple instance groups - `k8s-node-group-a-standard`, `k8s-node-group-b-gpu`, you would use a prefix of `k8s-node-group-`. + +In the event you want to explicitly specify MIGs instead of using auto-discovery, set members of the `autoscalingGroups` array directly - e.g. + +``` +# where 'n' is the index, starting at 0 +--set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE +``` + +### Azure AKS + +The following parameters are required: + +- `cloudProvider=azure` +- `autoscalingGroups[0].name=your-agent-pool,autoscalingGroups[0].maxSize=10,autoscalingGroups[0].minSize=1` +- `azureClientID: "your-service-principal-app-id"` +- `azureClientSecret: "your-service-principal-client-secret"` +- `azureSubscriptionID: "your-azure-subscription-id"` +- `azureTenantID: "your-azure-tenant-id"` +- `azureClusterName: "your-aks-cluster-name"` +- `azureResourceGroup: "your-aks-cluster-resource-group-name"` +- `azureVMType: "AKS"` +- `azureNodeResourceGroup: "your-aks-cluster-node-resource-group"` + +### OpenStack Magnum + +`cloudProvider: magnum` must be set, and then one of + +- `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts +- or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. + +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location of the cloud-config file on the host. + +Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). + +Install the chart with + +```console +$ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml +``` + +### Cluster-API + +`cloudProvider: clusterapi` must be set, and then one or more of + +- `autoDiscovery.clusterName` +- or `autoDiscovery.labels` + +See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details. + +Additional config parameters available, see the `values.yaml` for more details + +- `clusterAPIMode` +- `clusterAPIKubeconfigSecret` +- `clusterAPIWorkloadKubeconfigPath` +- `clusterAPICloudConfigPath` + +## Uninstalling the Chart + +To uninstall `my-release`: + +```console +$ helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +> **Tip**: List all releases using `helm list` or start clean with `helm uninstall my-release` + +## Additional Configuration + +### AWS - IAM + +The worker running the cluster autoscaler will need access to certain resources and actions depending on the version you run and your configuration of it. + +For the up-to-date IAM permissions required, please see the [cluster autoscaler's AWS Cloudprovider Readme](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#iam-policy) and switch to the tag of the cluster autoscaler image you are using. + +### AWS - IAM Roles for Service Accounts (IRSA) + +For Kubernetes clusters that use Amazon EKS, the service account can be configured with an IAM role using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) to avoid needing to grant access to the worker nodes for AWS resources. + +In order to accomplish this, you will first need to create a new IAM role with the above mentions policies. Take care in [configuring the trust relationship](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html#iam-role-configuration) to restrict access just to the service account used by cluster autoscaler. + +Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing. + +### Azure - Using azure workload identity + +You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure. + +You can also set the correct settings yourself instead of relying on this project. + +For example the following configuration will configure the Autoscaler to use your federated identity: + +```yaml +azureUseWorkloadIdentityExtension: true +extraEnv: + AZURE_CLIENT_ID: USER ASSIGNED IDENTITY CLIENT ID + AZURE_TENANT_ID: USER ASSIGNED IDENTITY TENANT ID + AZURE_FEDERATED_TOKEN_FILE: /var/run/secrets/tokens/azure-identity-token + AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/ +extraVolumes: +- name: azure-identity-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api://AzureADTokenExchange + expirationSeconds: 3600 + path: azure-identity-token +extraVolumeMounts: +- mountPath: /var/run/secrets/tokens + name: azure-identity-token + readOnly: true +``` + +## Troubleshooting + +The chart will succeed even if the container arguments are incorrect. A few minutes after starting `kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like + +``` +polling_autoscaler.go:111] Poll finished +static_autoscaler.go:97] Starting main loop +utils.go:435] No pod using affinity / antiaffinity found in cluster, disabling affinity predicate for this loop +static_autoscaler.go:230] Filtering out schedulables +``` + +If not, find a pod that the deployment created and `describe` it, paying close attention to the arguments under `Command`. e.g.: + +``` +Containers: + cluster-autoscaler: + Command: + ./cluster-autoscaler + --cloud-provider=aws +# if specifying ASGs manually + --nodes=1:10:your-scaling-group-name +# if using autodiscovery + --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + --v=4 +``` + +### PodSecurityPolicy + +Though enough for the majority of installations, the default PodSecurityPolicy _could_ be too restrictive depending on the specifics of your release. Please make sure to check that the template fits with any customizations made or disable it by setting `rbac.pspEnabled` to `false`. + +### VerticalPodAutoscaler + +The CA Helm Chart can install a [`VerticalPodAutoscaler`](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/README.md) object from Chart version `9.27.0` +onwards for the Cluster Autoscaler Deployment to scale the CA as appropriate, but for that, we +need to install the VPA to the cluster separately. A VPA can help minimize wasted resources +when usage spikes periodically or remediate containers that are being OOMKilled. + +The following example snippet can be used to install VPA that allows scaling down from the default recommendations of the deployment template: + +```yaml +vpa: + enabled: true + containerPolicy: + minAllowed: + cpu: 20m + memory: 50Mi +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| additionalLabels | object | `{}` | Labels to add to each object of the chart. | +| affinity | object | `{}` | Affinity for pod assignment | +| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup. Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. | +| autoDiscovery.labels | list | `[]` | Cluster-API labels to match https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery | +| autoDiscovery.roles | list | `["worker"]` | Magnum node group roles to match. | +| autoDiscovery.tags | list | `["k8s.io/cluster-autoscaler/enabled","k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }}"]` | ASG tags to match, run through `tpl`. | +| autoscalingGroups | list | `[]` | For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: 
 - name: asg1
   maxSize: 2
   minSize: 1
| +| autoscalingGroupsnamePrefix | list | `[]` | For GCE. At least one element is required if not using `autoDiscovery`. For example: 
 - name: ig01
   maxSize: 10
   minSize: 0
| +| awsAccessKeyID | string | `""` | AWS access key ID ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) | +| awsRegion | string | `"us-east-1"` | AWS region (required if `cloudProvider=aws`) | +| awsSecretAccessKey | string | `""` | AWS access secret key ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) | +| azureClientID | string | `""` | Service Principal ClientID with contributor permission to Cluster and Node ResourceGroup. Required if `cloudProvider=azure` | +| azureClientSecret | string | `""` | Service Principal ClientSecret with contributor permission to Cluster and Node ResourceGroup. Required if `cloudProvider=azure` | +| azureClusterName | string | `""` | Azure AKS cluster name. Required if `cloudProvider=azure` | +| azureNodeResourceGroup | string | `""` | Azure resource group where the cluster's nodes are located, typically set as `MC___`. Required if `cloudProvider=azure` | +| azureResourceGroup | string | `""` | Azure resource group that the cluster is located. Required if `cloudProvider=azure` | +| azureSubscriptionID | string | `""` | Azure subscription where the resources are located. Required if `cloudProvider=azure` | +| azureTenantID | string | `""` | Azure tenant where the resources are located. Required if `cloudProvider=azure` | +| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. | +| azureUseWorkloadIdentityExtension | bool | `false` | Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. | +| azureVMType | string | `"AKS"` | Azure VM type. | +| cloudConfigPath | string | `""` | Configuration file for cloud provider. | +| cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. | +| clusterAPICloudConfigPath | string | `"/etc/kubernetes/mgmt-kubeconfig"` | Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig` | +| clusterAPIConfigMapsNamespace | string | `""` | Namespace on the workload cluster to store Leader election and status configmaps | +| clusterAPIKubeconfigSecret | string | `""` | Secret containing kubeconfig for connecting to Cluster API managed workloadcluster Required if `cloudProvider=clusterapi` and `clusterAPIMode=kubeconfig-kubeconfig,kubeconfig-incluster or incluster-kubeconfig` | +| clusterAPIMode | string | `"incluster-incluster"` | Cluster API mode, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#connecting-cluster-autoscaler-to-cluster-api-management-and-workload-clusters Syntax: workloadClusterMode-ManagementClusterMode for `kubeconfig-kubeconfig`, `incluster-kubeconfig` and `single-kubeconfig` you always must mount the external kubeconfig using either `extraVolumeSecrets` or `extraMounts` and `extraVolumes` if you dont set `clusterAPIKubeconfigSecret`and thus use an in-cluster config or want to use a non capi generated kubeconfig you must do so for the workload kubeconfig as well | +| clusterAPIWorkloadKubeconfigPath | string | `"/etc/kubernetes/value"` | Path to kubeconfig for connecting to Cluster API managed workloadcluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or kubeconfig-incluster` | +| containerSecurityContext | object | `{}` | [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | +| deployment.annotations | object | `{}` | Annotations to add to the Deployment object. | +| dnsPolicy | string | `"ClusterFirst"` | Defaults to `ClusterFirst`. Valid values are: `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. | +| envFromConfigMap | string | `""` | ConfigMap name to use as envFrom. | +| envFromSecret | string | `""` | Secret name to use as envFrom. | +| expanderPriorities | object | `{}` | The expanderPriorities is used if `extraArgs.expander` contains `priority` and expanderPriorities is also set with the priorities. If `extraArgs.expander` contains `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md | +| extraArgs | object | `{"logtostderr":true,"stderrthreshold":"info","v":4}` | Additional container arguments. Refer to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca for the full list of cluster autoscaler parameters and their default values. Everything after the first _ will be ignored allowing the use of multi-string arguments. | +| extraEnv | object | `{}` | Additional container environment variables. | +| extraEnvConfigMaps | object | `{}` | Additional container environment variables from ConfigMaps. | +| extraEnvSecrets | object | `{}` | Additional container environment variables from Secrets. | +| extraVolumeMounts | list | `[]` | Additional volumes to mount. | +| extraVolumeSecrets | object | `{}` | Additional volumes to mount from Secrets. | +| extraVolumes | list | `[]` | Additional volumes. | +| fullnameOverride | string | `""` | String to fully override `cluster-autoscaler.fullname` template. | +| hostNetwork | bool | `false` | Whether to expose network interfaces of the host machine to pods. | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Image pull secrets | +| image.repository | string | `"registry.k8s.io/autoscaling/cluster-autoscaler"` | Image repository | +| image.tag | string | `"v1.27.2"` | Image tag | +| kubeTargetVersionOverride | string | `""` | Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. | +| magnumCABundlePath | string | `"/etc/kubernetes/ca-bundle.crt"` | Path to the host's CA bundle, from `ca-file` in the cloud-config file. | +| magnumClusterName | string | `""` | Cluster name or ID in Magnum. Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. | +| nameOverride | string | `""` | String to partially override `cluster-autoscaler.fullname` template (will maintain the release name) | +| nodeSelector | object | `{}` | Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. | +| podAnnotations | object | `{}` | Annotations to add to each pod. | +| podDisruptionBudget | object | `{"maxUnavailable":1}` | Pod disruption budget. | +| podLabels | object | `{}` | Labels to add to each pod. | +| priorityClassName | string | `"system-cluster-critical"` | priorityClassName | +| priorityConfigMapAnnotations | object | `{}` | Annotations to add to `cluster-autoscaler-priority-expander` ConfigMap. | +| prometheusRule.additionalLabels | object | `{}` | Additional labels to be set in metadata. | +| prometheusRule.enabled | bool | `false` | If true, creates a Prometheus Operator PrometheusRule. | +| prometheusRule.interval | string | `nil` | How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). | +| prometheusRule.namespace | string | `"monitoring"` | Namespace which Prometheus is running in. | +| prometheusRule.rules | list | `[]` | Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). | +| rbac.clusterScoped | bool | `true` | if set to false will only provision RBAC to alter resources in the current namespace. Most useful for Cluster-API | +| rbac.create | bool | `true` | If `true`, create and use RBAC resources. | +| rbac.pspEnabled | bool | `false` | If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. Must be used with `rbac.create` set to `true`. | +| rbac.serviceAccount.annotations | object | `{}` | Additional Service Account annotations. | +| rbac.serviceAccount.automountServiceAccountToken | bool | `true` | Automount API credentials for a Service Account. | +| rbac.serviceAccount.create | bool | `true` | If `true` and `rbac.create` is also true, a Service Account will be created. | +| rbac.serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. | +| replicaCount | int | `1` | Desired number of pods | +| resources | object | `{}` | Pod resource requests and limits. | +| securityContext | object | `{}` | [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | +| service.annotations | object | `{}` | Annotations to add to service | +| service.create | bool | `true` | If `true`, a Service will be created. | +| service.externalIPs | list | `[]` | List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. | +| service.labels | object | `{}` | Labels to add to service | +| service.loadBalancerIP | string | `""` | IP address to assign to load balancer (if supported). | +| service.loadBalancerSourceRanges | list | `[]` | List of IP CIDRs allowed access to load balancer (if supported). | +| service.portName | string | `"http"` | Name for service port. | +| service.servicePort | int | `8085` | Service port to expose. | +| service.type | string | `"ClusterIP"` | Type of service to create. | +| serviceMonitor.annotations | object | `{}` | Annotations to add to service monitor | +| serviceMonitor.enabled | bool | `false` | If true, creates a Prometheus Operator ServiceMonitor. | +| serviceMonitor.interval | string | `"10s"` | Interval that Prometheus scrapes Cluster Autoscaler metrics. | +| serviceMonitor.metricRelabelings | object | `{}` | MetricRelabelConfigs to apply to samples before ingestion. | +| serviceMonitor.namespace | string | `"monitoring"` | Namespace which Prometheus is running in. | +| serviceMonitor.path | string | `"/metrics"` | The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) | +| serviceMonitor.selector | object | `{"release":"prometheus-operator"}` | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. | +| tolerations | list | `[]` | List of node taints to tolerate (requires Kubernetes >= 1.6). | +| topologySpreadConstraints | list | `[]` | You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19). | +| updateStrategy | object | `{}` | [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) | +| vpa | object | `{"containerPolicy":{},"enabled":false,"updateMode":"Auto"}` | Configure a VerticalPodAutoscaler for the cluster-autoscaler Deployment. | +| vpa.containerPolicy | object | `{}` | [ContainerResourcePolicy](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L159). The containerName is always et to the deployment's container name. This value is required if VPA is enabled. | +| vpa.enabled | bool | `false` | If true, creates a VerticalPodAutoscaler. | +| vpa.updateMode | string | `"Auto"` | [UpdateMode](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L124) | diff --git a/magnum_cluster_api/charts/cluster-autoscaler/README.md.gotmpl b/magnum_cluster_api/charts/cluster-autoscaler/README.md.gotmpl new file mode 100644 index 00000000..611ad6bb --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/README.md.gotmpl @@ -0,0 +1,339 @@ +{{ template "chart.header" . }} + +{{ template "chart.description" . }} + +## TL;DR + +```console +$ helm repo add autoscaler https://kubernetes.github.io/autoscaler + +# Method 1 - Using Autodiscovery +$ helm install my-release autoscaler/cluster-autoscaler \ + --set 'autoDiscovery.clusterName'= + +# Method 2 - Specifying groups manually +$ helm install my-release autoscaler/cluster-autoscaler \ + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" +``` + +## Introduction + +This chart bootstraps a cluster-autoscaler deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Helm 3+ +- Kubernetes 1.8+ + - [Older versions](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#releases) may work by overriding the `image`. Cluster autoscaler internally simulates the scheduler and bugs between mismatched versions may be subtle. +- Azure AKS specific Prerequisites: + - Kubernetes 1.10+ with RBAC-enabled. + +## Previous Helm Chart + +The previous `cluster-autoscaler` Helm chart hosted at [helm/charts](https://github.com/helm/charts) has been moved to this repository in accordance with the [Deprecation timeline](https://github.com/helm/charts#deprecation-timeline). Note that a few things have changed between this version and the old version: + +- This repository **only** supports Helm chart installations using Helm 3+ since the `apiVersion` on the charts has been marked as `v2`. +- Previous versions of the Helm chart have not been migrated + +## Migration from 1.X to 9.X+ versions of this Chart + +**TL;DR:** +You should choose to use versions >=9.0.0 of the `cluster-autoscaler` chart published from this repository; previous versions, and the `cluster-autoscaler-chart` with versioning 1.X.X published from this repository are deprecated. + +
+ Previous versions of this chart - further details +On initial migration of this chart from the `helm/charts` repository this chart was renamed from `cluster-autoscaler` to `cluster-autoscaler-chart` due to technical limitations. This affected all `1.X` releases of the chart, version 2.0.0 of this chart exists only to mark the [`cluster-autoscaler-chart` chart](https://artifacthub.io/packages/helm/cluster-autoscaler/cluster-autoscaler-chart) as deprecated. + +Releases of the chart from `9.0.0` onwards return the naming of the chart to `cluster-autoscaler` and return to following the versioning established by the chart's previous location at . + +To migrate from a 1.X release of the chart to a `9.0.0` or later release, you should first uninstall your `1.X` install of the `cluster-autoscaler-chart` chart, before performing the installation of the new `cluster-autoscaler` chart. +
+ +## Migration from 9.0 to 9.1 + +Starting from `9.1.0` the `envFromConfigMap` value is expected to contain the name of a ConfigMap that is used as ref for `envFrom`, similar to `envFromSecret`. If you want to keep the previous behaviour of `envFromConfigMap` you must rename it to `extraEnvConfigMaps`. + +## Installing the Chart + +**By default, no deployment is created and nothing will autoscale**. + +You must provide some minimal configuration, either to specify instance groups or enable auto-discovery. It is not recommended to do both. + +Either: + +- Set `autoDiscovery.clusterName` and provide additional autodiscovery options if necessary **or** +- Set static node group configurations for one or more node groups (using `autoscalingGroups` or `autoscalingGroupsnamePrefix`). + +To create a valid configuration, follow instructions for your cloud provider: + +- [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +- [GCE](#gce) +- [Azure AKS](#azure-aks) +- [OpenStack Magnum](#openstack-magnum) + +### AWS - Using auto-discovery of tagged instance groups + +Auto-discovery finds ASGs tags as below and automatically manages them based on the min and max size specified in the ASG. `cloudProvider=aws` only. + +- Tag the ASGs with keys to match `.Values.autoDiscovery.tags`, by default: `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` +- Verify the [IAM Permissions](#aws---iam) +- Set `autoDiscovery.clusterName=` +- Set `awsRegion=` +- Set (option) `awsAccessKeyID=` and `awsSecretAccessKey=` if you want to [use AWS credentials directly instead of an instance role](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= +``` + +Alternatively with your own AWS credentials + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= \ + --set awsAccessKeyID= \ + --set awsSecretAccessKey= +``` + +#### Specifying groups manually + +Without autodiscovery, specify an array of elements each containing ASG name, min size, max size. The sizes specified here will be applied to the ASG, assuming IAM permissions are correctly configured. + +- Verify the [IAM Permissions](#aws---iam) +- Either provide a yaml file setting `autoscalingGroups` (see values.yaml) or use `--set` e.g.: + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" +``` + +#### Auto-discovery + +For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/`. + +The value of the tag does not matter, only the key. + +An example kops spec excerpt: + +```yaml +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + name: my.cluster.internal +spec: + additionalPolicies: + node: | + [ + {"Effect":"Allow","Action":["autoscaling:DescribeAutoScalingGroups","autoscaling:DescribeAutoScalingInstances","autoscaling:DescribeLaunchConfigurations","autoscaling:DescribeTags","autoscaling:SetDesiredCapacity","autoscaling:TerminateInstanceInAutoScalingGroup"],"Resource":"*"} + ] + ... +--- +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: my.cluster.internal + name: my-instances +spec: + cloudLabels: + k8s.io/cluster-autoscaler/enabled: "" + k8s.io/cluster-autoscaler/my.cluster.internal: "" + image: kops.io/k8s-1.8-debian-jessie-amd64-hvm-ebs-2018-01-14 + machineType: r4.large + maxSize: 4 + minSize: 0 +``` + +In this example you would need to `--set autoDiscovery.clusterName=my.cluster.internal` when installing. + +It is not recommended to try to mix this with setting `autoscalingGroups`. + +See [autoscaler AWS documentation](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup) for a more discussion of the setup. + +### GCE + +The following parameters are required: + +- `autoDiscovery.clusterName=any-name` +- `cloud-provider=gce` +- `autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefix[0].minSize=1` + +To use Managed Instance Group (MIG) auto-discovery, provide a YAML file setting `autoscalingGroupsnamePrefix` (see values.yaml) or use `--set` when installing the Chart - e.g. + +```console +$ helm install my-release autoscaler/cluster-autoscaler \ + --set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ + --set autoDiscovery.clusterName= \ + --set cloudProvider=gce +``` + +Note that `your-ig-prefix` should be a _prefix_ matching one or more MIGs, and _not_ the full name of the MIG. For example, to match multiple instance groups - `k8s-node-group-a-standard`, `k8s-node-group-b-gpu`, you would use a prefix of `k8s-node-group-`. + +In the event you want to explicitly specify MIGs instead of using auto-discovery, set members of the `autoscalingGroups` array directly - e.g. + +``` +# where 'n' is the index, starting at 0 +--set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE +``` + +### Azure AKS + +The following parameters are required: + +- `cloudProvider=azure` +- `autoscalingGroups[0].name=your-agent-pool,autoscalingGroups[0].maxSize=10,autoscalingGroups[0].minSize=1` +- `azureClientID: "your-service-principal-app-id"` +- `azureClientSecret: "your-service-principal-client-secret"` +- `azureSubscriptionID: "your-azure-subscription-id"` +- `azureTenantID: "your-azure-tenant-id"` +- `azureClusterName: "your-aks-cluster-name"` +- `azureResourceGroup: "your-aks-cluster-resource-group-name"` +- `azureVMType: "AKS"` +- `azureNodeResourceGroup: "your-aks-cluster-node-resource-group"` + +### OpenStack Magnum + +`cloudProvider: magnum` must be set, and then one of + +- `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts +- or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. + +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location of the cloud-config file on the host. + +Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). + +Install the chart with + +```console +$ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml +``` + +### Cluster-API + +`cloudProvider: clusterapi` must be set, and then one or more of + +- `autoDiscovery.clusterName` +- or `autoDiscovery.labels` + +See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details. + +Additional config parameters available, see the `values.yaml` for more details + +- `clusterAPIMode` +- `clusterAPIKubeconfigSecret` +- `clusterAPIWorkloadKubeconfigPath` +- `clusterAPICloudConfigPath` + +## Uninstalling the Chart + +To uninstall `my-release`: + +```console +$ helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +> **Tip**: List all releases using `helm list` or start clean with `helm uninstall my-release` + +## Additional Configuration + +### AWS - IAM + +The worker running the cluster autoscaler will need access to certain resources and actions depending on the version you run and your configuration of it. + +For the up-to-date IAM permissions required, please see the [cluster autoscaler's AWS Cloudprovider Readme](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#iam-policy) and switch to the tag of the cluster autoscaler image you are using. + +### AWS - IAM Roles for Service Accounts (IRSA) + +For Kubernetes clusters that use Amazon EKS, the service account can be configured with an IAM role using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) to avoid needing to grant access to the worker nodes for AWS resources. + +In order to accomplish this, you will first need to create a new IAM role with the above mentions policies. Take care in [configuring the trust relationship](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html#iam-role-configuration) to restrict access just to the service account used by cluster autoscaler. + +Once you have the IAM role configured, you would then need to `--set rbac.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=arn:aws:iam::123456789012:role/MyRoleName` when installing. + +### Azure - Using azure workload identity + +You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure. + +You can also set the correct settings yourself instead of relying on this project. + +For example the following configuration will configure the Autoscaler to use your federated identity: + +```yaml +azureUseWorkloadIdentityExtension: true +extraEnv: + AZURE_CLIENT_ID: USER ASSIGNED IDENTITY CLIENT ID + AZURE_TENANT_ID: USER ASSIGNED IDENTITY TENANT ID + AZURE_FEDERATED_TOKEN_FILE: /var/run/secrets/tokens/azure-identity-token + AZURE_AUTHORITY_HOST: https://login.microsoftonline.com/ +extraVolumes: +- name: azure-identity-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api://AzureADTokenExchange + expirationSeconds: 3600 + path: azure-identity-token +extraVolumeMounts: +- mountPath: /var/run/secrets/tokens + name: azure-identity-token + readOnly: true +``` + +## Troubleshooting + +The chart will succeed even if the container arguments are incorrect. A few minutes after starting `kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like + +``` +polling_autoscaler.go:111] Poll finished +static_autoscaler.go:97] Starting main loop +utils.go:435] No pod using affinity / antiaffinity found in cluster, disabling affinity predicate for this loop +static_autoscaler.go:230] Filtering out schedulables +``` + +If not, find a pod that the deployment created and `describe` it, paying close attention to the arguments under `Command`. e.g.: + +``` +Containers: + cluster-autoscaler: + Command: + ./cluster-autoscaler + --cloud-provider=aws +# if specifying ASGs manually + --nodes=1:10:your-scaling-group-name +# if using autodiscovery + --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + --v=4 +``` + +### PodSecurityPolicy + +Though enough for the majority of installations, the default PodSecurityPolicy _could_ be too restrictive depending on the specifics of your release. Please make sure to check that the template fits with any customizations made or disable it by setting `rbac.pspEnabled` to `false`. + +### VerticalPodAutoscaler + +The CA Helm Chart can install a [`VerticalPodAutoscaler`](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/README.md) object from Chart version `9.27.0` +onwards for the Cluster Autoscaler Deployment to scale the CA as appropriate, but for that, we +need to install the VPA to the cluster separately. A VPA can help minimize wasted resources +when usage spikes periodically or remediate containers that are being OOMKilled. + +The following example snippet can be used to install VPA that allows scaling down from the default recommendations of the deployment template: + +```yaml +vpa: + enabled: true + containerPolicy: + minAllowed: + cpu: 20m + memory: 50Mi +``` + +{{ template "chart.valuesSection" . }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/NOTES.txt b/magnum_cluster_api/charts/cluster-autoscaler/templates/NOTES.txt new file mode 100644 index 00000000..94e211e9 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/NOTES.txt @@ -0,0 +1,18 @@ +{{- if or .Values.autoDiscovery.clusterName .Values.autoscalingGroups -}} + +To verify that cluster-autoscaler has started, run: + + kubectl --namespace={{ .Release.Namespace }} get pods -l "app.kubernetes.io/name={{ template "cluster-autoscaler.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + +{{- else -}} + +############################################################################## +#### ERROR: You must specify values for either #### +#### autoDiscovery.clusterName or autoscalingGroups[] #### +############################################################################## + +The deployment and pod will not be created and the installation is not functional +See README: + open https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler + +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/_helpers.tpl b/magnum_cluster_api/charts/cluster-autoscaler/templates/_helpers.tpl new file mode 100644 index 00000000..944fd1cf --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/_helpers.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-autoscaler.name" -}} +{{- default (printf "%s-%s" .Values.cloudProvider .Chart.Name) .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "cluster-autoscaler.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default (printf "%s-%s" .Values.cloudProvider .Chart.Name) .Values.nameOverride -}} +{{- if ne $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-autoscaler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return instance and name labels. +*/}} +{{- define "cluster-autoscaler.instance-name" -}} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/name: {{ include "cluster-autoscaler.name" . | quote }} +{{- end -}} + + +{{/* +Return labels, including instance and name. +*/}} +{{- define "cluster-autoscaler.labels" -}} +{{ include "cluster-autoscaler.instance-name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +helm.sh/chart: {{ include "cluster-autoscaler.chart" . | quote }} +{{- if .Values.additionalLabels }} +{{ toYaml .Values.additionalLabels }} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "deployment.apiVersion" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "<1.9-0" $kubeTargetVersion -}} +{{- print "apps/v1beta2" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podsecuritypolicy. +*/}} +{{- define "podsecuritypolicy.apiVersion" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "<1.10-0" $kubeTargetVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "podDisruptionBudget.apiVersion" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "<1.21-0" $kubeTargetVersion -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the service account name used by the pod. +*/}} +{{- define "cluster-autoscaler.serviceAccountName" -}} +{{- if .Values.rbac.serviceAccount.create -}} + {{ default (include "cluster-autoscaler.fullname" .) .Values.rbac.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.rbac.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return true if the priority expander is enabled +*/}} +{{- define "cluster-autoscaler.priorityExpanderEnabled" -}} +{{- $expanders := splitList "," (default "" .Values.extraArgs.expander) -}} +{{- if has "priority" $expanders -}} +{{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the autodiscoveryparameters for clusterapi. +*/}} +{{- define "cluster-autoscaler.capiAutodiscoveryConfig" -}} +{{- if .Values.autoDiscovery.clusterName -}} +{{- print "clusterName=" -}}{{ .Values.autoDiscovery.clusterName }} +{{- end -}} +{{- if and .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels -}} +{{- print "," -}} +{{- end -}} +{{- if .Values.autoDiscovery.labels -}} +{{- range $i, $el := .Values.autoDiscovery.labels -}} +{{- if $i -}}{{- print "," -}}{{- end -}} +{{- range $key, $val := $el -}} +{{- $key -}}{{- print "=" -}}{{- $val -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrole.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrole.yaml new file mode 100644 index 00000000..4ef27829 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrole.yaml @@ -0,0 +1,174 @@ +{{- if and .Values.rbac.create .Values.rbac.clusterScoped -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - events + - endpoints + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + - apiGroups: + - "" + resources: + - pods/status + verbs: + - update + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - cluster-autoscaler + verbs: + - get + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - watch + - list + - get + - update + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - replicationcontrollers + - persistentvolumeclaims + - persistentvolumes + verbs: + - watch + - list + - get + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - watch + - list + - get + - apiGroups: + - batch + - extensions + resources: + - jobs + verbs: + - get + - list + - patch + - watch + - apiGroups: + - extensions + resources: + - replicasets + - daemonsets + verbs: + - watch + - list + - get + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - watch + - list + - apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - watch + - list + - get + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + - csidrivers + - csistoragecapacities + verbs: + - watch + - list + - get + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update +{{- if .Values.rbac.pspEnabled }} + - apiGroups: + - extensions + - policy + resources: + - podsecuritypolicies + resourceNames: + - {{ template "cluster-autoscaler.fullname" . }} + verbs: + - use +{{- end -}} +{{- if and ( and ( eq .Values.cloudProvider "clusterapi" ) ( .Values.rbac.clusterScoped ) ( or ( eq .Values.clusterAPIMode "incluster-incluster" ) ( eq .Values.clusterAPIMode "kubeconfig-incluster" ) ))}} + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machinepools + - machinepools/scale + - machines + - machinesets + verbs: + - get + - list + - update + - watch + - apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - openstackmachinetemplates + verbs: + - get + - list + - update + - watch +{{- end }} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrolebinding.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..d2384dc6 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create .Values.rbac.clusterScoped -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "cluster-autoscaler.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "cluster-autoscaler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/deployment.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/deployment.yaml new file mode 100644 index 00000000..ea5ba5c4 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/deployment.yaml @@ -0,0 +1,315 @@ +{{- if or ( or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels ) .Values.autoscalingGroups }} +{{/* one of the above is required */}} +apiVersion: {{ template "deployment.apiVersion" . }} +kind: Deployment +metadata: + annotations: +{{ toYaml .Values.deployment.annotations | indent 4 }} + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: +{{ include "cluster-autoscaler.instance-name" . | indent 6 }} + {{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 6 }} + {{- end }} +{{- if .Values.updateStrategy }} + strategy: + {{ toYaml .Values.updateStrategy | nindent 4 | trim }} +{{- end }} + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + labels: +{{ include "cluster-autoscaler.instance-name" . | indent 8 }} + {{- if .Values.additionalLabels }} +{{ toYaml .Values.additionalLabels | indent 8 }} + {{- end }} + {{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: "{{ .Values.dnsPolicy }}" + {{- end }} + {{- if .Values.hostNetwork }} + hostNetwork: {{ .Values.hostNetwork }} + {{- end }} + containers: + - name: {{ template "cluster-autoscaler.name" . }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + command: + - ./cluster-autoscaler + - --cloud-provider={{ .Values.cloudProvider }} + {{- if and (eq .Values.cloudProvider "clusterapi") (eq .Values.clusterAPIMode "kubeconfig-incluster") }} + - --namespace={{ .Values.clusterAPIConfigMapsNamespace | default "kube-system" }} + {{- else }} + - --namespace={{ .Release.Namespace }} + {{- end }} + {{- if .Values.autoscalingGroups }} + {{- range .Values.autoscalingGroups }} + - --nodes={{ .minSize }}:{{ .maxSize }}:{{ .name }} + {{- end }} + {{- end }} + {{- if eq .Values.cloudProvider "rancher" }} + {{- if .Values.cloudConfigPath }} + - --cloud-config={{ .Values.cloudConfigPath }} + {{- end }} + {{- end }} + {{- if eq .Values.cloudProvider "aws" }} + {{- if .Values.autoDiscovery.clusterName }} + - --node-group-auto-discovery=asg:tag={{ tpl (join "," .Values.autoDiscovery.tags) . }} + {{- end }} + {{- if .Values.cloudConfigPath }} + - --cloud-config={{ .Values.cloudConfigPath }} + {{- end }} + {{- else if eq .Values.cloudProvider "gce" }} + {{- if .Values.autoscalingGroupsnamePrefix }} + {{- range .Values.autoscalingGroupsnamePrefix }} + - --node-group-auto-discovery=mig:namePrefix={{ .name }},min={{ .minSize }},max={{ .maxSize }} + {{- end }} + {{- end }} + {{- if eq .Values.cloudProvider "oci-oke" }} + {{- if .Values.cloudConfigPath }} + - --nodes={{ .minSize }}:{{ .maxSize }}:{{ .name }} + - --balance-similar-node-groups + {{- end }} + {{- end }} + {{- else if eq .Values.cloudProvider "magnum" }} + {{- if .Values.autoDiscovery.clusterName }} + - --cluster-name={{ .Values.autoDiscovery.clusterName }} + - --node-group-auto-discovery=magnum:role={{ tpl (join "," .Values.autoDiscovery.roles) . }} + {{- else }} + - --cluster-name={{ .Values.magnumClusterName }} + {{- end }} + {{- else if eq .Values.cloudProvider "clusterapi" }} + {{- if or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels }} + - --node-group-auto-discovery=clusterapi:{{ template "cluster-autoscaler.capiAutodiscoveryConfig" . }} + {{- end }} + {{- if eq .Values.clusterAPIMode "incluster-kubeconfig"}} + - --cloud-config={{ .Values.clusterAPICloudConfigPath }} + {{- else if eq .Values.clusterAPIMode "kubeconfig-incluster"}} + - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }} + - --clusterapi-cloud-config-authoritative + {{- else if eq .Values.clusterAPIMode "kubeconfig-kubeconfig"}} + - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }} + - --cloud-config={{ .Values.clusterAPICloudConfigPath }} + {{- else if eq .Values.clusterAPIMode "single-kubeconfig"}} + - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }} + {{- end }} + {{- else if eq .Values.cloudProvider "azure" }} + {{- if .Values.autoDiscovery.clusterName }} + - --node-group-auto-discovery=label:cluster-autoscaler-enabled=true,cluster-autoscaler-name={{ .Values.autoDiscovery.clusterName }} + {{- end }} + {{- end }} + {{- if eq .Values.cloudProvider "magnum" }} + - --cloud-config={{ .Values.cloudConfigPath }} + {{- end }} + {{- range $key, $value := .Values.extraArgs }} + {{- if not (kindIs "invalid" $value) }} + - --{{ $key | mustRegexFind "^[^_]+" }}={{ $value }} + {{- else }} + - --{{ $key | mustRegexFind "^[^_]+" }} + {{- end }} + {{- end }} + env: + {{- if and (eq .Values.cloudProvider "aws") (ne .Values.awsRegion "") }} + - name: AWS_REGION + value: "{{ .Values.awsRegion }}" + {{- if .Values.awsAccessKeyID }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AwsAccessKeyId + name: {{ template "cluster-autoscaler.fullname" . }} + {{- end }} + {{- if .Values.awsSecretAccessKey }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AwsSecretAccessKey + name: {{ template "cluster-autoscaler.fullname" . }} + {{- end }} + {{- else if eq .Values.cloudProvider "azure" }} + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: SubscriptionID + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: ResourceGroup + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_VM_TYPE + valueFrom: + secretKeyRef: + key: VMType + name: {{ template "cluster-autoscaler.fullname" . }} + - name: AZURE_CLUSTER_NAME + valueFrom: + secretKeyRef: + key: ClusterName + name: {{ template "cluster-autoscaler.fullname" . }} + {{- if .Values.azureUseWorkloadIdentityExtension }} + - name: ARM_USE_WORKLOAD_IDENTITY_EXTENSION + value: "true" + {{- else if .Values.azureUseManagedIdentityExtension }} + - name: ARM_USE_MANAGED_IDENTITY_EXTENSION + value: "true" + {{- else }} + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + key: TenantID + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + key: ClientID + name: {{ template "cluster-autoscaler.fullname" . }} + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: ClientSecret + name: {{ template "cluster-autoscaler.fullname" . }} + - name: AZURE_NODE_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: NodeResourceGroup + name: {{ template "cluster-autoscaler.fullname" . }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.extraEnv }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.extraEnvConfigMaps }} + - name: {{ $key }} + valueFrom: + configMapKeyRef: + name: {{ default (include "cluster-autoscaler.fullname" $) $value.name }} + key: {{ required "Must specify key!" $value.key }} + {{- end }} + {{- range $key, $value := .Values.extraEnvSecrets }} + - name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ default (include "cluster-autoscaler.fullname" $) $value.name }} + key: {{ required "Must specify key!" $value.key }} + {{- end }} + {{- if or .Values.envFromSecret .Values.envFromConfigMap }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ .Values.envFromSecret }} + {{- end }} + {{- if .Values.envFromConfigMap }} + - configMapRef: + name: {{ .Values.envFromConfigMap }} + {{- end }} + {{- end }} + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- if .Values.containerSecurityContext }} + securityContext: + {{ toYaml .Values.containerSecurityContext | nindent 12 | trim }} + {{- end }} + {{- if or (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumeMounts .Values.clusterAPIKubeconfigSecret }} + volumeMounts: + {{- if eq .Values.cloudProvider "magnum" }} + - name: cloudconfig + mountPath: {{ .Values.cloudConfigPath }} + readOnly: true + {{- end }} + {{- if and (eq .Values.cloudProvider "magnum") (.Values.magnumCABundlePath) }} + - name: ca-bundle + mountPath: {{ .Values.magnumCABundlePath }} + readOnly: true + {{- end }} + {{- range $key, $value := .Values.extraVolumeSecrets }} + - name: {{ $key }} + mountPath: {{ required "Must specify mountPath!" $value.mountPath }} + readOnly: true + {{- end }} + {{- if .Values.clusterAPIKubeconfigSecret }} + - name: cluster-api-kubeconfig + mountPath: {{ .Values.clusterAPIWorkloadKubeconfigPath | trimSuffix "/value" }} + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + serviceAccountName: {{ template "cluster-autoscaler.serviceAccountName" . }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if .Values.securityContext }} + securityContext: + {{ toYaml .Values.securityContext | nindent 8 | trim }} + {{- end }} + {{- if or (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumes .Values.clusterAPIKubeconfigSecret }} + volumes: + {{- if eq .Values.cloudProvider "magnum" }} + - name: cloudconfig + hostPath: + path: {{ .Values.cloudConfigPath }} + {{- end }} + {{- if and (eq .Values.cloudProvider "magnum") (.Values.magnumCABundlePath) }} + - name: ca-bundle + hostPath: + path: {{ .Values.magnumCABundlePath }} + {{- end }} + {{- range $key, $value := .Values.extraVolumeSecrets }} + - name: {{ $key }} + secret: + secretName: {{ default (include "cluster-autoscaler.fullname" $) $value.name }} + {{- if $value.items }} + items: + {{- toYaml $value.items | nindent 14 }} + {{- end }} + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.clusterAPIKubeconfigSecret }} + - name: cluster-api-kubeconfig + secret: + secretName: {{ .Values.clusterAPIKubeconfigSecret }} + {{- end }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/pdb.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/pdb.yaml new file mode 100644 index 00000000..8ad78209 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/pdb.yaml @@ -0,0 +1,16 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: {{ template "podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: +{{ include "cluster-autoscaler.instance-name" . | indent 6 }} +{{- if .Values.podDisruptionBudget }} + {{ toYaml .Values.podDisruptionBudget | nindent 2 }} +{{- end }} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000..e3ce5997 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml @@ -0,0 +1,42 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: {{ template "podsecuritypolicy.apiVersion" . }} +kind: PodSecurityPolicy +metadata: + name: {{ template "cluster-autoscaler.fullname" . }} + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} +spec: + # Prevents running in privileged mode + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'secret' + - 'hostPath' + - 'emptyDir' + - 'projected' + - 'downwardAPI' + hostNetwork: {{ .Values.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml new file mode 100644 index 00000000..8259f14f --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml @@ -0,0 +1,25 @@ +{{- if hasKey .Values.extraArgs "expander" }} +{{- if and (.Values.expanderPriorities) (include "cluster-autoscaler.priorityExpanderEnabled" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-autoscaler-priority-expander + namespace: {{ .Release.Namespace }} + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + {{- if .Values.priorityConfigMapAnnotations }} + annotations: +{{ toYaml .Values.priorityConfigMapAnnotations | indent 4 }} + {{- end }} +data: + priorities: |- +{{- if kindIs "string" .Values.expanderPriorities }} +{{ .Values.expanderPriorities | indent 4 }} +{{- else }} +{{- range $k,$v := .Values.expanderPriorities }} + {{ $k | int }}: + {{- toYaml $v | nindent 6 }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/prometheusrule.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/prometheusrule.yaml new file mode 100644 index 00000000..097c969e --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/prometheusrule.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "cluster-autoscaler.fullname" . }} + {{- if .Values.prometheusRule.namespace }} + namespace: {{ .Values.prometheusRule.namespace }} + {{- end }} + labels: {{- toYaml .Values.prometheusRule.additionalLabels | nindent 4 }} +spec: + groups: + - name: {{ include "cluster-autoscaler.fullname" . }} + interval: {{ .Values.prometheusRule.interval }} + rules: {{- tpl (toYaml .Values.prometheusRule.rules) . | nindent 8 }} +{{- end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/role.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/role.yaml new file mode 100644 index 00000000..1fe013fe --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/role.yaml @@ -0,0 +1,80 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create +{{- if (include "cluster-autoscaler.priorityExpanderEnabled" .) }} + - list + - watch +{{- end }} + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cluster-autoscaler-status +{{- if (include "cluster-autoscaler.priorityExpanderEnabled" .) }} + - cluster-autoscaler-priority-expander +{{- end }} + verbs: + - delete + - get + - update +{{- if (include "cluster-autoscaler.priorityExpanderEnabled" .) }} + - watch +{{- end }} +{{- if eq (default "" (index .Values.extraArgs "leader-elect-resource-lock")) "configmaps" }} + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cluster-autoscaler + verbs: + - get + - update +{{- end }} +{{- if and ( and ( eq .Values.cloudProvider "clusterapi" ) ( not .Values.rbac.clusterScoped ) ( or ( eq .Values.clusterAPIMode "incluster-incluster" ) ( eq .Values.clusterAPIMode "kubeconfig-incluster" ) ))}} + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machinepools + - machinepools/scale + - machines + - machinesets + verbs: + - get + - list + - update + - watch +{{- end }} +{{- if ( not .Values.rbac.clusterScoped ) }} + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update +{{- end }} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/rolebinding.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/rolebinding.yaml new file mode 100644 index 00000000..ba5f0375 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "cluster-autoscaler.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "cluster-autoscaler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/secret.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/secret.yaml new file mode 100644 index 00000000..9c58d0fe --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/secret.yaml @@ -0,0 +1,21 @@ +{{- if or (eq .Values.cloudProvider "azure") (and (eq .Values.cloudProvider "aws") (not (has "" (list .Values.awsAccessKeyID .Values.awsSecretAccessKey)))) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +data: +{{- if eq .Values.cloudProvider "azure" }} + ClientID: "{{ .Values.azureClientID | b64enc }}" + ClientSecret: "{{ .Values.azureClientSecret | b64enc }}" + ResourceGroup: "{{ .Values.azureResourceGroup | b64enc }}" + SubscriptionID: "{{ .Values.azureSubscriptionID | b64enc }}" + TenantID: "{{ .Values.azureTenantID | b64enc }}" + VMType: "{{ .Values.azureVMType | b64enc }}" + ClusterName: "{{ .Values.azureClusterName | b64enc }}" + NodeResourceGroup: "{{ .Values.azureNodeResourceGroup | b64enc }}" +{{- else if eq .Values.cloudProvider "aws" }} + AwsAccessKeyId: "{{ .Values.awsAccessKeyID | b64enc }}" + AwsSecretAccessKey: "{{ .Values.awsSecretAccessKey | b64enc }}" +{{- end }} +{{- end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/service.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/service.yaml new file mode 100644 index 00000000..255aea44 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/service.yaml @@ -0,0 +1,39 @@ +{{- if .Values.service.create }} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: +{{- if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - port: {{ .Values.service.servicePort }} + protocol: TCP + targetPort: 8085 + name: {{ .Values.service.portName }} + selector: +{{ include "cluster-autoscaler.instance-name" . | indent 4 }} + type: "{{ .Values.service.type }}" +{{- end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/serviceaccount.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/serviceaccount.yaml new file mode 100644 index 00000000..29c2580c --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.rbac.create .Values.rbac.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- if .Values.rbac.serviceAccount.annotations }} + annotations: {{ toYaml .Values.rbac.serviceAccount.annotations | nindent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.rbac.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/servicemonitor.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/servicemonitor.yaml new file mode 100644 index 00000000..5468756f --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/servicemonitor.yaml @@ -0,0 +1,30 @@ +{{ if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "cluster-autoscaler.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} + {{- end }} + annotations: +{{ toYaml .Values.serviceMonitor.annotations | indent 4 }} + labels: + {{- range $key, $value := .Values.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + selector: + matchLabels: +{{ include "cluster-autoscaler.instance-name" . | indent 6 }} + endpoints: + - port: {{ .Values.service.portName }} + interval: {{ .Values.serviceMonitor.interval }} + path: {{ .Values.serviceMonitor.path }} + {{- if .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + namespaceSelector: + matchNames: + - {{.Release.Namespace}} +{{ end }} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/templates/vpa.yaml b/magnum_cluster_api/charts/cluster-autoscaler/templates/vpa.yaml new file mode 100644 index 00000000..b889beac --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/templates/vpa.yaml @@ -0,0 +1,20 @@ +{{- if .Values.vpa.enabled -}} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + targetRef: + apiVersion: {{ template "deployment.apiVersion" . }} + kind: Deployment + name: {{ template "cluster-autoscaler.fullname" . }} + updatePolicy: + updateMode: {{ .Values.vpa.updateMode | quote }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "cluster-autoscaler.name" . }} + {{- .Values.vpa.containerPolicy | toYaml | nindent 6 }} +{{- end -}} diff --git a/magnum_cluster_api/charts/cluster-autoscaler/values.yaml b/magnum_cluster_api/charts/cluster-autoscaler/values.yaml new file mode 100644 index 00000000..f414ae65 --- /dev/null +++ b/magnum_cluster_api/charts/cluster-autoscaler/values.yaml @@ -0,0 +1,398 @@ +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +# affinity -- Affinity for pod assignment +affinity: {} + +# additionalLabels -- Labels to add to each object of the chart. +additionalLabels: {} + +autoDiscovery: + # cloudProviders `aws`, `gce`, `azure`, `magnum` and `clusterapi` `oci-oke` are supported by auto-discovery at this time + # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup + + # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. + # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup. + # Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. + # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. + # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. + clusterName: # cluster.local + + # autoDiscovery.tags -- ASG tags to match, run through `tpl`. + tags: + - k8s.io/cluster-autoscaler/enabled + - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} + # - kubernetes.io/cluster/{{ .Values.autoDiscovery.clusterName }} + + # autoDiscovery.roles -- Magnum node group roles to match. + roles: + - worker + + # autoDiscovery.labels -- Cluster-API labels to match https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery + labels: [] + # - color: green + # - shape: circle +# autoscalingGroups -- For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: +# 
+# - name: asg1

+#   maxSize: 2

+#   minSize: 1
+#
+autoscalingGroups: [] +# - name: asg1 +# maxSize: 2 +# minSize: 1 +# - name: asg2 +# maxSize: 2 +# minSize: 1 + +# autoscalingGroupsnamePrefix -- For GCE. At least one element is required if not using `autoDiscovery`. For example: +# 
+# - name: ig01

+#   maxSize: 10

+#   minSize: 0
+#
+autoscalingGroupsnamePrefix: [] +# - name: ig01 +# maxSize: 10 +# minSize: 0 +# - name: ig02 +# maxSize: 10 +# minSize: 0 + +# awsAccessKeyID -- AWS access key ID ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) +awsAccessKeyID: "" + +# awsRegion -- AWS region (required if `cloudProvider=aws`) +awsRegion: us-east-1 + +# awsSecretAccessKey -- AWS access secret key ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) +awsSecretAccessKey: "" + +# azureClientID -- Service Principal ClientID with contributor permission to Cluster and Node ResourceGroup. +# Required if `cloudProvider=azure` +azureClientID: "" + +# azureClientSecret -- Service Principal ClientSecret with contributor permission to Cluster and Node ResourceGroup. +# Required if `cloudProvider=azure` +azureClientSecret: "" + +# azureClusterName -- Azure AKS cluster name. +# Required if `cloudProvider=azure` +azureClusterName: "" + +# azureNodeResourceGroup -- Azure resource group where the cluster's nodes are located, typically set as `MC___`. +# Required if `cloudProvider=azure` +azureNodeResourceGroup: "" + +# azureResourceGroup -- Azure resource group that the cluster is located. +# Required if `cloudProvider=azure` +azureResourceGroup: "" + +# azureSubscriptionID -- Azure subscription where the resources are located. +# Required if `cloudProvider=azure` +azureSubscriptionID: "" + +# azureTenantID -- Azure tenant where the resources are located. +# Required if `cloudProvider=azure` +azureTenantID: "" + +# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. +azureUseManagedIdentityExtension: false + +# azureUseWorkloadIdentityExtension -- Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. +azureUseWorkloadIdentityExtension: false + +# azureVMType -- Azure VM type. +azureVMType: "AKS" + +# cloudConfigPath -- Configuration file for cloud provider. +cloudConfigPath: "" + +# cloudProvider -- The cloud provider where the autoscaler runs. +# Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. +# `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. +# `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. +cloudProvider: aws + +# clusterAPICloudConfigPath -- Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig` +clusterAPICloudConfigPath: /etc/kubernetes/mgmt-kubeconfig + +# clusterAPIConfigMapsNamespace -- Namespace on the workload cluster to store Leader election and status configmaps +clusterAPIConfigMapsNamespace: "" + +# clusterAPIKubeconfigSecret -- Secret containing kubeconfig for connecting to Cluster API managed workloadcluster +# Required if `cloudProvider=clusterapi` and `clusterAPIMode=kubeconfig-kubeconfig,kubeconfig-incluster or incluster-kubeconfig` +clusterAPIKubeconfigSecret: "" + +# clusterAPIMode -- Cluster API mode, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#connecting-cluster-autoscaler-to-cluster-api-management-and-workload-clusters +# Syntax: workloadClusterMode-ManagementClusterMode +# for `kubeconfig-kubeconfig`, `incluster-kubeconfig` and `single-kubeconfig` you always must mount the external kubeconfig using either `extraVolumeSecrets` or `extraMounts` and `extraVolumes` +# if you dont set `clusterAPIKubeconfigSecret`and thus use an in-cluster config or want to use a non capi generated kubeconfig you must do so for the workload kubeconfig as well +clusterAPIMode: incluster-incluster # incluster-incluster, incluster-kubeconfig, kubeconfig-incluster, kubeconfig-kubeconfig, single-kubeconfig + +# clusterAPIWorkloadKubeconfigPath -- Path to kubeconfig for connecting to Cluster API managed workloadcluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or kubeconfig-incluster` +clusterAPIWorkloadKubeconfigPath: /etc/kubernetes/value + +# containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +containerSecurityContext: {} + # capabilities: + # drop: + # - ALL + +deployment: + # deployment.annotations -- Annotations to add to the Deployment object. + annotations: {} + +# dnsPolicy -- Defaults to `ClusterFirst`. Valid values are: +# `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. +# If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. +dnsPolicy: ClusterFirst + +# envFromConfigMap -- ConfigMap name to use as envFrom. +envFromConfigMap: "" + +# envFromSecret -- Secret name to use as envFrom. +envFromSecret: "" + +## Priorities Expander +# expanderPriorities -- The expanderPriorities is used if `extraArgs.expander` contains `priority` and expanderPriorities is also set with the priorities. +# If `extraArgs.expander` contains `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. +# See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md +expanderPriorities: {} + +# extraArgs -- Additional container arguments. +# Refer to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca for the full list of cluster autoscaler +# parameters and their default values. +# Everything after the first _ will be ignored allowing the use of multi-string arguments. +extraArgs: + logtostderr: true + stderrthreshold: info + v: 4 + # write-status-configmap: true + # status-config-map-name: cluster-autoscaler-status + # leader-elect: true + # leader-elect-resource-lock: endpoints + # skip-nodes-with-local-storage: true + # expander: random + # scale-down-enabled: true + # balance-similar-node-groups: true + # min-replica-count: 0 + # scale-down-utilization-threshold: 0.5 + # scale-down-non-empty-candidates-count: 30 + # max-node-provision-time: 15m0s + # scan-interval: 10s + # scale-down-delay-after-add: 10m + # scale-down-delay-after-delete: 0s + # scale-down-delay-after-failure: 3m + # scale-down-unneeded-time: 10m + # skip-nodes-with-system-pods: true + # balancing-ignore-label_1: first-label-to-ignore + # balancing-ignore-label_2: second-label-to-ignore + +# extraEnv -- Additional container environment variables. +extraEnv: {} + +# extraEnvConfigMaps -- Additional container environment variables from ConfigMaps. +extraEnvConfigMaps: {} + +# extraEnvSecrets -- Additional container environment variables from Secrets. +extraEnvSecrets: {} + +# extraVolumeMounts -- Additional volumes to mount. +extraVolumeMounts: [] + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +# extraVolumes -- Additional volumes. +extraVolumes: [] + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + +# extraVolumeSecrets -- Additional volumes to mount from Secrets. +extraVolumeSecrets: {} + # autoscaler-vol: + # mountPath: /data/autoscaler/ + # custom-vol: + # name: custom-secret + # mountPath: /data/custom/ + # items: + # - key: subkey + # path: mypath + +# fullnameOverride -- String to fully override `cluster-autoscaler.fullname` template. +fullnameOverride: "" + +# hostNetwork -- Whether to expose network interfaces of the host machine to pods. +hostNetwork: false + +image: + # image.repository -- Image repository + repository: registry.k8s.io/autoscaling/cluster-autoscaler + # image.tag -- Image tag + tag: v1.27.2 + # image.pullPolicy -- Image pull policy + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # image.pullSecrets -- Image pull secrets + pullSecrets: [] + # - myRegistrKeySecretName + +# kubeTargetVersionOverride -- Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. +kubeTargetVersionOverride: "" + +# magnumCABundlePath -- Path to the host's CA bundle, from `ca-file` in the cloud-config file. +magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt" + +# magnumClusterName -- Cluster name or ID in Magnum. +# Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. +magnumClusterName: "" + +# nameOverride -- String to partially override `cluster-autoscaler.fullname` template (will maintain the release name) +nameOverride: "" + +# nodeSelector -- Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. +nodeSelector: {} + +# podAnnotations -- Annotations to add to each pod. +podAnnotations: {} + +# podDisruptionBudget -- Pod disruption budget. +podDisruptionBudget: + maxUnavailable: 1 + # minAvailable: 2 + +# podLabels -- Labels to add to each pod. +podLabels: {} + +# priorityClassName -- priorityClassName +priorityClassName: "system-cluster-critical" + +# priorityConfigMapAnnotations -- Annotations to add to `cluster-autoscaler-priority-expander` ConfigMap. +priorityConfigMapAnnotations: {} + # key1: "value1" + # key2: "value2" + +## Custom PrometheusRule to be defined +## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart +## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions +prometheusRule: + # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule. + enabled: false + # prometheusRule.additionalLabels -- Additional labels to be set in metadata. + additionalLabels: {} + # prometheusRule.namespace -- Namespace which Prometheus is running in. + namespace: monitoring + # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). + interval: null + # prometheusRule.rules -- Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). + rules: [] + +rbac: + # rbac.create -- If `true`, create and use RBAC resources. + create: true + # rbac.pspEnabled -- If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. + # Must be used with `rbac.create` set to `true`. + pspEnabled: false + # rbac.clusterScoped -- if set to false will only provision RBAC to alter resources in the current namespace. Most useful for Cluster-API + clusterScoped: true + serviceAccount: + # rbac.serviceAccount.annotations -- Additional Service Account annotations. + annotations: {} + # rbac.serviceAccount.create -- If `true` and `rbac.create` is also true, a Service Account will be created. + create: true + # rbac.serviceAccount.name -- The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. + name: "" + # rbac.serviceAccount.automountServiceAccountToken -- Automount API credentials for a Service Account. + automountServiceAccountToken: true + +# replicaCount -- Desired number of pods +replicaCount: 1 + +# resources -- Pod resource requests and limits. +resources: {} + # limits: + # cpu: 100m + # memory: 300Mi + # requests: + # cpu: 100m + # memory: 300Mi + +# securityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) +securityContext: {} + # runAsNonRoot: true + # runAsUser: 1001 + # runAsGroup: 1001 + +service: + # service.create -- If `true`, a Service will be created. + create: true + # service.annotations -- Annotations to add to service + annotations: {} + # service.labels -- Labels to add to service + labels: {} + # service.externalIPs -- List of IP addresses at which the service is available. Ref: https://kubernetes.io/docs/user-guide/services/#external-ips. + externalIPs: [] + + # service.loadBalancerIP -- IP address to assign to load balancer (if supported). + loadBalancerIP: "" + # service.loadBalancerSourceRanges -- List of IP CIDRs allowed access to load balancer (if supported). + loadBalancerSourceRanges: [] + # service.servicePort -- Service port to expose. + servicePort: 8085 + # service.portName -- Name for service port. + portName: http + # service.type -- Type of service to create. + type: ClusterIP + +## Are you using Prometheus Operator? +serviceMonitor: + # serviceMonitor.enabled -- If true, creates a Prometheus Operator ServiceMonitor. + enabled: false + # serviceMonitor.interval -- Interval that Prometheus scrapes Cluster Autoscaler metrics. + interval: 10s + # serviceMonitor.namespace -- Namespace which Prometheus is running in. + namespace: monitoring + ## [Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#exporters) + # serviceMonitor.selector -- Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. + selector: + release: prometheus-operator + # serviceMonitor.path -- The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) + path: /metrics + # serviceMonitor.annotations -- Annotations to add to service monitor + annotations: {} + ## [RelabelConfig](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig) + # serviceMonitor.metricRelabelings -- MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: {} + +# tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6). +tolerations: [] + +# topologySpreadConstraints -- You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19). +topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: cluster-autoscaler + +# updateStrategy -- [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) +updateStrategy: {} + # rollingUpdate: + # maxSurge: 1 + # maxUnavailable: 0 + # type: RollingUpdate + +# vpa -- Configure a VerticalPodAutoscaler for the cluster-autoscaler Deployment. +vpa: + # vpa.enabled -- If true, creates a VerticalPodAutoscaler. + enabled: false + # vpa.updateMode -- [UpdateMode](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L124) + updateMode: "Auto" + # vpa.containerPolicy -- [ContainerResourcePolicy](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L159). The containerName is always et to the deployment's container name. This value is required if VPA is enabled. + containerPolicy: {} diff --git a/magnum_cluster_api/charts/patches/cluster-autoscaler/001-add-omt-to-clusterrole.patch b/magnum_cluster_api/charts/patches/cluster-autoscaler/001-add-omt-to-clusterrole.patch new file mode 100644 index 00000000..0f6e60ed --- /dev/null +++ b/magnum_cluster_api/charts/patches/cluster-autoscaler/001-add-omt-to-clusterrole.patch @@ -0,0 +1,19 @@ +diff --git a/cluster-autoscaler/templates/clusterrole.yaml b/cluster-autoscaler/templates/clusterrole.yaml +index 4ef2782..fb1dc2a 100644 +--- a/cluster-autoscaler/templates/clusterrole.yaml ++++ b/cluster-autoscaler/templates/clusterrole.yaml +@@ -170,5 +170,14 @@ rules: + - list + - update + - watch ++ - apiGroups: ++ - infrastructure.cluster.x-k8s.io ++ resources: ++ - openstackmachinetemplates ++ verbs: ++ - get ++ - list ++ - update ++ - watch + {{- end }} + {{- end -}} diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 19e9e7f7..df847727 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -75,7 +75,7 @@ def apply(self): release_name=self.cluster.stack_id, chart_ref=os.path.join( pkg_resources.resource_filename("magnum_cluster_api", "charts"), - "vendor/cluster-autoscaler/", + "cluster-autoscaler/", ), values={ "fullnameOverride": f"{self.cluster.stack_id}-autoscaler", @@ -207,7 +207,7 @@ def get_object(self) -> pykube.ConfigMap: pkg_resources.resource_filename( "magnum_cluster_api", "charts" ), - "vendor/cilium/", + "cilium/", ), values={ "cni": {"chainingMode": "portmap"}, diff --git a/tox.ini b/tox.ini index 3f59d1c9..c11a9e66 100644 --- a/tox.ini +++ b/tox.ini @@ -18,9 +18,18 @@ commands = commands = pytest magnum_cluster_api/tests/functional/ +[testenv:sync-charts] +skipsdist = True +deps = + chart-vendor +commands = + chart-vendor --charts-root magnum_cluster_api/charts + [testenv:linters] skipsdist = True deps = + {[testenv:sync-charts]deps} pre-commit commands = pre-commit run --all-files --show-diff-on-failure + chart-vendor --check --charts-root magnum_cluster_api/charts diff --git a/vendir.lock.yml b/vendir.lock.yml deleted file mode 100644 index 767fc7ce..00000000 --- a/vendir.lock.yml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: vendir.k14s.io/v1alpha1 -directories: -- contents: - - helmChart: - appVersion: 1.27.2 - version: 9.29.1 - path: cluster-autoscaler - - helmChart: - appVersion: 1.15.3 - version: 1.15.3 - path: cilium - path: magnum_cluster_api/charts/vendor -kind: LockConfig diff --git a/vendir.yml b/vendir.yml deleted file mode 100644 index f824dbc5..00000000 --- a/vendir.yml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: vendir.k14s.io/v1alpha1 -kind: Config -directories: - - path: magnum_cluster_api/charts/vendor - excludePaths: k8s-keystone-auth - contents: - - path: cluster-autoscaler - helmChart: - name: cluster-autoscaler - version: 9.29.1 - repository: - url: https://kubernetes.github.io/autoscaler - - path: cilium - helmChart: - name: cilium - version: 1.15.3 - repository: - url: https://helm.cilium.io/ diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index fafb1663..ceac5e2e 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -1,6 +1,9 @@ - job: name: magnum-cluster-api-linters parent: tox-linters + pre-run: zuul.d/playbooks/linters/pre.yml + required-projects: + - name: vexxhost/chart-vendor - job: name: magnum-cluster-api-unit diff --git a/hack/setup-vendir.sh b/zuul.d/playbooks/linters/pre.yml old mode 100755 new mode 100644 similarity index 67% rename from hack/setup-vendir.sh rename to zuul.d/playbooks/linters/pre.yml index 27ebacb2..bb407944 --- a/hack/setup-vendir.sh +++ b/zuul.d/playbooks/linters/pre.yml @@ -1,5 +1,3 @@ -#!/bin/bash -xe - # Copyright (c) 2024 VEXXHOST, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -14,10 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -# Versions to test -VENDIR_VERSION=${VENDIR_VERSION:-v0.40.0} - -# Install `vendir` CLI -curl -Lo /tmp/vendir https://github.com/carvel-dev/vendir/releases/download/${VENDIR_VERSION}/vendir-linux-amd64 -chmod +x /tmp/vendir -sudo mv /tmp/vendir /usr/local/bin/vendir +- hosts: all + roles: + - role: ensure-helm + helm_version: 3.13.3 + tasks: + - name: Install patchutils + become: true + ansible.builtin.package: + name: patchutils + state: present