From 5e41bbdcdf3495ec8633d3197500689c81387f54 Mon Sep 17 00:00:00 2001 From: Kuromesi <87558945+Kuromesi@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:54:32 +0800 Subject: [PATCH] add support for external certs (#108) Signed-off-by: Kuromesi --- versions/kruise/1.7.0-alpha.1/README.md | 71 ++++++++++++++++--- .../apps.kruise.io_statefulsets.yaml | 5 ++ .../1.7.0-alpha.1/templates/rbac_role.yaml | 8 ++- .../templates/webhookconfiguration.yaml | 18 +++-- versions/kruise/1.7.0-alpha.1/values.yaml | 8 +++ 5 files changed, 96 insertions(+), 14 deletions(-) diff --git a/versions/kruise/1.7.0-alpha.1/README.md b/versions/kruise/1.7.0-alpha.1/README.md index 09beea6..6730430 100644 --- a/versions/kruise/1.7.0-alpha.1/README.md +++ b/versions/kruise/1.7.0-alpha.1/README.md @@ -4,12 +4,20 @@ The following table lists the configurable parameters of the kruise chart and their default values. -### manager parameters +## setup parameters | Parameter | Description | Default | | ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | -| `featureGates` | Feature gates for Kruise, empty string means all enabled | ` ` | -| `installation.namespace` | namespace for kruise installation | `kruise-system` | +| `featureGates` | Feature gates for Kruise, empty string means all enabled | `""` | +| `installation.namespace` | Namespace for kruise installation | `kruise-system` | | `installation.createNamespace` | Whether to create the installation.namespace | `true` | +| `installation.roleListGroups` | ApiGroups which kruise is permit to list, default set to be all | `*` | +| `crds.managed` | Kruise will not install CRDs with chart if this is false | `true` | +| `imagePullSecrets` | The list of image pull secrets for kruise image | `[]` | + + +### manager parameters +| Parameter | Description | Default | +| ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | | `manager.log.level` | Log level that kruise-manager printed | `4` | | `manager.replicas` | Replicas of kruise-controller-manager deployment | `2` | | `manager.image.repository` | Repository for kruise-manager image | `openkruise/kruise-manager` | @@ -24,12 +32,8 @@ The following table lists the configurable parameters of the kruise chart and th | `manager.nodeAffinity` | Node affinity policy for kruise-manager pod | `{}` | | `manager.nodeSelector` | Node labels for kruise-manager pod | `{}` | | `manager.tolerations` | Tolerations for kruise-manager pod | `[]` | -| `webhookConfiguration.timeoutSeconds` | The timeoutSeconds for all webhook configuration | `30` | -| `crds.managed` | Kruise will not install CRDs with chart if this is false | `true` | | `manager.resyncPeriod` | Resync period of informer kruise-manager, defaults no resync | `0` | | `manager.hostNetwork` | Whether kruise-manager pod should run with hostnetwork | `false` | -| `imagePullSecrets` | The list of image pull secrets for kruise image | `false` | -| `enableKubeCacheMutationDetector` | Whether to enable KUBE_CACHE_MUTATION_DETECTOR | `false` | ### daemon parameters | Parameter | Description | Default | @@ -49,7 +53,15 @@ The following table lists the configurable parameters of the kruise chart and th | `daemon.credentialProvider.hostPath` | credential provider plugin node dir, will volume mount into kruise-daemon | `credential-provider-plugin` | | `daemon.credentialProvider.configmap` | credential provider yaml configmap name in kruise-system ns | `credential-provider-config` | -Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, +### other parameters +| Parameter | Description | Default | +| ----------------------------------------- | ------------------------------------------------------------ | ----------------------------- | +| `externalCerts.annotations` | Annotations to patch for webhook configuration and crd when featuregate `EnableExternalCerts` is enabled. For example, `cert-manager.io/inject-ca-from: kruise-system/kruise-webhook-certs`. | `{}` | +| `enableKubeCacheMutationDetector` | Whether to enable KUBE_CACHE_MUTATION_DETECTOR | `false` | +| `webhookConfiguration.timeoutSeconds` | The timeoutSeconds for all webhook configuration | `30` | +| `serviceAccount.annotations` | Annotations to patch for serviceAccounts | `{}` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, `helm install kruise https://... --set featureGates="AllAlpha=true"`. ### Optional: feature-gate @@ -80,6 +92,7 @@ Feature-gate controls some influential features in Kruise: | `ImagePullJobGate` | ImagePullJobGate enable imagepulljob-controller execute ImagePullJob | `false` | ImagePullJob and PreDownloadImageForInPlaceUpdate are disabled | | `ResourceDistributionGate` | ResourceDistributionGate enable resourcedistribution-controller execute ResourceDistribution. | `false` | ResourceDistribution disabled | | `DeletionProtectionForCRDCascadingGate` | DeletionProtectionForCRDCascadingGate enable deletionProtection for crd Cascading | `false` | CustomResourceDefinition deletion protection disabled | +| `EnableExternalCerts` | Using certs generated externally, cert-manager e.g., for webhook server | `false` | kruise-manager will generate self-signed certs for webhook server | If you want to configure the feature-gate, just set the parameter when install or upgrade. Such as: @@ -98,3 +111,45 @@ If you are in China and have problem to pull image from official DockerHub, you $ helm install kruise https://... --set manager.image.repository=openkruise-registry.cn-hangzhou.cr.aliyuncs.com/openkruise/kruise-manager ... ``` + +### Optional: use certificates with certificate provisioner like cert-manager + +Kruise needs certificates to enable mutating, validating and conversion webhooks. By default, kruise will generate self-signed certificates for webhook server. If you want to use certificates provisioned externally, taking cert-manager as an example, you can follow these steps when install or upgrade: + +1. Enable kruise reading certs generated externally by setting `featureGates=EnableExternalCerts=true` when install or upgrade. +2. Create issuer and certificate resources if you have not done this before. +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kruise-webhook + # consistent with installation.namespace + namespace: kruise-system +spec: + # where to store the certificates + # cert-manager would generate a secret kruise-system/kruise-webhook-certs with the certificates + # DO NOT CHANGE THE SECRET NAME SINCE KRUISE READ CERTS FROM THIS SECRET + secretName: kruise-webhook-certs + dnsNames: + - kruise-webhook-service.kruise-system.svc + - localhost + issuerRef: + name: selfsigned-kruise + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-kruise + namespace: kruise-system +spec: + selfSigned: {} +``` +3. Set the parameter `externalCerts.annotations` to set annotations for crd and webhook configurations for cert-manager to recognize and patch. For example: +```yaml +externalCerts: + annotations: + # inject certificates from Certificate resource kruise-system/kruise-webhook-certs + cert-manager.io/inject-ca-from: kruise-system/kruise-webhook-certs +``` +Visit [CA Injector - cert manager](https://cert-manager.io/docs/concepts/ca-injector/) for more details. \ No newline at end of file diff --git a/versions/kruise/1.7.0-alpha.1/templates/apps.kruise.io_statefulsets.yaml b/versions/kruise/1.7.0-alpha.1/templates/apps.kruise.io_statefulsets.yaml index 1bb9109..30bf88f 100644 --- a/versions/kruise/1.7.0-alpha.1/templates/apps.kruise.io_statefulsets.yaml +++ b/versions/kruise/1.7.0-alpha.1/templates/apps.kruise.io_statefulsets.yaml @@ -6,6 +6,11 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.14.0 + {{- if contains "EnableExternalCerts=true" .Values.featureGates }} + {{- range $key, $value := .Values.externalCerts.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} name: statefulsets.apps.kruise.io spec: conversion: diff --git a/versions/kruise/1.7.0-alpha.1/templates/rbac_role.yaml b/versions/kruise/1.7.0-alpha.1/templates/rbac_role.yaml index 3d7216d..5681c81 100644 --- a/versions/kruise/1.7.0-alpha.1/templates/rbac_role.yaml +++ b/versions/kruise/1.7.0-alpha.1/templates/rbac_role.yaml @@ -144,9 +144,11 @@ rules: verbs: - get - list + - watch + {{- if not (contains "EnableExternalCerts=true" .Values.featureGates) }} - patch - update - - watch + {{- end }} - apiGroups: - admissionregistration.k8s.io resources: @@ -154,9 +156,11 @@ rules: verbs: - get - list + - watch + {{- if not (contains "EnableExternalCerts=true" .Values.featureGates) }} - patch - update - - watch + {{- end }} - apiGroups: - apiextensions.k8s.io resources: diff --git a/versions/kruise/1.7.0-alpha.1/templates/webhookconfiguration.yaml b/versions/kruise/1.7.0-alpha.1/templates/webhookconfiguration.yaml index ecbdd8d..a5dcc8f 100644 --- a/versions/kruise/1.7.0-alpha.1/templates/webhookconfiguration.yaml +++ b/versions/kruise/1.7.0-alpha.1/templates/webhookconfiguration.yaml @@ -3,9 +3,14 @@ kind: MutatingWebhookConfiguration metadata: name: kruise-mutating-webhook-configuration annotations: + {{- if contains "EnableExternalCerts=true" .Values.featureGates }} + {{- range $key, $value := .Values.externalCerts.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} template: "" webhooks: -{{ if not (contains "PodWebhook=false" .Values.featureGates) }} +{{- if not (contains "PodWebhook=false" .Values.featureGates) }} - admissionReviewVersions: - v1 - v1beta1 @@ -37,7 +42,7 @@ webhooks: resources: - pods sideEffects: None -{{ end }} +{{- end }} - admissionReviewVersions: - v1 - v1beta1 @@ -287,6 +292,11 @@ kind: ValidatingWebhookConfiguration metadata: name: kruise-validating-webhook-configuration annotations: + {{- if contains "EnableExternalCerts=true" .Values.featureGates }} + {{- range $key, $value := .Values.externalCerts.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} template: "" webhooks: - admissionReviewVersions: @@ -508,7 +518,7 @@ webhooks: resources: - services sideEffects: None -{{ if not (contains "PodWebhook=false" .Values.featureGates) }} +{{- if not (contains "PodWebhook=false" .Values.featureGates) }} - admissionReviewVersions: - v1 - v1beta1 @@ -572,7 +582,7 @@ webhooks: resources: - pods/eviction sideEffects: None -{{ end }} +{{- end }} - admissionReviewVersions: - v1 - v1beta1 diff --git a/versions/kruise/1.7.0-alpha.1/values.yaml b/versions/kruise/1.7.0-alpha.1/values.yaml index 93b63f2..547f286 100644 --- a/versions/kruise/1.7.0-alpha.1/values.yaml +++ b/versions/kruise/1.7.0-alpha.1/values.yaml @@ -12,9 +12,17 @@ installation: featureGates: "ImagePullJobGate=true" +externalCerts: + # annotations to patch for webhook configuration and crd + # e.g. cert-manager.io/inject-ca-from: kruise-system/kruise-webhook + annotations: {} + # KUBE_CACHE_MUTATION_DETECTOR enableKubeCacheMutationDetector: false +# imagePullSecrets to pull kruise images +imagePullSecrets: [] + manager: # settings for log print log: