From beedc37a8dc2b013c6a716b111a4be444b6d17e8 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 26 Sep 2024 09:46:54 +0800 Subject: [PATCH] chore: remove the Halt termination policy (#8202) --- apis/apps/v1/cluster_types.go | 12 +- apis/apps/v1/types.go | 1 - apis/apps/v1alpha1/cluster_conversion.go | 9 + .../bases/apps.kubeblocks.io_clusters.yaml | 6 +- controllers/apps/cluster_controller.go | 4 - controllers/apps/cluster_controller_test.go | 38 ---- controllers/apps/component_controller_test.go | 4 +- controllers/apps/transform_utils.go | 57 ------ .../apps/transformer_cluster_deletion.go | 11 +- controllers/apps/transformer_cluster_halt.go | 65 ------- .../transformer_cluster_halt_recovering.go | 175 ------------------ .../apps/transformer_component_deletion.go | 21 +-- .../crds/apps.kubeblocks.io_clusters.yaml | 6 +- docs/developer_docs/api-reference/cluster.md | 15 +- pkg/constant/annotations.go | 30 ++- pkg/controller/plan/restore.go | 14 -- 16 files changed, 37 insertions(+), 431 deletions(-) delete mode 100644 controllers/apps/transformer_cluster_halt.go delete mode 100644 controllers/apps/transformer_cluster_halt_recovering.go diff --git a/apis/apps/v1/cluster_types.go b/apis/apps/v1/cluster_types.go index 19cf1f4ca4b..a960a562f89 100644 --- a/apis/apps/v1/cluster_types.go +++ b/apis/apps/v1/cluster_types.go @@ -131,10 +131,7 @@ type ClusterSpec struct { // Choose a policy based on the desired level of resource cleanup and data preservation: // // - `DoNotTerminate`: Prevents deletion of the Cluster. This policy ensures that all resources remain intact. - // - `Halt`: Deletes Cluster resources like Pods and Services but retains Persistent Volume Claims (PVCs), - // allowing for data preservation while stopping other operations. - // - `Delete`: Extends the `Halt` policy by also removing PVCs, leading to a thorough cleanup while - // removing all persistent data. + // - `Delete`: Deletes all runtime resources belong to the Cluster. // - `WipeOut`: An aggressive policy that deletes all Cluster resources, including volume snapshots and // backups in external storage. // This results in complete data removal and should be used cautiously, primarily in non-production environments @@ -244,17 +241,14 @@ type ClusterStatus struct { // TerminationPolicyType defines termination policy types. // // +enum -// +kubebuilder:validation:Enum={DoNotTerminate,Halt,Delete,WipeOut} +// +kubebuilder:validation:Enum={DoNotTerminate,Delete,WipeOut} type TerminationPolicyType string const ( // DoNotTerminate will block delete operation. DoNotTerminate TerminationPolicyType = "DoNotTerminate" - // Halt will delete workload resources such as statefulset, deployment workloads but keep PVCs. - Halt TerminationPolicyType = "Halt" - - // Delete is based on Halt and deletes PVCs. + // Delete will delete all runtime resources belong to the cluster. Delete TerminationPolicyType = "Delete" // WipeOut is based on Delete and wipe out all volume snapshots and snapshot data from backup storage location. diff --git a/apis/apps/v1/types.go b/apis/apps/v1/types.go index 32e876c683c..fe559f01cd6 100644 --- a/apis/apps/v1/types.go +++ b/apis/apps/v1/types.go @@ -78,7 +78,6 @@ const ( ) const ( - ConditionTypeHaltRecovery = "HaltRecovery" // ConditionTypeHaltRecovery describe Halt recovery processing stage ConditionTypeProvisioningStarted = "ProvisioningStarted" // ConditionTypeProvisioningStarted the operator starts resource provisioning to create or change the cluster ConditionTypeApplyResources = "ApplyResources" // ConditionTypeApplyResources the operator start to apply resources to create or change the cluster ConditionTypeReplicasReady = "ReplicasReady" // ConditionTypeReplicasReady all pods of components are ready diff --git a/apis/apps/v1alpha1/cluster_conversion.go b/apis/apps/v1alpha1/cluster_conversion.go index 049cab8355f..237e4ba0b62 100644 --- a/apis/apps/v1alpha1/cluster_conversion.go +++ b/apis/apps/v1alpha1/cluster_conversion.go @@ -117,6 +117,11 @@ func (r *Cluster) changesToCluster(cluster *appsv1.Cluster) { if len(r.Spec.ClusterDefRef) > 0 { cluster.Spec.ClusterDef = r.Spec.ClusterDefRef } + if r.Spec.TerminationPolicy == Halt { + cluster.Spec.TerminationPolicy = appsv1.DoNotTerminate + } else { + cluster.Spec.TerminationPolicy = appsv1.TerminationPolicyType(r.Spec.TerminationPolicy) + } } func (r *Cluster) changesFromCluster(cluster *appsv1.Cluster) { @@ -140,6 +145,7 @@ func (r *Cluster) changesFromCluster(cluster *appsv1.Cluster) { if len(cluster.Spec.ClusterDef) > 0 { r.Spec.ClusterDefRef = cluster.Spec.ClusterDef } + // appsv1.TerminationPolicyType is a subset of appsv1alpha1.TerminationPolicyType, it can be converted directly. } type clusterConverter struct { @@ -149,6 +155,7 @@ type clusterConverter struct { type clusterSpecConverter struct { ClusterVersionRef string `json:"clusterVersionRef,omitempty"` + TerminationPolicy TerminationPolicyType `json:"terminationPolicy"` Affinity *Affinity `json:"affinity,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` Tenancy TenancyType `json:"tenancy,omitempty"` @@ -186,6 +193,7 @@ type clusterCompStatusConverter struct { func (c *clusterConverter) fromCluster(cluster *Cluster) { c.Spec.ClusterVersionRef = cluster.Spec.ClusterVersionRef + c.Spec.TerminationPolicy = cluster.Spec.TerminationPolicy c.Spec.Affinity = cluster.Spec.Affinity c.Spec.Tolerations = cluster.Spec.Tolerations c.Spec.Tenancy = cluster.Spec.Tenancy @@ -227,6 +235,7 @@ func (c *clusterConverter) fromCluster(cluster *Cluster) { func (c *clusterConverter) toCluster(cluster *Cluster) { cluster.Spec.ClusterVersionRef = c.Spec.ClusterVersionRef + cluster.Spec.TerminationPolicy = c.Spec.TerminationPolicy cluster.Spec.Affinity = c.Spec.Affinity cluster.Spec.Tolerations = c.Spec.Tolerations cluster.Spec.Tenancy = c.Spec.Tenancy diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index be5aad680b6..19f781797f1 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -15884,10 +15884,7 @@ spec: - `DoNotTerminate`: Prevents deletion of the Cluster. This policy ensures that all resources remain intact. - - `Halt`: Deletes Cluster resources like Pods and Services but retains Persistent Volume Claims (PVCs), - allowing for data preservation while stopping other operations. - - `Delete`: Extends the `Halt` policy by also removing PVCs, leading to a thorough cleanup while - removing all persistent data. + - `Delete`: Deletes all runtime resources belong to the Cluster. - `WipeOut`: An aggressive policy that deletes all Cluster resources, including volume snapshots and backups in external storage. This results in complete data removal and should be used cautiously, primarily in non-production environments @@ -15898,7 +15895,6 @@ spec: The `WipeOut` policy is particularly risky in production environments due to its irreversible nature. enum: - DoNotTerminate - - Halt - Delete - WipeOut type: string diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index 5ce9be0a065..45a0ca97c86 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -127,12 +127,8 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // TODO: transformers are vertices, theirs' dependencies are edges, make plan Build stage a DAG. plan, errBuild := planBuilder. AddTransformer( - // handle cluster halt first - &clusterHaltTransformer{}, // handle cluster deletion &clusterDeletionTransformer{}, - // check is recovering from halted cluster - &clusterHaltRecoveryTransformer{}, // update finalizer and cd&cv labels &clusterAssureMetaTransformer{}, // validate cd & cv's existence and availability diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index f3caaff2925..a8268666938 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -784,36 +784,6 @@ var _ = Describe("Cluster Controller", func() { } } - testDeleteClusterWithHalt := func(createObj func(appsv1.TerminationPolicyType)) { - createObj(appsv1.Halt) - - transCtx := &clusterTransformContext{ - Context: testCtx.Ctx, - Client: testCtx.Cli, - } - namespacedKinds, clusteredKinds := kindsForWipeOut() - allKinds := append(namespacedKinds, clusteredKinds...) - createdObjs, err := getOwningNamespacedObjects(transCtx.Context, transCtx.Client, clusterObj.Namespace, getAppInstanceML(*clusterObj), allKinds) - Expect(err).Should(Succeed()) - - By("delete the cluster") - testapps.DeleteObject(&testCtx, clusterKey, &appsv1.Cluster{}) - Consistently(testapps.CheckObjExists(&testCtx, clusterKey, &appsv1.Cluster{}, true)).Should(Succeed()) - - By("check all cluster resources again") - objs, err := getOwningNamespacedObjects(transCtx.Context, transCtx.Client, clusterObj.Namespace, getAppInstanceML(*clusterObj), allKinds) - Expect(err).Should(Succeed()) - // check all objects existed before cluster deletion still be there - for key, obj := range createdObjs { - Expect(objs).Should(HaveKey(key)) - Expect(obj.GetUID()).Should(BeEquivalentTo(objs[key].GetUID())) - } - } - - testClusterHaltNRecovery := func(createObj func(appsv1.TerminationPolicyType)) { - // TODO(component) - } - deleteClusterWithBackup := func(terminationPolicy appsv1.TerminationPolicyType, backupRetainPolicy string) { By("mocking a retained backup") backupPolicyName := "test-backup-policy" @@ -952,14 +922,6 @@ var _ = Describe("Cluster Controller", func() { testDeleteClusterWithDoNotTerminate(createObj) }) - It("delete cluster with terminationPolicy=Halt", func() { - testDeleteClusterWithHalt(createObj) - }) - - It("cluster Halt and Recovery", func() { - testClusterHaltNRecovery(createObj) - }) - It("delete cluster with terminationPolicy=Delete", func() { testDeleteClusterWithDelete(createObj) }) diff --git a/controllers/apps/component_controller_test.go b/controllers/apps/component_controller_test.go index e88b0de48d7..31a82de54f5 100644 --- a/controllers/apps/component_controller_test.go +++ b/controllers/apps/component_controller_test.go @@ -20,7 +20,6 @@ along with this program. If not, see . package apps import ( - "encoding/json" "fmt" "strconv" "strings" @@ -362,14 +361,13 @@ var _ = Describe("Component Controller", func() { if storageSize == "" { storageSize = "1Gi" } - clusterBytes, _ := json.Marshal(clusterObj) testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterName, compName, testapps.DataVolumeName). AddLabelsInMap(map[string]string{ constant.AppInstanceLabelKey: clusterName, constant.KBAppComponentLabelKey: compName, constant.AppManagedByLabelKey: constant.AppName, - }).AddAnnotations(constant.LastAppliedClusterAnnotationKey, string(clusterBytes)). + }). SetStorage(storageSize). SetStorageClass(storageClassName). CheckedCreate(&testCtx) diff --git a/controllers/apps/transform_utils.go b/controllers/apps/transform_utils.go index ae3ff8a6ffa..03e4c1a04c3 100644 --- a/controllers/apps/transform_utils.go +++ b/controllers/apps/transform_utils.go @@ -21,23 +21,18 @@ package apps import ( "context" - "encoding/json" "reflect" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controller/graph" - "github.com/apecloud/kubeblocks/pkg/controller/model" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" ) @@ -167,58 +162,6 @@ func isVolumeClaimTemplatesEqual(a, b []appsv1.ClusterComponentVolumeClaimTempla return true } -func preserveObjects[T client.Object](ctx context.Context, cli client.Reader, graphCli model.GraphClient, dag *graph.DAG, - obj T, ml client.MatchingLabels, toPreserveKinds []client.ObjectList, finalizerName string, lastApplyAnnotationKey string) error { - if len(toPreserveKinds) == 0 { - return nil - } - - objs, err := getOwningNamespacedObjects(ctx, cli, obj.GetNamespace(), ml, toPreserveKinds) - if err != nil { - return err - } - - objSpec := obj.DeepCopyObject().(client.Object) - objSpec.SetNamespace("") - objSpec.SetName(obj.GetName()) - objSpec.SetUID(obj.GetUID()) - objSpec.SetResourceVersion("") - objSpec.SetGeneration(0) - objSpec.SetManagedFields(nil) - - b, err := json.Marshal(objSpec) - if err != nil { - return err - } - objJSON := string(b) - - for _, o := range objs { - origObj := o.DeepCopyObject().(client.Object) - controllerutil.RemoveFinalizer(o, finalizerName) - removeOwnerRefOfType(o, obj.GetObjectKind().GroupVersionKind()) - - annot := o.GetAnnotations() - if annot == nil { - annot = make(map[string]string) - } - annot[lastApplyAnnotationKey] = objJSON - o.SetAnnotations(annot) - graphCli.Update(dag, origObj, o) - } - return nil -} - -func removeOwnerRefOfType(obj client.Object, gvk schema.GroupVersionKind) { - ownerRefs := obj.GetOwnerReferences() - for i, ref := range ownerRefs { - if ref.Kind == gvk.Kind && ref.APIVersion == gvk.GroupVersion().String() { - ownerRefs = append(ownerRefs[:i], ownerRefs[i+1:]...) - break - } - } - obj.SetOwnerReferences(ownerRefs) -} - // isOwnedByComp is used to judge if the obj is owned by Component. func isOwnedByComp(obj client.Object) bool { for _, ref := range obj.GetOwnerReferences() { diff --git a/controllers/apps/transformer_cluster_deletion.go b/controllers/apps/transformer_cluster_deletion.go index 6625cf59d93..c8edd658296 100644 --- a/controllers/apps/transformer_cluster_deletion.go +++ b/controllers/apps/transformer_cluster_deletion.go @@ -62,10 +62,6 @@ func (t *clusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) return graph.ErrPrematureStop - case kbappsv1.Halt: - transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "Halt", - "spec.terminationPolicy %s is preventing deletion. Halt policy is deprecated is 0.9.1 and will have same meaning as DoNotTerminate.", cluster.Spec.TerminationPolicy) - return graph.ErrPrematureStop case kbappsv1.Delete: toDeleteNamespacedKinds, toDeleteNonNamespacedKinds = kindsForDelete() case kbappsv1.WipeOut: @@ -149,7 +145,7 @@ func kindsForDoNotTerminate() ([]client.ObjectList, []client.ObjectList) { return []client.ObjectList{}, []client.ObjectList{} } -func kindsForHalt() ([]client.ObjectList, []client.ObjectList) { +func kindsForDelete() ([]client.ObjectList, []client.ObjectList) { namespacedKinds, nonNamespacedKinds := kindsForDoNotTerminate() namespacedKindsPlus := []client.ObjectList{ &kbappsv1.ComponentList{}, @@ -161,11 +157,6 @@ func kindsForHalt() ([]client.ObjectList, []client.ObjectList) { return append(namespacedKinds, namespacedKindsPlus...), nonNamespacedKinds } -func kindsForDelete() ([]client.ObjectList, []client.ObjectList) { - namespacedKinds, nonNamespacedKinds := kindsForHalt() - return append(namespacedKinds, haltPreserveKinds()...), nonNamespacedKinds -} - func kindsForWipeOut() ([]client.ObjectList, []client.ObjectList) { namespacedKinds, nonNamespacedKinds := kindsForDelete() namespacedKindsPlus := []client.ObjectList{ diff --git a/controllers/apps/transformer_cluster_halt.go b/controllers/apps/transformer_cluster_halt.go deleted file mode 100644 index 2e547063cc4..00000000000 --- a/controllers/apps/transformer_cluster_halt.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package apps - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controller/graph" - "github.com/apecloud/kubeblocks/pkg/controller/model" -) - -type clusterHaltTransformer struct{} - -var _ graph.Transformer = &clusterHaltTransformer{} - -func (t *clusterHaltTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { - transCtx, _ := ctx.(*clusterTransformContext) - cluster := transCtx.OrigCluster - if !cluster.IsDeleting() || cluster.Spec.TerminationPolicy != appsv1.Halt { - return nil - } - - var ( - graphCli, _ = transCtx.Client.(model.GraphClient) - ml = getAppInstanceML(*cluster) - toPreserveKinds = haltPreserveKinds() - ) - return preserveClusterObjects(transCtx.Context, transCtx.Client, graphCli, dag, cluster, ml, toPreserveKinds) -} - -func haltPreserveKinds() []client.ObjectList { - return []client.ObjectList{ - &corev1.PersistentVolumeClaimList{}, - &corev1.SecretList{}, - &corev1.ConfigMapList{}, - } -} - -// preserveClusterObjects preserves the objects owned by the cluster when the cluster is being deleted -func preserveClusterObjects(ctx context.Context, cli client.Reader, graphCli model.GraphClient, dag *graph.DAG, - cluster *appsv1.Cluster, ml client.MatchingLabels, toPreserveKinds []client.ObjectList) error { - return preserveObjects(ctx, cli, graphCli, dag, cluster, ml, toPreserveKinds, constant.DBClusterFinalizerName, constant.LastAppliedClusterAnnotationKey) -} diff --git a/controllers/apps/transformer_cluster_halt_recovering.go b/controllers/apps/transformer_cluster_halt_recovering.go deleted file mode 100644 index 3d35ee391c7..00000000000 --- a/controllers/apps/transformer_cluster_halt_recovering.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package apps - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controller/graph" -) - -type clusterHaltRecoveryTransformer struct{} - -var _ graph.Transformer = &clusterHaltRecoveryTransformer{} - -func (t *clusterHaltRecoveryTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { - transCtx, _ := ctx.(*clusterTransformContext) - cluster := transCtx.Cluster - - if cluster.Status.ObservedGeneration != 0 { - // skip handling for cluster.status.observedGeneration > 0 - return nil - } - - listOptions := []client.ListOption{ - client.InNamespace(cluster.Namespace), - client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.Name, - }, - } - - pvcList := &corev1.PersistentVolumeClaimList{} - if err := transCtx.Client.List(transCtx.Context, pvcList, listOptions...); err != nil { - return newRequeueError(requeueDuration, err.Error()) - } - - if len(pvcList.Items) == 0 { - return nil - } - - emitError := func(newCondition metav1.Condition) error { - if newCondition.LastTransitionTime.IsZero() { - newCondition.LastTransitionTime = metav1.Now() - } - newCondition.Status = metav1.ConditionFalse - oldCondition := meta.FindStatusCondition(cluster.Status.Conditions, newCondition.Type) - if oldCondition == nil { - cluster.Status.Conditions = append(cluster.Status.Conditions, newCondition) - } else { - *oldCondition = newCondition - } - transCtx.EventRecorder.Event(transCtx.Cluster, corev1.EventTypeWarning, newCondition.Reason, newCondition.Message) - return graph.ErrPrematureStop - } - - // halt recovering from last applied record stored in pvc's annotation - l, ok := pvcList.Items[0].Annotations[constant.LastAppliedClusterAnnotationKey] - if !ok || l == "" { - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "UncleanedResources", - Message: fmt.Sprintf("found uncleaned resources, requires manual deletion, check with `kubectl -n %s get pvc,secret,cm -l %s=%s`", - cluster.Namespace, constant.AppInstanceLabelKey, cluster.Name), - }) - } - - lc := &appsv1.Cluster{} - if err := json.Unmarshal([]byte(l), lc); err != nil { - return newRequeueError(requeueDuration, err.Error()) - } - - // skip if same cluster UID - if lc.UID == cluster.UID { - return nil - } - - // check clusterDef equality - if cluster.Spec.ClusterDef != lc.Spec.ClusterDef { - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "HaltRecoveryFailed", - Message: fmt.Sprintf("not equal to last applied cluster.spec.clusterDef %s", lc.Spec.ClusterDef), - }) - } - - // check component len equality - if l := len(lc.Spec.ComponentSpecs); l != len(cluster.Spec.ComponentSpecs) { - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "HaltRecoveryFailed", - Message: fmt.Sprintf("inconsistent spec.componentSpecs counts to last applied cluster.spec.componentSpecs (len=%d)", l), - }) - } - - // check every components' equality - for _, comp := range cluster.Spec.ComponentSpecs { - found := false - for _, lastUsedComp := range lc.Spec.ComponentSpecs { - // only need to verify [name, componentDefRef, replicas] for equality - if comp.Name != lastUsedComp.Name { - continue - } - if comp.Replicas != lastUsedComp.Replicas { - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "HaltRecoveryFailed", - Message: fmt.Sprintf("not equal to last applied cluster.spec.componentSpecs[%s].replicas=%d", - comp.Name, lastUsedComp.Replicas), - }) - } - - // following only check resource related spec., will skip check if HaltRecoveryAllowInconsistentResAnnotKey - // annotation is specified - if cluster.Annotations[constant.HaltRecoveryAllowInconsistentResAnnotKey] == trueVal { - found = true - break - } - if !isVolumeClaimTemplatesEqual(comp.VolumeClaimTemplates, lastUsedComp.VolumeClaimTemplates) { - objJSON, _ := json.Marshal(&lastUsedComp.VolumeClaimTemplates) - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "HaltRecoveryFailed", - Message: fmt.Sprintf("not equal to last applied cluster.spec.componentSpecs[%s].volumeClaimTemplates=%s; add '%s=true' annotation to void this check", - comp.Name, objJSON, constant.HaltRecoveryAllowInconsistentResAnnotKey), - }) - } - - if !isResourceRequirementsEqual(comp.Resources, lastUsedComp.Resources) { - objJSON, _ := json.Marshal(&lastUsedComp.Resources) - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "HaltRecoveryFailed", - Message: fmt.Sprintf("not equal to last applied cluster.spec.componentSpecs[%s].resources=%s; add '%s=true' annotation to void this check", - comp.Name, objJSON, constant.HaltRecoveryAllowInconsistentResAnnotKey), - }) - } - - found = true - break - } - if !found { - return emitError(metav1.Condition{ - Type: appsv1.ConditionTypeHaltRecovery, - Reason: "HaltRecoveryFailed", - Message: fmt.Sprintf("cluster.spec.componentSpecs[%s] not found in last applied cluster", - comp.Name), - }) - } - } - return nil -} diff --git a/controllers/apps/transformer_component_deletion.go b/controllers/apps/transformer_component_deletion.go index 32071bd7b62..8c77b9a5fef 100644 --- a/controllers/apps/transformer_component_deletion.go +++ b/controllers/apps/transformer_component_deletion.go @@ -167,6 +167,9 @@ func compOwnedKinds() []client.ObjectList { return []client.ObjectList{ &workloads.InstanceSetList{}, &corev1.ServiceList{}, + &corev1.SecretList{}, + &corev1.ConfigMapList{}, + &corev1.PersistentVolumeClaimList{}, &dpv1alpha1.BackupList{}, &dpv1alpha1.RestoreList{}, &appsv1alpha1.ConfigurationList{}, @@ -175,24 +178,8 @@ func compOwnedKinds() []client.ObjectList { } } -func compOwnedPreserveKinds() []client.ObjectList { - return []client.ObjectList{ - &corev1.SecretList{}, - &corev1.ConfigMapList{}, - &corev1.PersistentVolumeClaimList{}, - } -} - -func kindsForCompDoNotTerminate() []client.ObjectList { - return []client.ObjectList{} -} - -func kindsForCompHalt() []client.ObjectList { - return append(kindsForCompDoNotTerminate(), compOwnedKinds()...) -} - func kindsForCompDelete() []client.ObjectList { - return append(kindsForCompHalt(), compOwnedPreserveKinds()...) + return compOwnedKinds() } func kindsForCompWipeOut() []client.ObjectList { diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index be5aad680b6..19f781797f1 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -15884,10 +15884,7 @@ spec: - `DoNotTerminate`: Prevents deletion of the Cluster. This policy ensures that all resources remain intact. - - `Halt`: Deletes Cluster resources like Pods and Services but retains Persistent Volume Claims (PVCs), - allowing for data preservation while stopping other operations. - - `Delete`: Extends the `Halt` policy by also removing PVCs, leading to a thorough cleanup while - removing all persistent data. + - `Delete`: Deletes all runtime resources belong to the Cluster. - `WipeOut`: An aggressive policy that deletes all Cluster resources, including volume snapshots and backups in external storage. This results in complete data removal and should be used cautiously, primarily in non-production environments @@ -15898,7 +15895,6 @@ spec: The `WipeOut` policy is particularly risky in production environments due to its irreversible nature. enum: - DoNotTerminate - - Halt - Delete - WipeOut type: string diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index 36714cb95fb..2d227199eb3 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -175,10 +175,7 @@ It defines how resources, data, and backups associated with a Cluster are manage Choose a policy based on the desired level of resource cleanup and data preservation:

  • DoNotTerminate: Prevents deletion of the Cluster. This policy ensures that all resources remain intact.
    • -
  • Halt: Deletes Cluster resources like Pods and Services but retains Persistent Volume Claims (PVCs), -allowing for data preservation while stopping other operations.
    • -
  • Delete: Extends the Halt policy by also removing PVCs, leading to a thorough cleanup while -removing all persistent data.
    • +
  • Delete: Deletes all runtime resources belong to the Cluster.
  • WipeOut: An aggressive policy that deletes all Cluster resources, including volume snapshots and backups in external storage. This results in complete data removal and should be used cautiously, primarily in non-production environments @@ -3187,10 +3184,7 @@ It defines how resources, data, and backups associated with a Cluster are manage Choose a policy based on the desired level of resource cleanup and data preservation:

    • DoNotTerminate: Prevents deletion of the Cluster. This policy ensures that all resources remain intact.
      • -
    • Halt: Deletes Cluster resources like Pods and Services but retains Persistent Volume Claims (PVCs), -allowing for data preservation while stopping other operations.
      • -
    • Delete: Extends the Halt policy by also removing PVCs, leading to a thorough cleanup while -removing all persistent data.
      • +
    • Delete: Deletes all runtime resources belong to the Cluster.
    • WipeOut: An aggressive policy that deletes all Cluster resources, including volume snapshots and backups in external storage. This results in complete data removal and should be used cautiously, primarily in non-production environments @@ -9701,14 +9695,11 @@ string

      "Delete"

      -

      Delete is based on Halt and deletes PVCs.

      +

      Delete will delete all runtime resources belong to the cluster.

      "DoNotTerminate"

      DoNotTerminate will block delete operation.

      -

      "Halt"

      -

      Halt will delete workload resources such as statefulset, deployment workloads but keep PVCs.

      -

      "WipeOut"

      WipeOut is based on Delete and wipe out all volume snapshots and snapshot data from backup storage location.

      diff --git a/pkg/constant/annotations.go b/pkg/constant/annotations.go index ad40dd9679f..c2dd7c352cc 100644 --- a/pkg/constant/annotations.go +++ b/pkg/constant/annotations.go @@ -25,22 +25,20 @@ const ( // annotations defined by KubeBlocks const ( - ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. - EncryptedSystemAccountsAnnotationKey = "kubeblocks.io/encrypted-system-accounts" // EncryptedSystemAccountsAnnotationKey saves the encrypted system accounts. - OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster - ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile - RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart - RestoreFromBackupAnnotationKey = "kubeblocks.io/restore-from-backup" - RestoreDoneAnnotationKey = "kubeblocks.io/restore-done" - BackupSourceTargetAnnotationKey = "kubeblocks.io/backup-source-target" // RestoreFromBackupAnnotationKey specifies the component to recover from the backup. - BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" - LastAppliedClusterAnnotationKey = "apps.kubeblocks.io/last-applied-cluster" - PVLastClaimPolicyAnnotationKey = "apps.kubeblocks.io/pv-last-claim-policy" - HaltRecoveryAllowInconsistentResAnnotKey = "clusters.apps.kubeblocks.io/allow-inconsistent-resource" - KubeBlocksGenerationKey = "kubeblocks.io/generation" - KBAppClusterUIDKey = "apps.kubeblocks.io/cluster-uid" - LastRoleSnapshotVersionAnnotationKey = "apps.kubeblocks.io/last-role-snapshot-version" - ComponentScaleInAnnotationKey = "apps.kubeblocks.io/component-scale-in" // ComponentScaleInAnnotationKey specifies whether the component is scaled in + ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. + EncryptedSystemAccountsAnnotationKey = "kubeblocks.io/encrypted-system-accounts" // EncryptedSystemAccountsAnnotationKey saves the encrypted system accounts. + OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster + ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile + RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart + RestoreFromBackupAnnotationKey = "kubeblocks.io/restore-from-backup" + RestoreDoneAnnotationKey = "kubeblocks.io/restore-done" + BackupSourceTargetAnnotationKey = "kubeblocks.io/backup-source-target" // RestoreFromBackupAnnotationKey specifies the component to recover from the backup. + BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" + PVLastClaimPolicyAnnotationKey = "apps.kubeblocks.io/pv-last-claim-policy" + KubeBlocksGenerationKey = "kubeblocks.io/generation" + KBAppClusterUIDKey = "apps.kubeblocks.io/cluster-uid" + LastRoleSnapshotVersionAnnotationKey = "apps.kubeblocks.io/last-role-snapshot-version" + ComponentScaleInAnnotationKey = "apps.kubeblocks.io/component-scale-in" // ComponentScaleInAnnotationKey specifies whether the component is scaled in // SkipImmutableCheckAnnotationKey specifies to skip the mutation check for the object. // The mutation check is only applied to the fields that are declared as immutable. diff --git a/pkg/controller/plan/restore.go b/pkg/controller/plan/restore.go index d9008c9732a..37b22ecb058 100644 --- a/pkg/controller/plan/restore.go +++ b/pkg/controller/plan/restore.go @@ -164,16 +164,6 @@ func (r *RestoreManager) BuildPrepareDataRestore(comp *component.SynthesizedComp if targetVolumes == nil { return nil, nil } - getClusterJSON := func() string { - clusterSpec := r.Cluster.DeepCopy() - clusterSpec.ObjectMeta = metav1.ObjectMeta{ - Name: clusterSpec.GetName(), - UID: clusterSpec.GetUID(), - } - clusterSpec.Status = appsv1.ClusterStatus{} - b, _ := json.Marshal(*clusterSpec) - return string(b) - } var templates []dpv1alpha1.RestoreVolumeClaim pvcLabels := constant.GetCompLabels(r.Cluster.Name, comp.Name) @@ -190,10 +180,6 @@ func (r *RestoreManager) BuildPrepareDataRestore(comp *component.SynthesizedComp ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: pvcLabels, - Annotations: map[string]string{ - // satisfy the detection of transformer_halt_recovering. - constant.LastAppliedClusterAnnotationKey: getClusterJSON(), - }, }, } // build pvc labels