From 3a4070428d576941d7fc893eaa8aec13efcdceaf Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Mon, 1 Jul 2024 13:10:58 +0300 Subject: [PATCH 1/3] use statefulset status instead of list pod by labels --- internal/controllers/database/sync.go | 35 +++------------- internal/controllers/databasenodeset/sync.go | 42 +++----------------- internal/controllers/storage/sync.go | 35 +++------------- internal/controllers/storagenodeset/sync.go | 32 ++------------- 4 files changed, 19 insertions(+), 125 deletions(-) diff --git a/internal/controllers/database/sync.go b/internal/controllers/database/sync.go index df0f6e0e..466f8f67 100644 --- a/internal/controllers/database/sync.go +++ b/internal/controllers/database/sync.go @@ -20,7 +20,6 @@ import ( "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck - "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) @@ -304,11 +303,11 @@ func (r *Reconciler) waitForStatefulSetToScale( return Continue, ctrl.Result{Requeue: false}, nil } - found := &appsv1.StatefulSet{} + foundStatefulSet := &appsv1.StatefulSet{} err := r.Get(ctx, types.NamespacedName{ Name: database.Name, Namespace: database.Namespace, - }, found) + }, foundStatefulSet) if err != nil { if apierrors.IsNotFound(err) { r.Recorder.Event( @@ -328,42 +327,18 @@ func (r *Reconciler) waitForStatefulSetToScale( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - podList := &corev1.PodList{} - opts := []client.ListOption{ - client.InNamespace(database.Namespace), - client.MatchingLabels{labels.StatefulsetComponent: database.Name}, - } - - err = r.List(ctx, podList, opts...) - if err != nil { - r.Recorder.Event( - database, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed to list StatefulSet pods: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } - - runningPods := 0 - for _, e := range podList.Items { - if resources.PodIsReady(e) { - runningPods++ - } - } - - if runningPods != int(database.Spec.Nodes) { + if foundStatefulSet.Status.ReadyReplicas != database.Spec.Nodes { r.Recorder.Event( database, corev1.EventTypeNormal, string(DatabaseProvisioning), - fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", runningPods, database.Spec.Nodes), + fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes), ) meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ Type: DatabaseProvisionedCondition, Status: metav1.ConditionFalse, Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running pods does not match expected: %d != %d", runningPods, database.Spec.Nodes), + Message: fmt.Sprintf("Number of running pods does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, database.Spec.Nodes), }) return r.updateStatus(ctx, database, DefaultRequeueDelay) } diff --git a/internal/controllers/databasenodeset/sync.go b/internal/controllers/databasenodeset/sync.go index 35e97b70..0e0b2cde 100644 --- a/internal/controllers/databasenodeset/sync.go +++ b/internal/controllers/databasenodeset/sync.go @@ -14,12 +14,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck - "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) @@ -204,11 +202,11 @@ func (r *Reconciler) waitForStatefulSetToScale( return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay) } - found := &appsv1.StatefulSet{} + foundStatefulSet := &appsv1.StatefulSet{} err := r.Get(ctx, types.NamespacedName{ Name: databaseNodeSet.Name, Namespace: databaseNodeSet.Namespace, - }, found) + }, foundStatefulSet) if err != nil { if apierrors.IsNotFound(err) { r.Recorder.Event( @@ -223,51 +221,23 @@ func (r *Reconciler) waitForStatefulSetToScale( databaseNodeSet, corev1.EventTypeWarning, "ControllerError", - fmt.Sprintf("Failed to get StatefulSets: %s", err), + fmt.Sprintf("Failed to get StatefulSet: %s", err), ) return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - matchingLabels := client.MatchingLabels{} - for k, v := range databaseNodeSet.Labels { - matchingLabels[k] = v - } - - podList := &corev1.PodList{} - opts := []client.ListOption{ - client.InNamespace(databaseNodeSet.Namespace), - client.MatchingLabels{labels.StatefulsetComponent: databaseNodeSet.Name}, - } - err = r.List(ctx, podList, opts...) - if err != nil { - r.Recorder.Event( - databaseNodeSet, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed to list StatefulSet pods: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } - - runningPods := 0 - for _, e := range podList.Items { - if resources.PodIsReady(e) { - runningPods++ - } - } - - if runningPods != int(databaseNodeSet.Spec.Nodes) { + if foundStatefulSet.Status.ReadyReplicas != databaseNodeSet.Spec.Nodes { r.Recorder.Event( databaseNodeSet, corev1.EventTypeNormal, string(DatabaseNodeSetProvisioning), - fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, databaseNodeSet.Spec.Nodes), + fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes), ) meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ Type: NodeSetProvisionedCondition, Status: metav1.ConditionFalse, Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", runningPods, databaseNodeSet.Spec.Nodes), + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, databaseNodeSet.Spec.Nodes), }) return r.updateStatus(ctx, databaseNodeSet, DefaultRequeueDelay) } diff --git a/internal/controllers/storage/sync.go b/internal/controllers/storage/sync.go index 9ece19d7..91a3593b 100644 --- a/internal/controllers/storage/sync.go +++ b/internal/controllers/storage/sync.go @@ -21,7 +21,6 @@ import ( "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck "github.com/ydb-platform/ydb-kubernetes-operator/internal/healthcheck" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) @@ -124,11 +123,11 @@ func (r *Reconciler) waitForStatefulSetToScale( return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay) } - found := &appsv1.StatefulSet{} + foundStatefulSet := &appsv1.StatefulSet{} err := r.Get(ctx, types.NamespacedName{ Name: storage.Name, Namespace: storage.Namespace, - }, found) + }, foundStatefulSet) if err != nil { if apierrors.IsNotFound(err) { r.Recorder.Event( @@ -148,42 +147,18 @@ func (r *Reconciler) waitForStatefulSetToScale( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - podList := &corev1.PodList{} - opts := []client.ListOption{ - client.InNamespace(storage.Namespace), - client.MatchingLabels{labels.StatefulsetComponent: storage.Name}, - } - - err = r.List(ctx, podList, opts...) - if err != nil { - r.Recorder.Event( - storage, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed to list StatefulSet pods: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } - - runningPods := 0 - for _, e := range podList.Items { - if resources.PodIsReady(e) { - runningPods++ - } - } - - if runningPods != int(storage.Spec.Nodes) { + if foundStatefulSet.Status.ReadyReplicas != storage.Spec.Nodes { r.Recorder.Event( storage, corev1.EventTypeNormal, string(StorageProvisioning), - fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, storage.Spec.Nodes), + fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes), ) meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ Type: StorageProvisionedCondition, Status: metav1.ConditionFalse, Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", runningPods, storage.Spec.Nodes), + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storage.Spec.Nodes), }) return r.updateStatus(ctx, storage, DefaultRequeueDelay) } diff --git a/internal/controllers/storagenodeset/sync.go b/internal/controllers/storagenodeset/sync.go index 953eae92..fc0b9ed4 100644 --- a/internal/controllers/storagenodeset/sync.go +++ b/internal/controllers/storagenodeset/sync.go @@ -14,12 +14,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck - "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) @@ -228,42 +226,18 @@ func (r *Reconciler) waitForStatefulSetToScale( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - podList := &corev1.PodList{} - opts := []client.ListOption{ - client.InNamespace(storageNodeSet.Namespace), - client.MatchingLabels{labels.StatefulsetComponent: storageNodeSet.Name}, - } - - err = r.List(ctx, podList, opts...) - if err != nil { - r.Recorder.Event( - storageNodeSet, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed to list StatefulSet pods: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } - - runningPods := 0 - for _, e := range podList.Items { - if resources.PodIsReady(e) { - runningPods++ - } - } - - if runningPods != int(storageNodeSet.Spec.Nodes) { + if foundStatefulSet.Status.ReadyReplicas != storageNodeSet.Spec.Nodes { r.Recorder.Event( storageNodeSet, corev1.EventTypeNormal, string(StorageNodeSetProvisioning), - fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, storageNodeSet.Spec.Nodes), + fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes), ) meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ Type: NodeSetProvisionedCondition, Status: metav1.ConditionFalse, Reason: ReasonInProgress, - Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", runningPods, storageNodeSet.Spec.Nodes), + Message: fmt.Sprintf("Number of running nodes does not match expected: %d != %d", foundStatefulSet.Status.ReadyReplicas, storageNodeSet.Spec.Nodes), }) return r.updateStatus(ctx, storageNodeSet, DefaultRequeueDelay) } From 94d6922c7bd1484a7feac928d6e9e41efc448680 Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Mon, 1 Jul 2024 13:11:27 +0300 Subject: [PATCH 2/3] fix missing "ydb.tech/remote-cluster" label --- internal/resources/database.go | 6 +++--- internal/resources/databasenodeset.go | 14 ++++++++++---- internal/resources/storage.go | 6 +++--- internal/resources/storagenodeset.go | 14 ++++++++++---- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/internal/resources/database.go b/internal/resources/database.go index d7fb2902..4276b3d0 100644 --- a/internal/resources/database.go +++ b/internal/resources/database.go @@ -200,6 +200,9 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res nodeSetLabels := databaseLabels.Copy() nodeSetLabels.Merge(nodeSetSpecInline.Labels) nodeSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: nodeSetSpecInline.Name}) + if nodeSetSpecInline.Remote != nil { + nodeSetLabels.Merge(map[string]string{labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster}) + } nodeSetAnnotations := CopyDict(b.Annotations) if nodeSetSpecInline.Annotations != nil { @@ -210,9 +213,6 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res databaseNodeSetSpec := b.recastDatabaseNodeSetSpecInline(nodeSetSpecInline.DeepCopy()) if nodeSetSpecInline.Remote != nil { - nodeSetLabels = nodeSetLabels.Merge(map[string]string{ - labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster, - }) nodeSetBuilders = append( nodeSetBuilders, &RemoteDatabaseNodeSetBuilder{ diff --git a/internal/resources/databasenodeset.go b/internal/resources/databasenodeset.go index 9e57519a..ce044d55 100644 --- a/internal/resources/databasenodeset.go +++ b/internal/resources/databasenodeset.go @@ -58,9 +58,15 @@ func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) [ ydbCr := api.RecastDatabaseNodeSet(b.Unwrap()) databaseLabels := labels.DatabaseLabels(ydbCr) + statefulSetName := b.Name statefulSetLabels := databaseLabels.Copy() - statefulSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: b.Labels[labels.DatabaseNodeSetComponent]}) - statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name}) + statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: statefulSetName}) + + databaseNodeSetName := b.Labels[labels.DatabaseNodeSetComponent] + statefulSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: databaseNodeSetName}) + if remoteCluster, exist := b.Labels[labels.RemoteClusterKey]; exist { + statefulSetLabels.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster}) + } statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) statefulSetAnnotations[annotations.ConfigurationChecksum] = GetConfigurationChecksum(b.Spec.Configuration) @@ -68,10 +74,10 @@ func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) [ var resourceBuilders []ResourceBuilder resourceBuilders = append(resourceBuilders, &DatabaseStatefulSetBuilder{ - Database: api.RecastDatabaseNodeSet(b.DatabaseNodeSet), + Database: ydbCr, RestConfig: restConfig, - Name: b.Name, + Name: statefulSetName, Labels: statefulSetLabels, Annotations: statefulSetAnnotations, }, diff --git a/internal/resources/storage.go b/internal/resources/storage.go index 63a173c7..38169a60 100644 --- a/internal/resources/storage.go +++ b/internal/resources/storage.go @@ -165,6 +165,9 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) nodeSetLabels := storageLabels.Copy() nodeSetLabels.Merge(nodeSetSpecInline.Labels) nodeSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: nodeSetSpecInline.Name}) + if nodeSetSpecInline.Remote != nil { + nodeSetLabels.Merge(map[string]string{labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster}) + } nodeSetAnnotations := CopyDict(b.Annotations) if nodeSetSpecInline.Annotations != nil { @@ -175,9 +178,6 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) storageNodeSetSpec := b.recastStorageNodeSetSpecInline(nodeSetSpecInline.DeepCopy()) if nodeSetSpecInline.Remote != nil { - nodeSetLabels = nodeSetLabels.Merge(map[string]string{ - labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster, - }) nodeSetBuilders = append( nodeSetBuilders, &RemoteStorageNodeSetBuilder{ diff --git a/internal/resources/storagenodeset.go b/internal/resources/storagenodeset.go index d206474f..0fd915a5 100644 --- a/internal/resources/storagenodeset.go +++ b/internal/resources/storagenodeset.go @@ -58,9 +58,15 @@ func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) [] ydbCr := api.RecastStorageNodeSet(b.Unwrap()) storageLabels := labels.StorageLabels(ydbCr) + statefulSetName := b.Name statefulSetLabels := storageLabels.Copy() - statefulSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: b.Labels[labels.DatabaseNodeSetComponent]}) - statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name}) + statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: statefulSetName}) + + storageNodeSetName := b.Labels[labels.StorageNodeSetComponent] + statefulSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: storageNodeSetName}) + if remoteCluster, exist := b.Labels[labels.RemoteClusterKey]; exist { + statefulSetLabels.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster}) + } statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) statefulSetAnnotations[annotations.ConfigurationChecksum] = GetConfigurationChecksum(b.Spec.Configuration) @@ -69,10 +75,10 @@ func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) [] resourceBuilders = append( resourceBuilders, &StorageStatefulSetBuilder{ - Storage: api.RecastStorageNodeSet(b.StorageNodeSet), + Storage: ydbCr, RestConfig: restConfig, - Name: b.Name, + Name: statefulSetName, Labels: statefulSetLabels, Annotations: statefulSetAnnotations, }, From 0fba44951e262ec3699cf7fcd1b33a1cf92c5ddb Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Mon, 1 Jul 2024 13:20:22 +0300 Subject: [PATCH 3/3] bump helm chart version --- deploy/ydb-operator/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index c1a658df..c1595dc3 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.17 +version: 0.5.18 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.5.17" +appVersion: "0.5.18"