From c5741409cca271b82e33e91e37e09b9ef5390d3f Mon Sep 17 00:00:00 2001 From: Aleksei Kobzev Date: Wed, 18 Oct 2023 13:09:32 +0700 Subject: [PATCH] e2e test with staticCreds --- api/v1alpha1/storage_webhook.go | 5 +- .../storage-block-4-2-config-staticCreds.yaml | 105 ++++++++++++++++++ e2e/tests/data/storage-block-4-2-config.yaml | 28 ++--- e2e/tests/smoke_test.go | 41 ++++++- e2e/tests/test-objects/objects.go | 55 ++++++++- internal/controllers/storage/init.go | 2 +- 6 files changed, 212 insertions(+), 24 deletions(-) create mode 100644 e2e/tests/data/storage-block-4-2-config-staticCreds.yaml diff --git a/api/v1alpha1/storage_webhook.go b/api/v1alpha1/storage_webhook.go index f3b343c5..849b1f36 100644 --- a/api/v1alpha1/storage_webhook.go +++ b/api/v1alpha1/storage_webhook.go @@ -29,10 +29,7 @@ var _ webhook.Defaulter = &Storage{} type PartialYamlConfig struct { DomainsConfig struct { SecurityConfig struct { - EnforceUserTokenRequirement bool `yaml:"enforce_user_token_requirement"` - MonitoringAllowedSIDs []string `yaml:"monitoring_allowed_sids"` - AdministrationAllowedSIDs []string `yaml:"administration_allowed_sids"` - VieweAllowedSIDs []string `yaml:"viewer_allowed_sids"` + EnforceUserTokenRequirement bool `yaml:"enforce_user_token_requirement"` } `yaml:"security_config"` } `yaml:"domains_config"` } diff --git a/e2e/tests/data/storage-block-4-2-config-staticCreds.yaml b/e2e/tests/data/storage-block-4-2-config-staticCreds.yaml new file mode 100644 index 00000000..efa13ae2 --- /dev/null +++ b/e2e/tests/data/storage-block-4-2-config-staticCreds.yaml @@ -0,0 +1,105 @@ +static_erasure: block-4-2 +host_configs: + - drive: + - path: SectorMap:1:1 + type: SSD + host_config_id: 1 +domains_config: + security_config: + enforce_user_token_requirement: true + domain: + - name: Root + storage_pool_types: + - kind: ssd + pool_config: + box_id: 1 + erasure_species: block-4-2 + kind: ssd + pdisk_filter: + - property: + - type: SSD + vdisk_kind: Default + state_storage: + - ring: + node: [1, 2, 3, 4, 5, 6, 7, 8] + nto_select: 5 + ssid: 1 +table_service_config: + sql_version: 1 +actor_system_config: + executor: + - name: System + threads: 1 + type: BASIC + - name: User + threads: 1 + type: BASIC + - name: Batch + threads: 1 + type: BASIC + - name: IO + threads: 1 + time_per_mailbox_micro_secs: 100 + type: IO + - name: IC + spin_threshold: 10 + threads: 4 + time_per_mailbox_micro_secs: 100 + type: BASIC + scheduler: + progress_threshold: 10000 + resolution: 256 + spin_threshold: 0 +blob_storage_config: + service_set: + groups: + - erasure_species: block-4-2 + rings: + - fail_domains: + - vdisk_locations: + - node_id: storage-0 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-1 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-2 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-3 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-4 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-5 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-6 + pdisk_category: SSD + path: SectorMap:1:1 + - vdisk_locations: + - node_id: storage-7 + pdisk_category: SSD + path: SectorMap:1:1 +channel_profile_config: + profile: + - channel: + - erasure_species: block-4-2 + pdisk_category: 1 + storage_pool_kind: ssd + - erasure_species: block-4-2 + pdisk_category: 1 + storage_pool_kind: ssd + - erasure_species: block-4-2 + pdisk_category: 1 + storage_pool_kind: ssd + profile_id: 0 +grpc_config: + port: 2135 diff --git a/e2e/tests/data/storage-block-4-2-config.yaml b/e2e/tests/data/storage-block-4-2-config.yaml index 420ecdc4..8e9f81cf 100644 --- a/e2e/tests/data/storage-block-4-2-config.yaml +++ b/e2e/tests/data/storage-block-4-2-config.yaml @@ -6,20 +6,16 @@ host_configs: host_config_id: 1 domains_config: domain: - # There can be only one root domain in a cluster. Domain name prefixes all scheme objects names, e.g. full name of a table table1 in database db1 - # in a cluster with domains_config.domain.name parameter set to Root would be equal to /Root/db1/table1 - name: Root storage_pool_types: - kind: ssd pool_config: box_id: 1 - # fault tolerance mode name - none, block-4-2, or mirror-3-dc. - # See docs for more details https://ydb.tech/en/docs/deploy/configuration/config#domains-blob erasure_species: block-4-2 kind: ssd pdisk_filter: - property: - - type: SSD # device type to match host_configs.drive.type + - type: SSD vdisk_kind: Default state_storage: - ring: @@ -29,34 +25,34 @@ domains_config: table_service_config: sql_version: 1 actor_system_config: - executor: - - name: System + executor: + - name: System threads: 1 type: BASIC - - name: User + - name: User threads: 1 type: BASIC - - name: Batch - threads: 1 + - name: Batch + threads: 1 type: BASIC - - name: IO + - name: IO threads: 1 time_per_mailbox_micro_secs: 100 type: IO - - name: IC + - name: IC spin_threshold: 10 - threads: 4 + threads: 4 time_per_mailbox_micro_secs: 100 type: BASIC scheduler: progress_threshold: 10000 resolution: 256 spin_threshold: 0 -blob_storage_config: # configuration of static blobstorage group. +blob_storage_config: service_set: groups: - - erasure_species: block-4-2 # fault tolerance mode name for the static group - rings: # in block-4-2 must have exactly 1 ring or availability zone. + - erasure_species: block-4-2 + rings: - fail_domains: - vdisk_locations: - node_id: storage-0 diff --git a/e2e/tests/smoke_test.go b/e2e/tests/smoke_test.go index 66dd6a29..3bc82504 100644 --- a/e2e/tests/smoke_test.go +++ b/e2e/tests/smoke_test.go @@ -110,7 +110,6 @@ var _ = Describe("Operator smoke test", func() { BeforeEach(func() { storageSample = testobjects.DefaultStorage(filepath.Join(".", "data", "storage-block-4-2-config.yaml")) - databaseSample = testobjects.DefaultDatabase() ctx = context.Background() @@ -214,6 +213,46 @@ var _ = Describe("Operator smoke test", func() { }) }) + It("storage webhook check, create storage withouth staticCredentials config", func() { + By("issuing create commands...") + storageSample = testobjects.StorageWithStaticCredentials(filepath.Join(".", "data", "storage-block-4-2-config.yaml")) + Expect(k8sClient.Create(ctx, storageSample)).ShouldNot(Succeed()) + }) + + It("operatorConnection check, create storage with default staticCredentials", func() { + By("issuing create commands...") + storageSample = testobjects.StorageWithStaticCredentials(filepath.Join(".", "data", "storage-block-4-2-config-staticCreds.yaml")) + Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed()) + defer func() { + Expect(k8sClient.Delete(ctx, storageSample)).Should(Succeed()) + }() + + storage := v1alpha1.Storage{} + Eventually(func(g Gomega) bool { + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name, + Namespace: testobjects.YdbNamespace, + }, &storage)).Should(Succeed()) + + return meta.IsStatusConditionPresentAndEqual( + storage.Status.Conditions, + "StorageReady", + metav1.ConditionTrue, + ) && storage.Status.State == testobjects.ReadyStatus + }, Timeout, Interval).Should(BeTrue()) + + By("checking that all the storage pods are running and ready...") + storagePods := corev1.PodList{} + Expect(k8sClient.List(ctx, &storagePods, client.InNamespace(testobjects.YdbNamespace), client.MatchingLabels{ + "ydb-cluster": "kind-storage", + })).Should(Succeed()) + Expect(len(storagePods.Items)).Should(BeEquivalentTo(storageSample.Spec.Nodes)) + for _, pod := range storagePods.Items { + Expect(pod.Status.Phase).To(BeEquivalentTo("Running")) + Expect(podIsReady(pod.Status.Conditions)).To(BeTrue()) + } + }) + It("storage.State goes Pending -> Preparing -> Provisioning -> Initializing -> Ready", func() { Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed()) defer func() { diff --git a/e2e/tests/test-objects/objects.go b/e2e/tests/test-objects/objects.go index b544dd5c..b980c16f 100644 --- a/e2e/tests/test-objects/objects.go +++ b/e2e/tests/test-objects/objects.go @@ -43,11 +43,58 @@ func constructAntiAffinityFor(key, value string) *corev1.Affinity { func DefaultStorage(storageYamlConfigPath string) *v1alpha1.Storage { storageConfig, err := os.ReadFile(storageYamlConfigPath) + Expect(err).To(BeNil()) defaultPolicy := corev1.PullIfNotPresent + storageAntiAffinity := constructAntiAffinityFor("ydb-cluster", "kind-storage") + return &v1alpha1.Storage{ + ObjectMeta: metav1.ObjectMeta{ + Name: StorageName, + Namespace: YdbNamespace, + }, + Spec: v1alpha1.StorageSpec{ + Nodes: 8, + Configuration: string(storageConfig), + Erasure: "block-4-2", + DataStore: []corev1.PersistentVolumeClaimSpec{}, + Service: v1alpha1.StorageServices{ + GRPC: v1alpha1.GRPCService{ + TLSConfiguration: &v1alpha1.TLSConfiguration{ + Enabled: false, + }, + Service: v1alpha1.Service{IPFamilies: []corev1.IPFamily{"IPv4"}}, + }, + Interconnect: v1alpha1.InterconnectService{ + TLSConfiguration: &v1alpha1.TLSConfiguration{ + Enabled: false, + }, + Service: v1alpha1.Service{IPFamilies: []corev1.IPFamily{"IPv4"}}, + }, + Status: v1alpha1.StatusService{ + Service: v1alpha1.Service{IPFamilies: []corev1.IPFamily{"IPv4"}}, + }, + }, + Domain: DefaultDomain, + Resources: corev1.ResourceRequirements{}, + Image: v1alpha1.PodImage{ + Name: YdbImage, + PullPolicyName: &defaultPolicy, + }, + AdditionalLabels: map[string]string{"ydb-cluster": "kind-storage"}, + Affinity: storageAntiAffinity, + Monitoring: &v1alpha1.MonitoringOptions{ + Enabled: false, + }, + }, + } +} + +func StorageWithStaticCredentials(storageYamlConfigPath string) *v1alpha1.Storage { + storageConfig, err := os.ReadFile(storageYamlConfigPath) Expect(err).To(BeNil()) + defaultPolicy := corev1.PullIfNotPresent storageAntiAffinity := constructAntiAffinityFor("ydb-cluster", "kind-storage") return &v1alpha1.Storage{ @@ -56,7 +103,12 @@ func DefaultStorage(storageYamlConfigPath string) *v1alpha1.Storage { Namespace: YdbNamespace, }, Spec: v1alpha1.StorageSpec{ - Nodes: 8, + Nodes: 8, + OperatorConnection: &v1alpha1.ConnectionOptions{ + StaticCredentials: &v1alpha1.StaticCredentialsAuth{ + Username: "root", + }, + }, Configuration: string(storageConfig), Erasure: "block-4-2", DataStore: []corev1.PersistentVolumeClaimSpec{}, @@ -94,7 +146,6 @@ func DefaultStorage(storageYamlConfigPath string) *v1alpha1.Storage { func DefaultDatabase() *v1alpha1.Database { defaultPolicy := corev1.PullIfNotPresent - databaseAntiAffinity := constructAntiAffinityFor("ydb-cluster", "kind-database") return &v1alpha1.Database{ diff --git a/internal/controllers/storage/init.go b/internal/controllers/storage/init.go index ebf00dfb..c8e7d926 100644 --- a/internal/controllers/storage/init.go +++ b/internal/controllers/storage/init.go @@ -115,7 +115,7 @@ func (r *Reconciler) initializeStorage( } if storage.Spec.OperatorConnection != nil { - ydbCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + ydbCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() token, err := creds.Token( metadata.AppendToOutgoingContext(ydbCtx, "x-ydb-database", storage.Spec.Domain),