From 1706e79e82beb8d8a4fdef1690e9c58e43c9bf01 Mon Sep 17 00:00:00 2001 From: razo7 Date: Thu, 27 Jul 2023 11:40:10 +0300 Subject: [PATCH] Add e2e tests for workload deletion Update e2e tests to create one VA and one pod which will be deleted by FAR pod --- test/e2e/far_e2e_test.go | 118 ++++++++++++++++++++++++++++++++------ test/e2e/utils/command.go | 10 +++- 2 files changed, 108 insertions(+), 20 deletions(-) diff --git a/test/e2e/far_e2e_test.go b/test/e2e/far_e2e_test.go index e53c2114..f073d029 100644 --- a/test/e2e/far_e2e_test.go +++ b/test/e2e/far_e2e_test.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -20,7 +21,6 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/medik8s/fence-agents-remediation/api/v1alpha1" - "github.com/medik8s/fence-agents-remediation/controllers" "github.com/medik8s/fence-agents-remediation/pkg/utils" e2eUtils "github.com/medik8s/fence-agents-remediation/test/e2e/utils" ) @@ -31,13 +31,19 @@ const ( fenceAgentAction = "reboot" nodeIdentifierPrefixAWS = "--plug" nodeIdentifierPrefixIPMI = "--ipport" + succeesRebootMessage = "\"Success: Rebooted" containerName = "manager" + testVolumeAttachment = "test-va" + testContainerName = "test-container" + testPodName = "test-pod" //TODO: try to minimize timeout // eventually parameters - timeoutLogs = 3 * time.Minute - timeoutReboot = 6 * time.Minute // fencing with fence_aws should be completed within 6 minutes - pollInterval = 10 * time.Second + timeoutLogs = 3 * time.Minute + timeoutReboot = 6 * time.Minute // fencing with fence_aws should be completed within 6 minutes + timeoutDeletion = 10 * time.Second // this timeout is used after all the other steps have been succesfult + pollDeletion = 250 * time.Millisecond + pollInterval = 10 * time.Second ) var previousNodeName string @@ -75,9 +81,12 @@ var _ = Describe("FAR E2e", func() { Context("stress cluster", func() { var ( - nodeName string - nodeBootTimeBefore time.Time - err error + err error + testNodeName string + va *storagev1.VolumeAttachment + pod *corev1.Pod + creationTimePod, nodeBootTimeBefore time.Time + far *v1alpha1.FenceAgentsRemediation ) BeforeEach(func() { nodes := &corev1.NodeList{} @@ -89,25 +98,37 @@ var _ = Describe("FAR E2e", func() { Fail("No worker nodes found in the cluster") } - nodeName = randomizeWorkerNode(nodes) - previousNodeName = nodeName - nodeNameParam := v1alpha1.NodeName(nodeName) + testNodeName = randomizeWorkerNode(nodes) + previousNodeName = testNodeName + nodeNameParam := v1alpha1.NodeName(testNodeName) parameterName := v1alpha1.ParameterName(nodeIdentifierPrefix) testNodeID := testNodeParam[parameterName][nodeNameParam] - log.Info("Testing Node", "Node name", nodeName, "Node ID", testNodeID) + log.Info("Testing Node", "Node name", testNodeName, "Node ID", testNodeID) // save the node's boot time prior to the fence agent call - nodeBootTimeBefore, err = e2eUtils.GetBootTime(clientSet, nodeName, testNsName, log) + nodeBootTimeBefore, err = e2eUtils.GetBootTime(clientSet, testNodeName, testNsName, log) Expect(err).ToNot(HaveOccurred(), "failed to get boot time of the node") - far := createFAR(nodeName, fenceAgent, testShareParam, testNodeParam) + + // create tested pod, and save it's creation time + // it will be deleted by FAR CR + pod = e2eUtils.GetPod(testNodeName, testContainerName) + pod.Name = testPodName + pod.Namespace = testNsName + Expect(k8sClient.Create(context.Background(), pod)).To(Succeed()) + log.Info("Tested pod has been created", "pod", testPodName) + creationTimePod = metav1.Now().Time + va = createVA(testNodeName) + DeferCleanup(verifyCleanSetup, va, pod) + + far = createFAR(testNodeName, fenceAgent, testShareParam, testNodeParam) DeferCleanup(deleteFAR, far) }) When("running FAR to reboot two nodes", func() { It("should successfully remediate the first node", func() { - checkRemediation(nodeName, nodeBootTimeBefore) + checkRemediation(testNodeName, nodeBootTimeBefore, creationTimePod, va, pod) }) It("should successfully remediate the second node", func() { - checkRemediation(nodeName, nodeBootTimeBefore) + checkRemediation(testNodeName, nodeBootTimeBefore, creationTimePod, va, pod) }) }) }) @@ -213,6 +234,27 @@ func randomizeWorkerNode(nodes *corev1.NodeList) string { return nodeName } +// createVA creates dummy volume attachment for testing the resource deletion +func createVA(nodeName string) *storagev1.VolumeAttachment { + pv := "test-pv" + va := &storagev1.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: testVolumeAttachment, + Namespace: testNsName, + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: pv, + Source: storagev1.VolumeAttachmentSource{}, + NodeName: nodeName, + }, + } + + va.Spec.Source.PersistentVolumeName = &pv + ExpectWithOffset(1, k8sClient.Create(context.Background(), va)).To(Succeed()) + log.Info("Volume attachment has been created", "va", va.Name) + return va +} + // createFAR assigns the input to FenceAgentsRemediation object, creates CR, and returns the CR object func createFAR(nodeName string, agent string, sharedParameters map[v1alpha1.ParameterName]string, nodeParameters map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string) *v1alpha1.FenceAgentsRemediation { far := &v1alpha1.FenceAgentsRemediation{ @@ -238,6 +280,21 @@ func deleteFAR(far *v1alpha1.FenceAgentsRemediation) { }, 2*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to delete far") } +// verifyCleanSetup deletes an old pod and old va if it was not deleted from FAR CR +func verifyCleanSetup(va *storagev1.VolumeAttachment, pod *corev1.Pod) { + newVa := &storagev1.VolumeAttachment{} + if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa); err == nil { + Expect(k8sClient.Delete(context.Background(), newVa)).To(Succeed()) + log.Info("cleanup: Volume attachment has not been deleted by remediation", "va name", va.Name) + } + + newPod := &corev1.Pod{} + if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod); err == nil { + Expect(k8sClient.Delete(context.Background(), newPod)).To(Succeed()) + log.Info("cleanup: Pod has not been deleted by remediation", "pod name", pod.Name) + } +} + // wasFarTaintAdded checks whether the FAR taint was added to the tested node func wasFarTaintAdded(nodeName string) { farTaint := utils.CreateFARNoExecuteTaint() @@ -291,8 +348,29 @@ func wasNodeRebooted(nodeName string, nodeBootTimeBefore time.Time) { log.Info("successful reboot", "node", nodeName, "offset between last boot", nodeBootTimeAfter.Sub(nodeBootTimeBefore), "new boot time", nodeBootTimeAfter) } +// checkVaDeleted verifies if the va has already been deleted due to resource deletion +func checkVaDeleted(va *storagev1.VolumeAttachment) { + EventuallyWithOffset(1, func() bool { + newVa := &storagev1.VolumeAttachment{} + err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa) + return apiErrors.IsNotFound(err) + + }, timeoutDeletion, pollDeletion).Should(BeTrue()) + log.Info("Volume Attachment has already been deleted", "va name", va.Name) +} + +// checkPodDeleted vefifies if the pod has already been deleted due to resource deletion +func checkPodDeleted(pod *corev1.Pod) { + ConsistentlyWithOffset(1, func() bool { + newPod := &corev1.Pod{} + err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod) + return apiErrors.IsNotFound(err) + }, timeoutDeletion, pollDeletion).Should(BeTrue()) + log.Info("Pod has already been deleted", "pod name", pod.Name) +} + // checkRemediation verify whether the node was remediated -func checkRemediation(nodeName string, nodeBootTimeBefore time.Time) { +func checkRemediation(nodeName string, nodeBootTimeBefore time.Time, oldPodCreationTime time.Time, va *storagev1.VolumeAttachment, pod *corev1.Pod) { By("Check if FAR NoExecute taint was added") wasFarTaintAdded(nodeName) @@ -300,8 +378,14 @@ func checkRemediation(nodeName string, nodeBootTimeBefore time.Time) { // TODO: When reboot is running only once and it is running on FAR node, then FAR pod will // be recreated on a new node and since the FA command won't be exuected again, then the log // won't include any success message - checkFarLogs(controllers.SuccessFAResponse) + checkFarLogs(succeesRebootMessage) By("Getting new node's boot time") wasNodeRebooted(nodeName, nodeBootTimeBefore) + + By("checking if old VA has been deleted") + checkVaDeleted(va) + + By("checking if old pod has been deleted") + checkPodDeleted(pod) } diff --git a/test/e2e/utils/command.go b/test/e2e/utils/command.go index 491b1890..ae16ab38 100644 --- a/test/e2e/utils/command.go +++ b/test/e2e/utils/command.go @@ -24,6 +24,10 @@ import ( "github.com/medik8s/fence-agents-remediation/api/v1alpha1" ) +const ( + containerTestName = "test-command" +) + // GetBootTime gets the boot time of the given node by running a pod on it executing uptime command func GetBootTime(c *kubernetes.Clientset, nodeName string, ns string, log logr.Logger) (time.Time, error) { emptyTime := time.Time{} @@ -44,7 +48,7 @@ func GetBootTime(c *kubernetes.Clientset, nodeName string, ns string, log logr.L func RunCommandInCluster(c *kubernetes.Clientset, nodeName string, ns string, command string, log logr.Logger) (string, error) { // create a pod and wait that it's running - pod := getPod(nodeName) + pod := GetPod(nodeName, containerTestName) pod, err := c.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{}) if err != nil { return "", err @@ -144,7 +148,7 @@ func waitForCondition(c *kubernetes.Clientset, pod *corev1.Pod, conditionType co }) } -func getPod(nodeName string) *corev1.Pod { +func GetPod(nodeName, containerName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "far-test-", @@ -161,7 +165,7 @@ func getPod(nodeName string) *corev1.Pod { RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { - Name: "test", + Name: containerName, Image: "registry.access.redhat.com/ubi8/ubi-minimal", SecurityContext: &corev1.SecurityContext{ Privileged: pointer.Bool(true),