Skip to content

Commit

Permalink
Test resource deletion in E2E with one pod and one va
Browse files Browse the repository at this point in the history 
Check deleted pod and va after resource deletion
  • Loading branch information
@razo7
razo7 committed Jul 26, 2023
1 parent 5fc0b73 commit 462c810
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 126 deletions.
195 changes: 72 additions & 123 deletions test/e2e/far_e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const (
nodeIdentifierPrefixIPMI = "--ipport"
succeesRebootMessage = "\"Success: Rebooted"
containerName = "manager"
testVolumeAttachment = "test-va"
testContainerName = "test-container"
testPodName = "test-pod"

Expand Down Expand Up @@ -83,6 +84,7 @@ var _ = Describe("FAR E2e", func() {
err error
testNodeName string
va *storagev1.VolumeAttachment
pod *corev1.Pod
creationTimePod, nodeBootTimeBefore time.Time
far *v1alpha1.FenceAgentsRemediation
)
Expand All @@ -109,21 +111,24 @@ var _ = Describe("FAR E2e", func() {

// create tested pod, and save it's creation time
// it will be deleted by FAR CR
pod := buildPod(testContainerName, testPodName, testNodeName)
pod = e2eUtils.GetPod(testNodeName, testContainerName)
pod.Name = testPodName
pod.Namespace = testNsName
Expect(k8sClient.Create(context.Background(), pod)).To(Succeed())
log.Info("Tested pod has been created", "pod", testPodName)
creationTimePod = metav1.Now().Time

va = createVA(testNodeName)
defer verifyCleanSetup(va, pod)

far = createFAR(testNodeName, fenceAgent, testShareParam, testNodeParam)
DeferCleanup(deleteFAR, far)
})
When("running FAR to reboot two nodes", func() {
It("should successfully remediate the first node", func() {
checkRemediation(testNodeName, succeesRebootMessage, nodeBootTimeBefore, far, creationTimePod, va)
checkRemediation(testNodeName, nodeBootTimeBefore, creationTimePod, va, pod)
})
It("should successfully remediate the second node", func() {
checkRemediation(testNodeName, succeesRebootMessage, nodeBootTimeBefore, far, creationTimePod, va)
checkRemediation(testNodeName, nodeBootTimeBefore, creationTimePod, va, pod)
})
})
})
Expand Down Expand Up @@ -229,6 +234,27 @@ func randomizeWorkerNode(nodes *corev1.NodeList) string {
return nodeName
}

// createVA creates dummy volume attachment for testing the resource deletion
func createVA(nodeName string) *storagev1.VolumeAttachment {
pv := "test-pv"
va := &storagev1.VolumeAttachment{
ObjectMeta: metav1.ObjectMeta{
Name: testVolumeAttachment,
Namespace: testNsName,
},
Spec: storagev1.VolumeAttachmentSpec{
Attacher: pv,
Source: storagev1.VolumeAttachmentSource{},
NodeName: nodeName,
},
}

va.Spec.Source.PersistentVolumeName = &pv
ExpectWithOffset(1, k8sClient.Create(context.Background(), va)).To(Succeed())
log.Info("Volume attachment has been created", "va", va.Name)
return va
}

// createFAR assigns the input to FenceAgentsRemediation object, creates CR, and returns the CR object
func createFAR(nodeName string, agent string, sharedParameters map[v1alpha1.ParameterName]string, nodeParameters map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string) *v1alpha1.FenceAgentsRemediation {
far := &v1alpha1.FenceAgentsRemediation{
Expand All @@ -254,6 +280,23 @@ func deleteFAR(far *v1alpha1.FenceAgentsRemediation) {
}, 2*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to delete far")
}

// verifyCleanSetup delete an old pod and old va if it was not deleted from FAR CR
func verifyCleanSetup(va *storagev1.VolumeAttachment, pod *corev1.Pod) {
newVa := &storagev1.VolumeAttachment{}
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa)
if err == nil {
Expect(k8sClient.Delete(context.Background(), newVa)).To(Succeed())
log.Info("Volume attachment was not deleted by remediation", "va name", va.Name)
}

newPod := &corev1.Pod{}
err = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod)
if err == nil {
Expect(k8sClient.Delete(context.Background(), newPod)).To(Succeed())
log.Info("Pod was not deleted by remediation", "pod name", pod.Name)
}
}

// wasFarTaintAdded checks whether the FAR taint was added to the tested node
func wasFarTaintAdded(nodeName string) {
farTaint := utils.CreateFARNoExecuteTaint()
Expand Down Expand Up @@ -307,8 +350,29 @@ func wasNodeRebooted(nodeName string, nodeBootTimeBefore time.Time) {
log.Info("successful reboot", "node", nodeName, "offset between last boot", nodeBootTimeAfter.Sub(nodeBootTimeBefore), "new boot time", nodeBootTimeAfter)
}

// checkVaDeleted verifies if the va has already been deleted due to resource deletion
func checkVaDeleted(va *storagev1.VolumeAttachment) {
EventuallyWithOffset(1, func() bool {
newVa := &storagev1.VolumeAttachment{}
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa)
return apiErrors.IsNotFound(err)

}, timeoutDeletion, pollDeletion).Should(BeTrue())
log.Info("Volume Attachment has already been deleted", "va name", va.Name)
}

// checkPodDeleted vefifies if the pod has already been deleted due to resource deletion
func checkPodDeleted(pod *corev1.Pod) {
ConsistentlyWithOffset(1, func() bool {
newPod := &corev1.Pod{}
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod)
return apiErrors.IsNotFound(err)
}, timeoutDeletion, pollDeletion).Should(BeTrue())
log.Info("Pod has already been deleted", "pod name", pod.Name)
}

// checkRemediation verify whether the node was remediated
func checkRemediation(nodeName, logString string, nodeBootTimeBefore time.Time, far *v1alpha1.FenceAgentsRemediation, oldPodCreationTime time.Time, va *storagev1.VolumeAttachment) {
func checkRemediation(nodeName string, nodeBootTimeBefore time.Time, oldPodCreationTime time.Time, va *storagev1.VolumeAttachment, pod *corev1.Pod) {
By("Check if FAR NoExecute taint was added")
wasFarTaintAdded(nodeName)

Expand All @@ -321,124 +385,9 @@ func checkRemediation(nodeName, logString string, nodeBootTimeBefore time.Time,
By("Getting new node's boot time")
wasNodeRebooted(nodeName, nodeBootTimeBefore)

By("checking whether old pods and VA have been evicted")
didResourceRescheduled(far, testPodName, oldPodCreationTime, va)
}

func createVA(nodeName string) *storagev1.VolumeAttachment {
foo := "foo"
va := &storagev1.VolumeAttachment{
ObjectMeta: metav1.ObjectMeta{
Name: "some-va",
Namespace: testNsName,
},
Spec: storagev1.VolumeAttachmentSpec{
Attacher: foo,
Source: storagev1.VolumeAttachmentSource{},
NodeName: nodeName,
},
}

va.Spec.Source.PersistentVolumeName = &foo
ExpectWithOffset(1, k8sClient.Create(context.Background(), va)).To(Succeed())
log.Info("Volume attachment has been created", "va", va.Name)
return va
}

// buildPod builds a pod
func buildPod(containerName, podName, nodeName string) *corev1.Pod {
pod := &corev1.Pod{}
pod.Name = podName
// testedPod should be reside on unhealthy node
pod.Spec.NodeName = nodeName
pod.Namespace = testNsName
container := corev1.Container{
Name: containerName,
Image: "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1:",
}
toleration := corev1.Toleration{
Key: v1alpha1.FARNoExecuteTaintKey,
Operator: corev1.TolerationOpEqual,
Effect: corev1.TaintEffectNoExecute,
}
pod.Spec.Containers = []corev1.Container{container}
pod.Spec.Tolerations = []corev1.Toleration{toleration}
return pod
}

func getPod(podName string) *corev1.Pod {
podTemp := client.ObjectKey{Name: podName, Namespace: testNsName}
pod := &corev1.Pod{}
Expect(k8sClient.Get(context.Background(), podTemp, pod)).To(Succeed(), "tested pod is missing")
return pod
}

func checkPodRescheduled(podName string, oldPodCreationTime time.Time) {
var newPodNode string
EventuallyWithOffset(1, func() time.Time {
pod := getPod(podName)
newPodNode = pod.Spec.NodeName
return pod.CreationTimestamp.Time
}, timeoutDeletion, pollDeletion).Should(BeTemporally(">", oldPodCreationTime), "pod creation timestamp hasn't been changes")
log.Info("Pod was rescheduled at another node", "new node", newPodNode)
}

func didResourceRescheduled(far *v1alpha1.FenceAgentsRemediation, testedPod string, oldPodCreationTime time.Time, va *storagev1.VolumeAttachment) {
By("checking if old VA has been deleted")
checkVaDeleted(va)
checkPodRescheduled(testedPod, oldPodCreationTime)
//Simulate NHC trying to delete FAR
// deleteAndWait(far)
// far = nil

// checkNoExecuteTaintRemoved(controlPlaneNode)
}

func checkVaDeleted(va *storagev1.VolumeAttachment) {
EventuallyWithOffset(1, func() bool {
newVa := &storagev1.VolumeAttachment{}
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa)
return apiErrors.IsNotFound(err)

}, timeoutDeletion, pollDeletion).Should(BeTrue())
log.Info("Volume Attachment has already been deleted", "va name", va.Name)
By("checking if old pod has been deleted")
checkPodDeleted(pod)
}

// func deleteAndWait(resource client.Object) {
// // Delete
// EventuallyWithOffset(1, func() error {
// err := k8sClient.Delete(context.Background(), resource)
// if apiErrors.IsNotFound(err) {
// return nil
// }
// return err
// }, 2*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to delete resource")
// // Wait until deleted
// EventuallyWithOffset(1, func() bool {
// err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(resource), resource)
// if apiErrors.IsNotFound(err) {
// return true
// }
// return false
// }, 2*time.Minute, 10*time.Second).Should(BeTrue(), "resource not deleted in time")
// }

// func checkNoExecuteTaintRemoved(node *corev1.Node) {
// By("checking if NoExecute taint was removed")
// EventuallyWithOffset(1, func() error {
// key := client.ObjectKey{
// Name: node.GetName(),
// }
// newNode := &v1.Node{}
// if err := k8sClient.Get(context.Background(), key, newNode); err != nil {
// logger.Error(err, "error getting node")
// return err
// }
// logger.Info("", "taints", newNode.Spec.Taints)
// for _, taint := range newNode.Spec.Taints {
// if taint.MatchTaint(controllers.NodeNoExecuteTaint) {
// return fmt.Errorf("NoExecute taint still exists")
// }
// }
// return nil
// }, 1*time.Minute, 10*time.Second).Should(Succeed())
// }
10 changes: 7 additions & 3 deletions test/e2e/utils/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ import (
"github.com/medik8s/fence-agents-remediation/api/v1alpha1"
)

const (
containerTestName = "test-command"
)

// GetBootTime gets the boot time of the given node by running a pod on it executing uptime command
func GetBootTime(c *kubernetes.Clientset, nodeName string, ns string, log logr.Logger) (time.Time, error) {
emptyTime := time.Time{}
Expand All @@ -44,7 +48,7 @@ func GetBootTime(c *kubernetes.Clientset, nodeName string, ns string, log logr.L
func RunCommandInCluster(c *kubernetes.Clientset, nodeName string, ns string, command string, log logr.Logger) (string, error) {

// create a pod and wait that it's running
pod := getPod(nodeName)
pod := GetPod(nodeName, containerTestName)
pod, err := c.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{})
if err != nil {
return "", err
Expand Down Expand Up @@ -144,7 +148,7 @@ func waitForCondition(c *kubernetes.Clientset, pod *corev1.Pod, conditionType co
})
}

func getPod(nodeName string) *corev1.Pod {
func GetPod(nodeName, containerName string) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "far-test-",
Expand All @@ -163,7 +167,7 @@ func getPod(nodeName string) *corev1.Pod {
RestartPolicy: corev1.RestartPolicyNever,
Containers: []corev1.Container{
{
Name: "test",
Name: containerName,
Image: "registry.access.redhat.com/ubi8/ubi-minimal",
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.Bool(true),
Expand Down

0 comments on commit 462c810

Please sign in to comment.