Skip to content

Commit

Permalink
Add e2e tests for workload deletion
Browse files Browse the repository at this point in the history 
Update e2e tests to create one VA and one pod which will be deleted by FAR pod
  • Loading branch information
@razo7
razo7 committed Jul 27, 2023
1 parent a655808 commit a99eff0
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 18 deletions.
115 changes: 100 additions & 15 deletions test/e2e/far_e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
Expand All @@ -32,12 +33,17 @@ const (
nodeIdentifierPrefixAWS = "--plug"
nodeIdentifierPrefixIPMI = "--ipport"
containerName = "manager"
testVolumeAttachment = "test-va"
testContainerName = "test-container"
testPodName = "test-pod"

//TODO: try to minimize timeout
// eventually parameters
timeoutLogs = 3 * time.Minute
timeoutReboot = 6 * time.Minute // fencing with fence_aws should be completed within 6 minutes
pollInterval = 10 * time.Second
timeoutLogs = 3 * time.Minute
timeoutReboot = 6 * time.Minute // fencing with fence_aws should be completed within 6 minutes
timeoutDeletion = 10 * time.Second // this timeout is used after all the other steps have been succesfult
pollDeletion = 250 * time.Millisecond
pollInterval = 10 * time.Second
)

var previousNodeName string
Expand Down Expand Up @@ -75,9 +81,12 @@ var _ = Describe("FAR E2e", func() {

Context("stress cluster", func() {
var (
nodeName string
nodeBootTimeBefore time.Time
err error
err error
testNodeName string
va *storagev1.VolumeAttachment
pod *corev1.Pod
creationTimePod, nodeBootTimeBefore time.Time
far *v1alpha1.FenceAgentsRemediation
)
BeforeEach(func() {
nodes := &corev1.NodeList{}
Expand All @@ -89,25 +98,37 @@ var _ = Describe("FAR E2e", func() {
Fail("No worker nodes found in the cluster")
}

nodeName = randomizeWorkerNode(nodes)
previousNodeName = nodeName
nodeNameParam := v1alpha1.NodeName(nodeName)
testNodeName = randomizeWorkerNode(nodes)
previousNodeName = testNodeName
nodeNameParam := v1alpha1.NodeName(testNodeName)
parameterName := v1alpha1.ParameterName(nodeIdentifierPrefix)
testNodeID := testNodeParam[parameterName][nodeNameParam]
log.Info("Testing Node", "Node name", nodeName, "Node ID", testNodeID)
log.Info("Testing Node", "Node name", testNodeName, "Node ID", testNodeID)

// save the node's boot time prior to the fence agent call
nodeBootTimeBefore, err = e2eUtils.GetBootTime(clientSet, nodeName, testNsName, log)
nodeBootTimeBefore, err = e2eUtils.GetBootTime(clientSet, testNodeName, testNsName, log)
Expect(err).ToNot(HaveOccurred(), "failed to get boot time of the node")
far := createFAR(nodeName, fenceAgent, testShareParam, testNodeParam)

// create tested pod, and save it's creation time
// it will be deleted by FAR CR
pod = e2eUtils.GetPod(testNodeName, testContainerName)
pod.Name = testPodName
pod.Namespace = testNsName
Expect(k8sClient.Create(context.Background(), pod)).To(Succeed())
log.Info("Tested pod has been created", "pod", testPodName)
creationTimePod = metav1.Now().Time
va = createVA(testNodeName)
DeferCleanup(verifyCleanSetup, va, pod)

far = createFAR(testNodeName, fenceAgent, testShareParam, testNodeParam)
DeferCleanup(deleteFAR, far)
})
When("running FAR to reboot two nodes", func() {
It("should successfully remediate the first node", func() {
checkRemediation(nodeName, nodeBootTimeBefore)
checkRemediation(testNodeName, nodeBootTimeBefore, creationTimePod, va, pod)
})
It("should successfully remediate the second node", func() {
checkRemediation(nodeName, nodeBootTimeBefore)
checkRemediation(testNodeName, nodeBootTimeBefore, creationTimePod, va, pod)
})
})
})
Expand Down Expand Up @@ -213,6 +234,28 @@ func randomizeWorkerNode(nodes *corev1.NodeList) string {
return nodeName
}

// createVA creates dummy volume attachment for testing the resource deletion
func createVA(nodeName string) *storagev1.VolumeAttachment {
pv := "test-pv"
va := &storagev1.VolumeAttachment{
ObjectMeta: metav1.ObjectMeta{
Name: testVolumeAttachment,
Namespace: testNsName,
},
Spec: storagev1.VolumeAttachmentSpec{
Attacher: pv,
Source: storagev1.VolumeAttachmentSource{
PersistentVolumeName: &pv,
},
NodeName: nodeName,
},
}

ExpectWithOffset(1, k8sClient.Create(context.Background(), va)).To(Succeed())
log.Info("Volume attachment has been created", "va", va.Name)
return va
}

// createFAR assigns the input to FenceAgentsRemediation object, creates CR, and returns the CR object
func createFAR(nodeName string, agent string, sharedParameters map[v1alpha1.ParameterName]string, nodeParameters map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string) *v1alpha1.FenceAgentsRemediation {
far := &v1alpha1.FenceAgentsRemediation{
Expand All @@ -238,6 +281,21 @@ func deleteFAR(far *v1alpha1.FenceAgentsRemediation) {
}, 2*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to delete far")
}

// verifyCleanSetup deletes an old pod and old va if it was not deleted from FAR CR
func verifyCleanSetup(va *storagev1.VolumeAttachment, pod *corev1.Pod) {
newVa := &storagev1.VolumeAttachment{}
if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa); err == nil {
Expect(k8sClient.Delete(context.Background(), newVa)).To(Succeed())
log.Info("cleanup: Volume attachment has not been deleted by remediation", "va name", va.Name)
}

newPod := &corev1.Pod{}
if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod); err == nil {
Expect(k8sClient.Delete(context.Background(), newPod)).To(Succeed())
log.Info("cleanup: Pod has not been deleted by remediation", "pod name", pod.Name)
}
}

// wasFarTaintAdded checks whether the FAR taint was added to the tested node
func wasFarTaintAdded(nodeName string) {
farTaint := utils.CreateFARNoExecuteTaint()
Expand Down Expand Up @@ -291,8 +349,29 @@ func wasNodeRebooted(nodeName string, nodeBootTimeBefore time.Time) {
log.Info("successful reboot", "node", nodeName, "offset between last boot", nodeBootTimeAfter.Sub(nodeBootTimeBefore), "new boot time", nodeBootTimeAfter)
}

// checkVaDeleted verifies if the va has already been deleted due to resource deletion
func checkVaDeleted(va *storagev1.VolumeAttachment) {
EventuallyWithOffset(1, func() bool {
newVa := &storagev1.VolumeAttachment{}
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(va), newVa)
return apiErrors.IsNotFound(err)

}, timeoutDeletion, pollDeletion).Should(BeTrue())
log.Info("Volume Attachment has already been deleted", "va name", va.Name)
}

// checkPodDeleted vefifies if the pod has already been deleted due to resource deletion
func checkPodDeleted(pod *corev1.Pod) {
ConsistentlyWithOffset(1, func() bool {
newPod := &corev1.Pod{}
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(pod), newPod)
return apiErrors.IsNotFound(err)
}, timeoutDeletion, pollDeletion).Should(BeTrue())
log.Info("Pod has already been deleted", "pod name", pod.Name)
}

// checkRemediation verify whether the node was remediated
func checkRemediation(nodeName string, nodeBootTimeBefore time.Time) {
func checkRemediation(nodeName string, nodeBootTimeBefore time.Time, oldPodCreationTime time.Time, va *storagev1.VolumeAttachment, pod *corev1.Pod) {
By("Check if FAR NoExecute taint was added")
wasFarTaintAdded(nodeName)

Expand All @@ -304,4 +383,10 @@ func checkRemediation(nodeName string, nodeBootTimeBefore time.Time) {

By("Getting new node's boot time")
wasNodeRebooted(nodeName, nodeBootTimeBefore)

By("checking if old VA has been deleted")
checkVaDeleted(va)

By("checking if old pod has been deleted")
checkPodDeleted(pod)
}
10 changes: 7 additions & 3 deletions test/e2e/utils/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ import (
"github.com/medik8s/fence-agents-remediation/api/v1alpha1"
)

const (
containerTestName = "test-command"
)

// GetBootTime gets the boot time of the given node by running a pod on it executing uptime command
func GetBootTime(c *kubernetes.Clientset, nodeName string, ns string, log logr.Logger) (time.Time, error) {
emptyTime := time.Time{}
Expand All @@ -44,7 +48,7 @@ func GetBootTime(c *kubernetes.Clientset, nodeName string, ns string, log logr.L
func RunCommandInCluster(c *kubernetes.Clientset, nodeName string, ns string, command string, log logr.Logger) (string, error) {

// create a pod and wait that it's running
pod := getPod(nodeName)
pod := GetPod(nodeName, containerTestName)
pod, err := c.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{})
if err != nil {
return "", err
Expand Down Expand Up @@ -144,7 +148,7 @@ func waitForCondition(c *kubernetes.Clientset, pod *corev1.Pod, conditionType co
})
}

func getPod(nodeName string) *corev1.Pod {
func GetPod(nodeName, containerName string) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "far-test-",
Expand All @@ -161,7 +165,7 @@ func getPod(nodeName string) *corev1.Pod {
RestartPolicy: corev1.RestartPolicyNever,
Containers: []corev1.Container{
{
Name: "test",
Name: containerName,
Image: "registry.access.redhat.com/ubi8/ubi-minimal",
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.Bool(true),
Expand Down

0 comments on commit a99eff0

Please sign in to comment.