From da710de21a985dca27472af1f0f5d68c81e59fc4 Mon Sep 17 00:00:00 2001 From: Patryk Diak Date: Thu, 19 Sep 2024 11:45:52 +0200 Subject: [PATCH 01/14] Services E2Es: Reset test variables before each test Fixes: https://github.com/ovn-org/ovn-kubernetes/issues/4733 Signed-off-by: Patryk Diak (cherry picked from commit d6f145e01a683b3a9d71c41289a7efc8ba8699d0) --- test/e2e/service.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/e2e/service.go b/test/e2e/service.go index f4babd385fc..054c5bd06ac 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -708,11 +708,8 @@ var _ = ginkgo.Describe("Services", func() { var nodes *v1.NodeList var err error nodeIPs := make(map[string]map[int]string) - var egressPod *v1.Pod var egressNode string - targetSecondaryNode := node{ - name: "egressSecondaryTargetNode-allowed", - } + var targetSecondaryNode node const ( endpointHTTPPort = 80 @@ -722,6 +719,14 @@ var _ = ginkgo.Describe("Services", func() { clientContainerName = "npclient" ) + ginkgo.BeforeEach(func() { + nodeIPs = make(map[string]map[int]string) + egressNode = "" + targetSecondaryNode = node{ + name: "egressSecondaryTargetNode-allowed", + } + }) + ginkgo.AfterEach(func() { ginkgo.By("Cleaning up external container") deleteClusterExternalContainer(clientContainerName) @@ -745,7 +750,6 @@ var _ = ginkgo.Describe("Services", func() { e2ekubectl.RunKubectlOrDie("default", "delete", "eip", "egressip", "--ignore-not-found=true") e2ekubectl.RunKubectlOrDie("default", "label", "node", egressNode, "k8s.ovn.org/egress-assignable-") tearDownNetworkAndTargetForMultiNIC([]string{egressNode}, targetSecondaryNode) - targetSecondaryNode.nodeIP = "" } }) @@ -924,7 +928,7 @@ var _ = ginkgo.Describe("Services", func() { } ginkgo.By("Choosing egressIP pod") - egressPod = endPoints[0] + egressPod := endPoints[0] framework.Logf("EgressIP pod is %s/%s", endPoints[0].Namespace, endPoints[0].Name) ginkgo.By("Label egress node" + egressNode + " create external container to send egress traffic to via secondary MultiNIC EIP") From 7d1f869e9aed08290f224d171d1dfc51928d87ae Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Mon, 6 May 2024 10:59:35 +0200 Subject: [PATCH 02/14] egressfirewall: avoid nil dereference on node delete Signed-off-by: Nadia Pinaeva (cherry picked from commit ab00a4ba29e181356f826d89a56f3bda186d68ee) --- go-controller/pkg/ovn/egressfirewall.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go-controller/pkg/ovn/egressfirewall.go b/go-controller/pkg/ovn/egressfirewall.go index 9995cb61f90..62684b9b240 100644 --- a/go-controller/pkg/ovn/egressfirewall.go +++ b/go-controller/pkg/ovn/egressfirewall.go @@ -733,7 +733,6 @@ func (oc *DefaultNetworkController) getEgressFirewallACLDbIDs(namespace string, } func (oc *DefaultNetworkController) updateEgressFirewallForNode(oldNode, newNode *kapi.Node) error { - var addressesToAdd []string var addressesToRemove []string var err error @@ -774,7 +773,7 @@ func (oc *DefaultNetworkController) updateEgressFirewallForNode(oldNode, newNode // matches or not we shouldn't have those addresses anymore rule.to.nodeAddrs.Delete(addressesToRemove...) // check if selector matches - if selector.Matches(labels.Set(newNode.Labels)) { + if newNode != nil && selector.Matches(labels.Set(newNode.Labels)) { rule.to.nodeAddrs.Insert(addressesToAdd...) } modifiedRuleIDs = append(modifiedRuleIDs, rule.id) From cebf08e92b8f90ce46ece83a3d4020afe8a5f7de Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 21:22:31 +0200 Subject: [PATCH 03/14] e2e: Add applyEF function Signed-off-by: Nadia Pinaeva (cherry picked from commit ba8766a1ff698923a2ebe75e9df72d2fb268c786) --- test/e2e/egress_firewall.go | 135 +++++++++++------------------------- 1 file changed, 40 insertions(+), 95 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index 3009f14dd50..5074caabd30 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -63,6 +63,29 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { }, 10*time.Second).Should(gomega.BeTrue(), fmt.Sprintf("expected egress firewall in namespace %s to be successfully applied", namespace)) } + applyEF := func(egressFirewallConfig, namespace string) { + // write the config to a file for application and defer the removal + if err := os.WriteFile(egressFirewallYamlFile, []byte(egressFirewallConfig), 0644); err != nil { + framework.Failf("Unable to write CRD config to disk: %v", err) + } + defer func() { + if err := os.Remove(egressFirewallYamlFile); err != nil { + framework.Logf("Unable to remove the CRD config from disk: %v", err) + } + }() + // create the CRD config parameters + applyArgs := []string{ + "apply", + "--namespace=" + namespace, + "-f", + egressFirewallYamlFile, + } + framework.Logf("Applying EgressFirewall configuration: %s ", applyArgs) + // apply the egress firewall configuration + e2ekubectl.RunKubectlOrDie(namespace, applyArgs...) + waitForEFApplied(namespace) + } + f := wrappedTestFramework(svcname) // node2ndaryIPs holds the nodeName as the key and the value is // a map with ipFamily(v4 or v6) as the key and the secondaryIP as the value @@ -118,7 +141,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { ginkgo.It("Should validate the egress firewall policy functionality against remote hosts", func() { srcPodName := "e2e-egress-fw-src-pod" - frameworkNsFlag := fmt.Sprintf("--namespace=%s", f.Namespace.Name) testContainer := fmt.Sprintf("%s-container", srcPodName) testContainerFlag := fmt.Sprintf("--container=%s", testContainer) // egress firewall crd yaml configuration @@ -142,26 +164,8 @@ spec: to: cidrSelector: %s `, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR) - // write the config to a file for application and defer the removal - if err := os.WriteFile(egressFirewallYamlFile, []byte(egressFirewallConfig), 0644); err != nil { - framework.Failf("Unable to write CRD config to disk: %v", err) - } - defer func() { - if err := os.Remove(egressFirewallYamlFile); err != nil { - framework.Logf("Unable to remove the CRD config from disk: %v", err) - } - }() - // create the CRD config parameters - applyArgs := []string{ - "apply", - frameworkNsFlag, - "-f", - egressFirewallYamlFile, - } - framework.Logf("Applying EgressFirewall configuration: %s ", applyArgs) - // apply the egress firewall configuration - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, applyArgs...) - waitForEFApplied(f.Namespace.Name) + applyEF(egressFirewallConfig, f.Namespace.Name) + // create the pod that will be used as the source for the connectivity test createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) @@ -231,7 +235,6 @@ spec: ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() { srcPodName := "e2e-egress-fw-src-pod" - frameworkNsFlag := fmt.Sprintf("--namespace=%s", f.Namespace.Name) testContainer := fmt.Sprintf("%s-container", srcPodName) testContainerFlag := fmt.Sprintf("--container=%s", testContainer) // use random labels in case test runs again since it's a pain to remove the label from the node @@ -263,26 +266,7 @@ spec: cidrSelector: %s `, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, f.Namespace.Name, labelMatch, exFWDenyCIDR) framework.Logf("Egress Firewall CR generated: %s", egressFirewallConfig) - // write the config to a file for application and defer the removal - if err := os.WriteFile(egressFirewallYamlFile, []byte(egressFirewallConfig), 0644); err != nil { - framework.Failf("Unable to write CRD config to disk: %v", err) - } - defer func() { - if err := os.Remove(egressFirewallYamlFile); err != nil { - framework.Logf("Unable to remove the CRD config from disk: %v", err) - } - }() - // create the CRD config parameters - applyArgs := []string{ - "apply", - frameworkNsFlag, - "-f", - egressFirewallYamlFile, - } - framework.Logf("Applying EgressFirewall configuration: %s ", applyArgs) - // apply the egress firewall configuration - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, applyArgs...) - waitForEFApplied(f.Namespace.Name) + applyEF(egressFirewallConfig, f.Namespace.Name) // create the pod that will be used as the source for the connectivity test createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) // create host networked pod @@ -435,7 +419,6 @@ spec: }) ginkgo.It("Should validate the egress firewall DNS does not deadlock when adding many dnsNames", func() { - frameworkNsFlag := fmt.Sprintf("--namespace=%s", f.Namespace.Name) var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall apiVersion: k8s.ovn.org/v1 metadata: @@ -492,26 +475,7 @@ spec: to: cidrSelector: %s `, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR) - // write the config to a file for application and defer the removal - if err := os.WriteFile(egressFirewallYamlFile, []byte(egressFirewallConfig), 0644); err != nil { - framework.Failf("Unable to write CRD config to disk: %v", err) - } - defer func() { - if err := os.Remove(egressFirewallYamlFile); err != nil { - framework.Logf("Unable to remove the CRD config from disk: %v", err) - } - }() - // create the CRD config parameters - applyArgs := []string{ - "apply", - frameworkNsFlag, - "-f", - egressFirewallYamlFile, - } - framework.Logf("Applying EgressFirewall configuration: %s ", applyArgs) - // apply the egress firewall configuration - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, applyArgs...) - waitForEFApplied(f.Namespace.Name) + applyEF(egressFirewallConfig, f.Namespace.Name) }) ginkgo.It("Should validate the egress firewall allows inbound connections", func() { @@ -527,41 +491,12 @@ spec: externalContainerName := "e2e-egress-fw-external-container" externalContainerPort := 1234 - frameworkNsFlag := fmt.Sprintf("--namespace=%s", f.Namespace.Name) testContainer := fmt.Sprintf("%s-container", efPodName) testContainerFlag := fmt.Sprintf("--container=%s", testContainer) denyCIDR := "0.0.0.0/0" if IsIPv6Cluster(f.ClientSet) { denyCIDR = "::/0" } - // egress firewall crd yaml configuration - var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall -apiVersion: k8s.ovn.org/v1 -metadata: - name: default - namespace: %s -spec: - egress: - - type: Deny - to: - cidrSelector: %s -`, f.Namespace.Name, denyCIDR) - // write the config to a file for application and defer the removal - if err := os.WriteFile(egressFirewallYamlFile, []byte(egressFirewallConfig), 0644); err != nil { - framework.Failf("Unable to write CRD config to disk: %v", err) - } - defer func() { - if err := os.Remove(egressFirewallYamlFile); err != nil { - framework.Logf("Unable to remove the CRD config from disk: %v", err) - } - }() - // create the CRD config parameters - applyArgs := []string{ - "apply", - frameworkNsFlag, - "-f", - egressFirewallYamlFile, - } ginkgo.By("Creating the egress firewall pod") // 1. create nodePort service and external container @@ -614,9 +549,19 @@ spec: } // 3. Apply deny-all egress firewall and wait for it to be applied - framework.Logf("Applying EgressFirewall configuration: %s ", applyArgs) - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, applyArgs...) - waitForEFApplied(f.Namespace.Name) + // egress firewall crd yaml configuration + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default + namespace: %s +spec: + egress: + - type: Deny + to: + cidrSelector: %s +`, f.Namespace.Name, denyCIDR) + applyEF(egressFirewallConfig, f.Namespace.Name) // 4. Check that only inbound traffic is allowed // pod -> external container should be blocked From 2e105cbcc60d91b269da70ea4cab444f73f2b8cc Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 21:33:57 +0200 Subject: [PATCH 04/14] e2e: add checkConnectivity functions for EF tests. Update connectivity timeout to 3 seconds and allow 2 retries both for positive and negative cases Signed-off-by: Nadia Pinaeva (cherry picked from commit 09b08ece9edef11ec7d9463a0d1cbc0b292929c6) --- test/e2e/egress_firewall.go | 130 +++++++++++++----------------------- 1 file changed, 46 insertions(+), 84 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index 5074caabd30..cf381366e58 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -32,7 +32,7 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { ovnContainer string = "ovnkube-node" egressFirewallYamlFile string = "egress-fw.yml" - testTimeout string = "5" + testTimeout int = 3 retryInterval = 1 * time.Second retryTimeout = 30 * time.Second ciNetworkName = "kind" @@ -87,6 +87,37 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } f := wrappedTestFramework(svcname) + + checkConnectivity := func(srcPodName, dstIP string, dstPort int, shouldSucceed bool) { + testContainer := fmt.Sprintf("%s-container", srcPodName) + testContainerFlag := fmt.Sprintf("--container=%s", testContainer) + if shouldSucceed { + gomega.Eventually(func() bool { + _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", + "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)) + return err == nil + }, time.Duration(2*testTimeout)*time.Second).Should(gomega.BeTrue(), + fmt.Sprintf("expected connection from %s to [%s]:%d to suceed", srcPodName, dstIP, dstPort)) + } else { + gomega.Consistently(func() bool { + _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", + "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)) + return err != nil + }, time.Duration(2*testTimeout)*time.Second).Should(gomega.BeTrue(), + fmt.Sprintf("expected connection from %s to [%s]:%d to fail", srcPodName, dstIP, dstPort)) + } + } + + checkExternalContainerConnectivity := func(containerName, dstIP string, dstPort int) { + cmd := []string{"docker", "exec", containerName, "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)} + framework.Logf("Running command %v", cmd) + _, err := runCommand(cmd...) + if err != nil { + framework.Failf("Failed to connect from external container %s to %s:%d: %v", + containerName, dstIP, dstPort, err) + } + } + // node2ndaryIPs holds the nodeName as the key and the value is // a map with ipFamily(v4 or v6) as the key and the secondaryIP as the value // This is defined here globally to allow us to cleanup in AfterEach @@ -141,8 +172,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { ginkgo.It("Should validate the egress firewall policy functionality against remote hosts", func() { srcPodName := "e2e-egress-fw-src-pod" - testContainer := fmt.Sprintf("%s-container", srcPodName) - testContainerFlag := fmt.Sprintf("--container=%s", testContainer) // egress firewall crd yaml configuration var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall apiVersion: k8s.ovn.org/v1 @@ -169,68 +198,21 @@ spec: // create the pod that will be used as the source for the connectivity test createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) - // In very rare cases the 'nc' test commands do not return the expected result on the first try, - // but while testing has reproduced these cases, the condition is temporary and only initially - // after the pod has been created. Eventually the traffic tests work as expected, so they - // are wrapped in PollImmediate with a total duration equal to 4 times the pollingDuration (5 seconds) - // which should provide for 3 tries before a failure is actually flagged - testTimeoutInt, err := strconv.Atoi(testTimeout) - if err != nil { - framework.Failf("failed to parse test timeout duration: %v", err) - } - pollingDuration := time.Duration(4*testTimeoutInt) * time.Second - // Verify the remote host/port as explicitly allowed by the firewall policy is reachable ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) - err = wait.PollImmediate(2, pollingDuration, func() (bool, error) { - _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpDnsDest, "53") - if err == nil { - return true, nil - } - return false, nil - }) - if err != nil { - framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, ovnContainer, serverNodeInfo.name, err) - } + checkConnectivity(srcPodName, exFWPermitTcpDnsDest, 53, true) // Verify the remote host/port as implicitly denied by the firewall policy is not reachable ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest)) - err = wait.PollImmediate(2, pollingDuration, func() (bool, error) { - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWDenyTcpDnsDest, "53") - if err != nil { - return false, nil - } - return true, nil - }) - if err != wait.ErrWaitTimeout { - framework.Failf("Succeeded in connecting the implicitly denied remote host %s from container %s on node %s: %v", exFWDenyTcpDnsDest, ovnContainer, serverNodeInfo.name, err) - } + checkConnectivity(srcPodName, exFWDenyTcpDnsDest, 53, false) // Verify the explicitly allowed host/port tcp port 80 rule is functional ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - err = wait.PollImmediate(2, pollingDuration, func() (bool, error) { - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpWwwDest, "80") - if err == nil { - return true, nil - } - return false, nil - }) - if err != nil { - framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) - } + checkConnectivity(srcPodName, exFWPermitTcpWwwDest, 80, true) // Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - err = wait.PollImmediate(2, pollingDuration, func() (bool, error) { - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpWwwDest, "443") - if err != nil { - return false, nil - } - return true, nil - }) - if err != wait.ErrWaitTimeout { - framework.Failf("Succeeded in connecting the implicitly denied remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) - } + checkConnectivity(srcPodName, exFWPermitTcpWwwDest, 443, false) }) ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() { @@ -338,25 +320,25 @@ spec: // Verify basic external connectivity to ensure egress firewall is working for normal conditions ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpDnsDest, "53") + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpDnsDest, "53") if err != nil { framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, srcPodName, serverNodeInfo.name, err) } // Verify the remote host/port as implicitly denied by the firewall policy is not reachable ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWDenyTcpDnsDest, "53") + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWDenyTcpDnsDest, "53") if err == nil { framework.Failf("Succeeded in connecting the implicitly denied remote host %s from container %s on node %s", exFWDenyTcpDnsDest, ovnContainer, serverNodeInfo.name) } // Verify the explicitly allowed host/port tcp port 80 rule is functional ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpWwwDest, "80") + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "80") if err != nil { framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) } // Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpWwwDest, "443") + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "443") if err == nil { framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) } @@ -491,8 +473,6 @@ spec: externalContainerName := "e2e-egress-fw-external-container" externalContainerPort := 1234 - testContainer := fmt.Sprintf("%s-container", efPodName) - testContainerFlag := fmt.Sprintf("--container=%s", testContainer) denyCIDR := "0.0.0.0/0" if IsIPv6Cluster(f.ClientSet) { denyCIDR = "::/0" @@ -529,24 +509,15 @@ spec: } ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d", efPodName, externalContainerIP, externalContainerPort)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", efPodName, testContainerFlag, - "--", "nc", "-vz", "-w", testTimeout, externalContainerIP, strconv.Itoa(externalContainerPort)) - if err != nil { - framework.Failf("Failed to connect from pod to external container, before egress firewall is applied") - } + checkConnectivity(efPodName, externalContainerIP, externalContainerPort, true) + // external container -> nodePort svc should work svc, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) framework.ExpectNoError(err, "failed to fetch service: %s in namespace %s", serviceName, f.Namespace.Name) ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", externalContainerIP, nodeIP, svc.Spec.Ports[0].NodePort)) - cmd := []string{"docker", "exec", externalContainerName, "nc", "-vz", "-w", testTimeout, nodeIP, strconv.Itoa(int(svc.Spec.Ports[0].NodePort))} - framework.Logf("Running command %v", cmd) - _, err = runCommand(cmd...) - if err != nil { - framework.Failf("Failed to connect to nodePort service from external container %s, before egress firewall is applied: %v", - externalContainerName, err) - } + checkExternalContainerConnectivity(externalContainerName, nodeIP, int(svc.Spec.Ports[0].NodePort)) // 3. Apply deny-all egress firewall and wait for it to be applied // egress firewall crd yaml configuration @@ -567,20 +538,11 @@ spec: // pod -> external container should be blocked ginkgo.By(fmt.Sprintf("Verifying connection from pod %s to external container %s is blocked:%d", efPodName, externalContainerIP, externalContainerPort)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", efPodName, testContainerFlag, - "--", "nc", "-vz", "-w", testTimeout, externalContainerIP, strconv.Itoa(externalContainerPort)) - if err == nil { - framework.Failf("Egress firewall doesn't block connection from pod to external container") - } + checkConnectivity(efPodName, externalContainerIP, externalContainerPort, false) + // external container -> nodePort svc should work ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", externalContainerIP, nodeIP, svc.Spec.Ports[0].NodePort)) - cmd = []string{"docker", "exec", externalContainerName, "nc", "-vz", "-w", testTimeout, nodeIP, strconv.Itoa(int(svc.Spec.Ports[0].NodePort))} - framework.Logf("Running command %v", cmd) - _, err = runCommand(cmd...) - if err != nil { - framework.Failf("Failed to connect to nodePort service from external container %s: %v", - externalContainerName, err) - } + checkExternalContainerConnectivity(externalContainerName, nodeIP, int(svc.Spec.Ports[0].NodePort)) }) }) From 7a2daaf12a8c5a478705185759590c2242256976 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:05:36 +0200 Subject: [PATCH 05/14] e2e: use external containers for EF connectivity checks to make it work with ipv6, since github runners don't have any routes for IPv6. Split current test that checks allow IP and allow CIDR+port into 2 tests to limit the amount of required external containers. Bonus: the only test that used external containers doesn't need to create them anymore, as they are created in beforeEach Signed-off-by: Nadia Pinaeva (cherry picked from commit 2372a085784326e8f70c118157312f8733eab559) --- test/e2e/egress_firewall.go | 145 +++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 51 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index cf381366e58..afa11a4ffb3 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -36,6 +36,10 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { retryInterval = 1 * time.Second retryTimeout = 30 * time.Second ciNetworkName = "kind" + externalContainerName1 = "e2e-egress-fw-external-container1" + externalContainerName2 = "e2e-egress-fw-external-container2" + externalContainerPort1 = 1111 + externalContainerPort2 = 2222 ) type nodeInfo struct { @@ -44,13 +48,15 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } var ( - serverNodeInfo nodeInfo - exFWPermitTcpDnsDest string - singleIPMask string - exFWDenyTcpDnsDest string - exFWPermitTcpWwwDest string - exFWPermitCIDR string - exFWDenyCIDR string + serverNodeInfo nodeInfo + exFWPermitTcpDnsDest string + exFWDenyTcpDnsDest string + exFWPermitTcpWwwDest string + exFWPermitCIDR string + exFWDenyCIDR string + denyAllCIDR string + singleIPMask, subnetMask string + externalContainer1IP, externalContainer2IP string ) waitForEFApplied := func(namespace string) { @@ -154,6 +160,34 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { exFWDenyCIDR = "::/0" singleIPMask = "128" } + + externalContainer1IPV4, externalContainer1IPV6 := createClusterExternalContainer(externalContainerName1, agnhostImage, + []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort1, externalContainerPort1)}, + []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort1)}) + + externalContainer2IPV4, externalContainer2IPV6 := createClusterExternalContainer(externalContainerName2, agnhostImage, + []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort2, externalContainerPort2)}, + []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort2)}) + + if IsIPv6Cluster(f.ClientSet) { + externalContainer1IP = externalContainer1IPV6 + externalContainer2IP = externalContainer2IPV6 + } else { + externalContainer1IP = externalContainer1IPV4 + externalContainer2IP = externalContainer2IPV4 + } + + singleIPMask = "32" + subnetMask = "24" + if IsIPv6Cluster(f.ClientSet) { + singleIPMask = "128" + subnetMask = "64" + } + + denyAllCIDR = "0.0.0.0/0" + if IsIPv6Cluster(f.ClientSet) { + denyAllCIDR = "::/0" + } }) ginkgo.AfterEach(func() { @@ -168,9 +202,11 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } } } + deleteClusterExternalContainer(externalContainerName1) + deleteClusterExternalContainer(externalContainerName2) }) - ginkgo.It("Should validate the egress firewall policy functionality against remote hosts", func() { + ginkgo.It("Should validate the egress firewall policy functionality for allowed IP", func() { srcPodName := "e2e-egress-fw-src-pod" // egress firewall crd yaml configuration var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall @@ -183,36 +219,60 @@ spec: - type: Allow to: cidrSelector: %s/%s - - type: Allow + - type: Deny to: cidrSelector: %s +`, f.Namespace.Name, externalContainer1IP, singleIPMask, denyAllCIDR) + applyEF(egressFirewallConfig, f.Namespace.Name) + + // create the pod that will be used as the source for the connectivity test + createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) + + // Verify the remote host/port as explicitly allowed by the firewall policy is reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined "+ + "by the external firewall policy", externalContainer1IP)) + checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true) + + // Verify the remote host/port as implicitly denied by the firewall policy is not reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined "+ + "by the external firewall policy", externalContainer2IP)) + checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false) + }) + + ginkgo.It("Should validate the egress firewall policy functionality for allowed CIDR and port", func() { + srcPodName := "e2e-egress-fw-src-pod" + // egress firewall crd yaml configuration + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default + namespace: %s +spec: + egress: + - type: Allow + to: + cidrSelector: %s/%s ports: - protocol: TCP - port: 80 + port: %d - type: Deny to: cidrSelector: %s -`, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR) +`, f.Namespace.Name, externalContainer1IP, subnetMask, externalContainerPort1, denyAllCIDR) applyEF(egressFirewallConfig, f.Namespace.Name) // create the pod that will be used as the source for the connectivity test createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) // Verify the remote host/port as explicitly allowed by the firewall policy is reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) - checkConnectivity(srcPodName, exFWPermitTcpDnsDest, 53, true) + ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed port on host %s is permitted as "+ + "defined by the external firewall policy", externalContainer1IP)) + checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true) // Verify the remote host/port as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest)) - checkConnectivity(srcPodName, exFWDenyTcpDnsDest, 53, false) - - // Verify the explicitly allowed host/port tcp port 80 rule is functional - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - checkConnectivity(srcPodName, exFWPermitTcpWwwDest, 80, true) - - // Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - checkConnectivity(srcPodName, exFWPermitTcpWwwDest, 443, false) + ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as "+ + "defined by the external firewall policy", externalContainer2IP)) + checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false) }) ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() { @@ -470,13 +530,6 @@ spec: efPodPort := 1234 serviceName := "service-for-pods" servicePort := 31234 - externalContainerName := "e2e-egress-fw-external-container" - externalContainerPort := 1234 - - denyCIDR := "0.0.0.0/0" - if IsIPv6Cluster(f.ClientSet) { - denyCIDR = "::/0" - } ginkgo.By("Creating the egress firewall pod") // 1. create nodePort service and external container @@ -495,32 +548,22 @@ spec: err = framework.WaitForServiceEndpointsNum(context.TODO(), f.ClientSet, f.Namespace.Name, serviceName, 1, time.Second, wait.ForeverTestTimeout) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name) - nodeIP := serverNodeInfo.nodeIP - externalContainerIPV4, externalContainerIPV6 := createClusterExternalContainer(externalContainerName, agnhostImage, - []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort, externalContainerPort)}, - []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort)}) - defer deleteClusterExternalContainer(externalContainerName) - // 2. Check connectivity works both ways // pod -> external container should work - externalContainerIP := externalContainerIPV4 - if IsIPv6Cluster(f.ClientSet) { - externalContainerIP = externalContainerIPV6 - } ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d", - efPodName, externalContainerIP, externalContainerPort)) - checkConnectivity(efPodName, externalContainerIP, externalContainerPort, true) + efPodName, externalContainer1IP, externalContainerPort1)) + checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, true) // external container -> nodePort svc should work svc, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) framework.ExpectNoError(err, "failed to fetch service: %s in namespace %s", serviceName, f.Namespace.Name) + nodeIP := serverNodeInfo.nodeIP ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", - externalContainerIP, nodeIP, svc.Spec.Ports[0].NodePort)) - checkExternalContainerConnectivity(externalContainerName, nodeIP, int(svc.Spec.Ports[0].NodePort)) + externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) + checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) // 3. Apply deny-all egress firewall and wait for it to be applied - // egress firewall crd yaml configuration var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall apiVersion: k8s.ovn.org/v1 metadata: @@ -531,18 +574,18 @@ spec: - type: Deny to: cidrSelector: %s -`, f.Namespace.Name, denyCIDR) +`, f.Namespace.Name, denyAllCIDR) applyEF(egressFirewallConfig, f.Namespace.Name) // 4. Check that only inbound traffic is allowed // pod -> external container should be blocked - ginkgo.By(fmt.Sprintf("Verifying connection from pod %s to external container %s is blocked:%d", - efPodName, externalContainerIP, externalContainerPort)) - checkConnectivity(efPodName, externalContainerIP, externalContainerPort, false) + ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d is blocked", + efPodName, externalContainer1IP, externalContainerPort1)) + checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, false) // external container -> nodePort svc should work ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", - externalContainerIP, nodeIP, svc.Spec.Ports[0].NodePort)) - checkExternalContainerConnectivity(externalContainerName, nodeIP, int(svc.Spec.Ports[0].NodePort)) + externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) + checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) }) }) From 422f5fa2bba36b20cac6fb966748cc39b85905da Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:14:08 +0200 Subject: [PATCH 06/14] e2e: move tests using external containers into their own context to avoid unneeded container creation. No extra changes Signed-off-by: Nadia Pinaeva (cherry picked from commit 82b2bf1e0fa8ca424155d388aae651153f2c38fc) --- test/e2e/egress_firewall.go | 360 +++++++++++++++++++----------------- 1 file changed, 187 insertions(+), 173 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index afa11a4ffb3..8b222fa7f6d 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -35,11 +35,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { testTimeout int = 3 retryInterval = 1 * time.Second retryTimeout = 30 * time.Second - ciNetworkName = "kind" - externalContainerName1 = "e2e-egress-fw-external-container1" - externalContainerName2 = "e2e-egress-fw-external-container2" - externalContainerPort1 = 1111 - externalContainerPort2 = 2222 ) type nodeInfo struct { @@ -48,15 +43,14 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } var ( - serverNodeInfo nodeInfo - exFWPermitTcpDnsDest string - exFWDenyTcpDnsDest string - exFWPermitTcpWwwDest string - exFWPermitCIDR string - exFWDenyCIDR string - denyAllCIDR string - singleIPMask, subnetMask string - externalContainer1IP, externalContainer2IP string + serverNodeInfo nodeInfo + exFWPermitTcpDnsDest string + exFWDenyTcpDnsDest string + exFWPermitTcpWwwDest string + exFWPermitCIDR string + exFWDenyCIDR string + denyAllCIDR string + singleIPMask string ) waitForEFApplied := func(namespace string) { @@ -94,36 +88,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { f := wrappedTestFramework(svcname) - checkConnectivity := func(srcPodName, dstIP string, dstPort int, shouldSucceed bool) { - testContainer := fmt.Sprintf("%s-container", srcPodName) - testContainerFlag := fmt.Sprintf("--container=%s", testContainer) - if shouldSucceed { - gomega.Eventually(func() bool { - _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", - "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)) - return err == nil - }, time.Duration(2*testTimeout)*time.Second).Should(gomega.BeTrue(), - fmt.Sprintf("expected connection from %s to [%s]:%d to suceed", srcPodName, dstIP, dstPort)) - } else { - gomega.Consistently(func() bool { - _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", - "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)) - return err != nil - }, time.Duration(2*testTimeout)*time.Second).Should(gomega.BeTrue(), - fmt.Sprintf("expected connection from %s to [%s]:%d to fail", srcPodName, dstIP, dstPort)) - } - } - - checkExternalContainerConnectivity := func(containerName, dstIP string, dstPort int) { - cmd := []string{"docker", "exec", containerName, "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)} - framework.Logf("Running command %v", cmd) - _, err := runCommand(cmd...) - if err != nil { - framework.Failf("Failed to connect from external container %s to %s:%d: %v", - containerName, dstIP, dstPort, err) - } - } - // node2ndaryIPs holds the nodeName as the key and the value is // a map with ipFamily(v4 or v6) as the key and the secondaryIP as the value // This is defined here globally to allow us to cleanup in AfterEach @@ -161,29 +125,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { singleIPMask = "128" } - externalContainer1IPV4, externalContainer1IPV6 := createClusterExternalContainer(externalContainerName1, agnhostImage, - []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort1, externalContainerPort1)}, - []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort1)}) - - externalContainer2IPV4, externalContainer2IPV6 := createClusterExternalContainer(externalContainerName2, agnhostImage, - []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort2, externalContainerPort2)}, - []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort2)}) - - if IsIPv6Cluster(f.ClientSet) { - externalContainer1IP = externalContainer1IPV6 - externalContainer2IP = externalContainer2IPV6 - } else { - externalContainer1IP = externalContainer1IPV4 - externalContainer2IP = externalContainer2IPV4 - } - - singleIPMask = "32" - subnetMask = "24" - if IsIPv6Cluster(f.ClientSet) { - singleIPMask = "128" - subnetMask = "64" - } - denyAllCIDR = "0.0.0.0/0" if IsIPv6Cluster(f.ClientSet) { denyAllCIDR = "::/0" @@ -202,14 +143,86 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } } } - deleteClusterExternalContainer(externalContainerName1) - deleteClusterExternalContainer(externalContainerName2) }) - ginkgo.It("Should validate the egress firewall policy functionality for allowed IP", func() { - srcPodName := "e2e-egress-fw-src-pod" - // egress firewall crd yaml configuration - var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall + ginkgo.Context("with external containers", func() { + const ( + ciNetworkName = "kind" + externalContainerName1 = "e2e-egress-fw-external-container1" + externalContainerName2 = "e2e-egress-fw-external-container2" + externalContainerPort1 = 1111 + externalContainerPort2 = 2222 + ) + + var ( + singleIPMask, subnetMask string + externalContainer1IP, externalContainer2IP string + ) + + checkConnectivity := func(srcPodName, dstIP string, dstPort int, shouldSucceed bool) { + testContainer := fmt.Sprintf("%s-container", srcPodName) + testContainerFlag := fmt.Sprintf("--container=%s", testContainer) + if shouldSucceed { + gomega.Eventually(func() bool { + _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", + "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)) + return err == nil + }, time.Duration(2*testTimeout)*time.Second).Should(gomega.BeTrue(), + fmt.Sprintf("expected connection from %s to [%s]:%d to suceed", srcPodName, dstIP, dstPort)) + } else { + gomega.Consistently(func() bool { + _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", + "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)) + return err != nil + }, time.Duration(2*testTimeout)*time.Second).Should(gomega.BeTrue(), + fmt.Sprintf("expected connection from %s to [%s]:%d to fail", srcPodName, dstIP, dstPort)) + } + } + + checkExternalContainerConnectivity := func(containerName, dstIP string, dstPort int) { + cmd := []string{"docker", "exec", containerName, "nc", "-vz", "-w", fmt.Sprint(testTimeout), dstIP, fmt.Sprint(dstPort)} + framework.Logf("Running command %v", cmd) + _, err := runCommand(cmd...) + if err != nil { + framework.Failf("Failed to connect from external container %s to %s:%d: %v", + containerName, dstIP, dstPort, err) + } + } + + ginkgo.BeforeEach(func() { + externalContainer1IPV4, externalContainer1IPV6 := createClusterExternalContainer(externalContainerName1, agnhostImage, + []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort1, externalContainerPort1)}, + []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort1)}) + + externalContainer2IPV4, externalContainer2IPV6 := createClusterExternalContainer(externalContainerName2, agnhostImage, + []string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort2, externalContainerPort2)}, + []string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort2)}) + + if IsIPv6Cluster(f.ClientSet) { + externalContainer1IP = externalContainer1IPV6 + externalContainer2IP = externalContainer2IPV6 + } else { + externalContainer1IP = externalContainer1IPV4 + externalContainer2IP = externalContainer2IPV4 + } + + singleIPMask = "32" + subnetMask = "24" + if IsIPv6Cluster(f.ClientSet) { + singleIPMask = "128" + subnetMask = "64" + } + }) + + ginkgo.AfterEach(func() { + deleteClusterExternalContainer(externalContainerName1) + deleteClusterExternalContainer(externalContainerName2) + }) + + ginkgo.It("Should validate the egress firewall policy functionality for allowed IP", func() { + srcPodName := "e2e-egress-fw-src-pod" + // egress firewall crd yaml configuration + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall apiVersion: k8s.ovn.org/v1 metadata: name: default @@ -223,26 +236,26 @@ spec: to: cidrSelector: %s `, f.Namespace.Name, externalContainer1IP, singleIPMask, denyAllCIDR) - applyEF(egressFirewallConfig, f.Namespace.Name) - - // create the pod that will be used as the source for the connectivity test - createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) - - // Verify the remote host/port as explicitly allowed by the firewall policy is reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined "+ - "by the external firewall policy", externalContainer1IP)) - checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true) - - // Verify the remote host/port as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined "+ - "by the external firewall policy", externalContainer2IP)) - checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false) - }) - - ginkgo.It("Should validate the egress firewall policy functionality for allowed CIDR and port", func() { - srcPodName := "e2e-egress-fw-src-pod" - // egress firewall crd yaml configuration - var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall + applyEF(egressFirewallConfig, f.Namespace.Name) + + // create the pod that will be used as the source for the connectivity test + createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) + + // Verify the remote host/port as explicitly allowed by the firewall policy is reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined "+ + "by the external firewall policy", externalContainer1IP)) + checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true) + + // Verify the remote host/port as implicitly denied by the firewall policy is not reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined "+ + "by the external firewall policy", externalContainer2IP)) + checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false) + }) + + ginkgo.It("Should validate the egress firewall policy functionality for allowed CIDR and port", func() { + srcPodName := "e2e-egress-fw-src-pod" + // egress firewall crd yaml configuration + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall apiVersion: k8s.ovn.org/v1 metadata: name: default @@ -259,20 +272,90 @@ spec: to: cidrSelector: %s `, f.Namespace.Name, externalContainer1IP, subnetMask, externalContainerPort1, denyAllCIDR) - applyEF(egressFirewallConfig, f.Namespace.Name) + applyEF(egressFirewallConfig, f.Namespace.Name) + + // create the pod that will be used as the source for the connectivity test + createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) + + // Verify the remote host/port as explicitly allowed by the firewall policy is reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed port on host %s is permitted as "+ + "defined by the external firewall policy", externalContainer1IP)) + checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true) + + // Verify the remote host/port as implicitly denied by the firewall policy is not reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as "+ + "defined by the external firewall policy", externalContainer2IP)) + checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false) + }) + + ginkgo.It("Should validate the egress firewall allows inbound connections", func() { + // 1. Create nodePort service and external container + // 2. Check connectivity works both ways + // 3. Apply deny-all egress firewall + // 4. Check only inbound traffic is allowed + + efPodName := "e2e-egress-fw-pod" + efPodPort := 1234 + serviceName := "service-for-pods" + servicePort := 31234 + + ginkgo.By("Creating the egress firewall pod") + // 1. create nodePort service and external container + endpointsSelector := map[string]string{"servicebackend": "true"} + _, err := createPod(f, efPodName, serverNodeInfo.name, f.Namespace.Name, + []string{"/agnhost", "netexec", fmt.Sprintf("--http-port=%d", efPodPort)}, endpointsSelector) + if err != nil { + framework.Failf("Failed to create pod %s: %v", efPodName, err) + } - // create the pod that will be used as the source for the connectivity test - createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) + ginkgo.By("Creating the nodePort service") + _, err = createServiceForPodsWithLabel(f, f.Namespace.Name, int32(servicePort), strconv.Itoa(efPodPort), "NodePort", endpointsSelector) + framework.ExpectNoError(err, fmt.Sprintf("unable to create nodePort service, err: %v", err)) - // Verify the remote host/port as explicitly allowed by the firewall policy is reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed port on host %s is permitted as "+ - "defined by the external firewall policy", externalContainer1IP)) - checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true) + ginkgo.By("Waiting for the endpoints to pop up") + err = framework.WaitForServiceEndpointsNum(context.TODO(), f.ClientSet, f.Namespace.Name, serviceName, 1, time.Second, wait.ForeverTestTimeout) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name) - // Verify the remote host/port as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as "+ - "defined by the external firewall policy", externalContainer2IP)) - checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false) + // 2. Check connectivity works both ways + // pod -> external container should work + ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d", + efPodName, externalContainer1IP, externalContainerPort1)) + checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, true) + + // external container -> nodePort svc should work + svc, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to fetch service: %s in namespace %s", serviceName, f.Namespace.Name) + + nodeIP := serverNodeInfo.nodeIP + ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", + externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) + checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) + + // 3. Apply deny-all egress firewall and wait for it to be applied + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default + namespace: %s +spec: + egress: + - type: Deny + to: + cidrSelector: %s +`, f.Namespace.Name, denyAllCIDR) + applyEF(egressFirewallConfig, f.Namespace.Name) + + // 4. Check that only inbound traffic is allowed + // pod -> external container should be blocked + ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d is blocked", + efPodName, externalContainer1IP, externalContainerPort1)) + checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, false) + + // external container -> nodePort svc should work + ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", + externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) + checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) + }) }) ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() { @@ -519,73 +602,4 @@ spec: `, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR) applyEF(egressFirewallConfig, f.Namespace.Name) }) - - ginkgo.It("Should validate the egress firewall allows inbound connections", func() { - // 1. Create nodePort service and external container - // 2. Check connectivity works both ways - // 3. Apply deny-all egress firewall - // 4. Check only inbound traffic is allowed - - efPodName := "e2e-egress-fw-pod" - efPodPort := 1234 - serviceName := "service-for-pods" - servicePort := 31234 - - ginkgo.By("Creating the egress firewall pod") - // 1. create nodePort service and external container - endpointsSelector := map[string]string{"servicebackend": "true"} - _, err := createPod(f, efPodName, serverNodeInfo.name, f.Namespace.Name, - []string{"/agnhost", "netexec", fmt.Sprintf("--http-port=%d", efPodPort)}, endpointsSelector) - if err != nil { - framework.Failf("Failed to create pod %s: %v", efPodName, err) - } - - ginkgo.By("Creating the nodePort service") - _, err = createServiceForPodsWithLabel(f, f.Namespace.Name, int32(servicePort), strconv.Itoa(efPodPort), "NodePort", endpointsSelector) - framework.ExpectNoError(err, fmt.Sprintf("unable to create nodePort service, err: %v", err)) - - ginkgo.By("Waiting for the endpoints to pop up") - err = framework.WaitForServiceEndpointsNum(context.TODO(), f.ClientSet, f.Namespace.Name, serviceName, 1, time.Second, wait.ForeverTestTimeout) - framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name) - - // 2. Check connectivity works both ways - // pod -> external container should work - ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d", - efPodName, externalContainer1IP, externalContainerPort1)) - checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, true) - - // external container -> nodePort svc should work - svc, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) - framework.ExpectNoError(err, "failed to fetch service: %s in namespace %s", serviceName, f.Namespace.Name) - - nodeIP := serverNodeInfo.nodeIP - ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", - externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) - checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) - - // 3. Apply deny-all egress firewall and wait for it to be applied - var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall -apiVersion: k8s.ovn.org/v1 -metadata: - name: default - namespace: %s -spec: - egress: - - type: Deny - to: - cidrSelector: %s -`, f.Namespace.Name, denyAllCIDR) - applyEF(egressFirewallConfig, f.Namespace.Name) - - // 4. Check that only inbound traffic is allowed - // pod -> external container should be blocked - ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d is blocked", - efPodName, externalContainer1IP, externalContainerPort1)) - checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, false) - - // external container -> nodePort svc should work - ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d", - externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) - checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) - }) }) From 1838308074f719f0c7199f2352dd825ff898ab69 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:22:37 +0200 Subject: [PATCH 07/14] e2e: Add EF test to make sure internal connections (pod2pod) are not affected by deny all. Signed-off-by: Nadia Pinaeva (cherry picked from commit ded63ddf04eaf50c3cdf21b6a159793e30ea84c3) --- test/e2e/egress_firewall.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index 8b222fa7f6d..573d8fa9d94 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -356,6 +356,43 @@ spec: externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort)) checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort)) }) + + ginkgo.It("Should validate the egress firewall doesn't affect internal connections", func() { + srcPodName := "e2e-egress-fw-src-pod" + dstPodName := "e2e-egress-fw-dst-pod" + dstPort := 1234 + // egress firewall crd yaml configuration + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default + namespace: %s +spec: + egress: + - type: Deny + to: + cidrSelector: %s +`, f.Namespace.Name, denyAllCIDR) + applyEF(egressFirewallConfig, f.Namespace.Name) + + // create the pod that will be used as the source for the connectivity test + createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) + + // create dst pod + dstPod, err := createPod(f, dstPodName, serverNodeInfo.name, f.Namespace.Name, + []string{"/agnhost", "netexec", fmt.Sprintf("--http-port=%d", dstPort)}, nil) + if err != nil { + framework.Failf("Failed to create dst pod %s: %v", dstPodName, err) + } + dstPodIP := dstPod.Status.PodIP + + ginkgo.By(fmt.Sprintf("Verifying connectivity to an internal pod %s is permitted", dstPodName)) + checkConnectivity(srcPodName, dstPodIP, dstPort, true) + + ginkgo.By(fmt.Sprintf("Verifying connectivity to an external host %s is not permitted as defined "+ + "by the external firewall policy", externalContainer1IP)) + checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, false) + }) }) ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() { From 1ad8c43d74dedd9c1502099a83e940766b9d3b2d Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:32:01 +0200 Subject: [PATCH 08/14] e2e: add chais test before node test. To verify no deadlock, we need an intensive follow up workload. Node-selector testing work the best, as node events handling includes iterating over all egress firewalls internally. Signed-off-by: Nadia Pinaeva (cherry picked from commit e2f4c7442b421b89c832e218a154e58f247dafb3) --- test/e2e/egress_firewall.go | 389 ++++++++++++++++++------------------ 1 file changed, 200 insertions(+), 189 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index 573d8fa9d94..3f5a33d84f4 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/onsi/ginkgo/extensions/table" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" @@ -395,193 +396,14 @@ spec: }) }) - ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() { - srcPodName := "e2e-egress-fw-src-pod" - testContainer := fmt.Sprintf("%s-container", srcPodName) - testContainerFlag := fmt.Sprintf("--container=%s", testContainer) - // use random labels in case test runs again since it's a pain to remove the label from the node - labelMatch := randStr(5) - // egress firewall crd yaml configuration - var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall -apiVersion: k8s.ovn.org/v1 -metadata: - name: default - namespace: %s -spec: - egress: - - type: Allow - to: - cidrSelector: %s/%s - - type: Allow - to: - cidrSelector: %s - ports: - - protocol: TCP - port: 80 - - type: Allow - to: - nodeSelector: - matchLabels: - %s: %s - - type: Deny - to: - cidrSelector: %s -`, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, f.Namespace.Name, labelMatch, exFWDenyCIDR) - framework.Logf("Egress Firewall CR generated: %s", egressFirewallConfig) - applyEF(egressFirewallConfig, f.Namespace.Name) - // create the pod that will be used as the source for the connectivity test - createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) - // create host networked pod - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - ginkgo.By("Creating host network pods on each node") - // get random port in case the test retries and port is already in use on host node - rand.Seed(time.Now().UnixNano()) - min := 9900 - max := 9999 - hostNetworkPort := rand.Intn(max-min+1) + min - framework.Logf("Random host networked port chosen: %d", hostNetworkPort) - for _, node := range nodes.Items { - // this creates a udp / http netexec listener which is able to receive the "hostname" - // command. We use this to validate that each endpoint is received at least once - args := []string{ - "netexec", - fmt.Sprintf("--http-port=%d", hostNetworkPort), - fmt.Sprintf("--udp-port=%d", hostNetworkPort), - } - - // create host networked Pods - _, err := createPod(f, node.Name+"-hostnet-ep", node.Name, f.Namespace.Name, []string{}, map[string]string{}, func(p *v1.Pod) { - p.Spec.Containers[0].Args = args - p.Spec.HostNetwork = true - }) - - framework.ExpectNoError(err) - - } - - ginkgo.By("Selecting additional IP addresses for serverNode on which source pod lives (networking routing to secondaryIP address on other nodes is harder to achieve)") - // add new secondary IP from node subnet to the node where the source pod lives on, - // if the cluster is v6 add an ipv6 address - toCurlSecondaryNodeIPAddresses := sets.NewString() - // Calculate and store for AfterEach new target IP addresses. - var newIP string - if node2ndaryIPs[serverNodeInfo.name] == nil { - node2ndaryIPs[serverNodeInfo.name] = make(map[int]string) - } - if utilnet.IsIPv6String(e2enode.GetAddresses(&nodes.Items[1], v1.NodeInternalIP)[0]) { - newIP = "fc00:f853:ccd:e794::" + strconv.Itoa(12) - framework.Logf("Secondary nodeIP %s for node %s", serverNodeInfo.name, newIP) - node2ndaryIPs[serverNodeInfo.name][6] = newIP - } else { - newIP = "172.18.1." + strconv.Itoa(13) - framework.Logf("Secondary nodeIP %s for node %s", serverNodeInfo.name, newIP) - node2ndaryIPs[serverNodeInfo.name][4] = newIP - } - - ginkgo.By("Adding additional IP addresses to node on which source pod lives") - for nodeName, ipFamilies := range node2ndaryIPs { - for _, ip := range ipFamilies { - // manually add the a secondary IP to each node - framework.Logf("Adding IP %s to node %s", ip, nodeName) - _, err = runCommand(containerRuntime, "exec", nodeName, "ip", "addr", "add", ip, "dev", "breth0") - if err != nil && !strings.Contains(err.Error(), "Address already assigned") { - framework.Failf("failed to add new IP address %s to node %s: %v", ip, nodeName, err) - } - toCurlSecondaryNodeIPAddresses.Insert(ip) - } - } - - // Verify basic external connectivity to ensure egress firewall is working for normal conditions - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpDnsDest, "53") - if err != nil { - framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, srcPodName, serverNodeInfo.name, err) - } - // Verify the remote host/port as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWDenyTcpDnsDest, "53") - if err == nil { - framework.Failf("Succeeded in connecting the implicitly denied remote host %s from container %s on node %s", exFWDenyTcpDnsDest, ovnContainer, serverNodeInfo.name) - } - // Verify the explicitly allowed host/port tcp port 80 rule is functional - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "80") - if err != nil { - framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) - } - // Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "443") - if err == nil { - framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) - } - - ginkgo.By("Should NOT be able to reach each host networked pod via node selector") - for _, node := range nodes.Items { - path := fmt.Sprintf("http://%s:%d/hostname", node.Status.Addresses[0].Address, hostNetworkPort) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) - if err == nil { - framework.Failf("Was able to curl node %s from container %s on node %s with no allow rule for egress firewall", node.Name, srcPodName, serverNodeInfo.name) - } - } - - ginkgo.By("Should NOT be able to reach each secondary hostIP via node selector") - for _, address := range toCurlSecondaryNodeIPAddresses.List() { - if !IsIPv6Cluster(f.ClientSet) && utilnet.IsIPv6String(address) || IsIPv6Cluster(f.ClientSet) && !utilnet.IsIPv6String(address) { - continue - } - path := fmt.Sprintf("http://%s:%d/hostname", address, hostNetworkPort) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) - if err == nil { - framework.Failf("Was able to curl node %s from container %s on nodeIP %s with no allow rule for egress firewall", address, srcPodName, serverNodeInfo.name) - } - } - - ginkgo.By("Apply label to nodes " + f.Namespace.Name + ":" + labelMatch) - patch := struct { - Metadata map[string]interface{} `json:"metadata"` - }{ - Metadata: map[string]interface{}{ - "labels": map[string]string{f.Namespace.Name: labelMatch}, - }, - } - for _, node := range nodes.Items { - patchData, err := json.Marshal(&patch) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - f.ClientSet.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.MergePatchType, patchData, metav1.PatchOptions{}) - } - - ginkgo.By("Should be able to reach each host networked pod via node selector") - for _, node := range nodes.Items { - path := fmt.Sprintf("http://%s:%d/hostname", node.Status.Addresses[0].Address, hostNetworkPort) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) - if err != nil { - framework.Failf("Failed to curl node %s from container %s on node %s: %v", node.Name, srcPodName, serverNodeInfo.name, err) - } - } - - ginkgo.By("Should be able to reach secondary hostIP via node selector") - for _, address := range toCurlSecondaryNodeIPAddresses.List() { - if !IsIPv6Cluster(f.ClientSet) && utilnet.IsIPv6String(address) || IsIPv6Cluster(f.ClientSet) && !utilnet.IsIPv6String(address) { - continue - } - path := fmt.Sprintf("http://%s:%d/hostname", address, hostNetworkPort) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) - if err != nil { - framework.Failf("Failed to curl node %s from container %s on nodeIP %s", address, srcPodName, serverNodeInfo.name) - } - } - }) - - ginkgo.It("Should validate the egress firewall DNS does not deadlock when adding many dnsNames", func() { - var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall + table.DescribeTable("Should validate the egress firewall policy functionality against cluster nodes by using node selector", + func(chaosTesting bool) { + if chaosTesting { + // apply egressfirewall with many dns names, then delete and check that the next node-selector egress firewall + // is handled correctly. + // Using node selector is the best way to check internal egress firewall locking, as node event handler + // iterates over all existing egress firewalls. + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall apiVersion: k8s.ovn.org/v1 metadata: name: default @@ -637,6 +459,195 @@ spec: to: cidrSelector: %s `, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR) - applyEF(egressFirewallConfig, f.Namespace.Name) - }) + applyEF(egressFirewallConfig, f.Namespace.Name) + framework.Logf("Deleting EgressFirewall in namespace %s", f.Namespace.Name) + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "delete", "egressfirewall", "default") + } + + srcPodName := "e2e-egress-fw-src-pod" + testContainer := fmt.Sprintf("%s-container", srcPodName) + testContainerFlag := fmt.Sprintf("--container=%s", testContainer) + // use random labels in case test runs again since it's a pain to remove the label from the node + labelMatch := randStr(5) + // egress firewall crd yaml configuration + var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default + namespace: %s +spec: + egress: + - type: Allow + to: + cidrSelector: %s/%s + - type: Allow + to: + cidrSelector: %s + ports: + - protocol: TCP + port: 80 + - type: Allow + to: + nodeSelector: + matchLabels: + %s: %s + - type: Deny + to: + cidrSelector: %s +`, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, f.Namespace.Name, labelMatch, exFWDenyCIDR) + framework.Logf("Egress Firewall CR generated: %s", egressFirewallConfig) + applyEF(egressFirewallConfig, f.Namespace.Name) + // create the pod that will be used as the source for the connectivity test + createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) + // create host networked pod + nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) + framework.ExpectNoError(err) + + if len(nodes.Items) < 3 { + framework.Failf( + "Test requires >= 3 Ready nodes, but there are only %v nodes", + len(nodes.Items)) + } + ginkgo.By("Creating host network pods on each node") + // get random port in case the test retries and port is already in use on host node + rand.Seed(time.Now().UnixNano()) + min := 9900 + max := 9999 + hostNetworkPort := rand.Intn(max-min+1) + min + framework.Logf("Random host networked port chosen: %d", hostNetworkPort) + for _, node := range nodes.Items { + // this creates a udp / http netexec listener which is able to receive the "hostname" + // command. We use this to validate that each endpoint is received at least once + args := []string{ + "netexec", + fmt.Sprintf("--http-port=%d", hostNetworkPort), + fmt.Sprintf("--udp-port=%d", hostNetworkPort), + } + + // create host networked Pods + _, err := createPod(f, node.Name+"-hostnet-ep", node.Name, f.Namespace.Name, []string{}, map[string]string{}, func(p *v1.Pod) { + p.Spec.Containers[0].Args = args + p.Spec.HostNetwork = true + }) + + framework.ExpectNoError(err) + + } + + ginkgo.By("Selecting additional IP addresses for serverNode on which source pod lives (networking routing to secondaryIP address on other nodes is harder to achieve)") + // add new secondary IP from node subnet to the node where the source pod lives on, + // if the cluster is v6 add an ipv6 address + toCurlSecondaryNodeIPAddresses := sets.NewString() + // Calculate and store for AfterEach new target IP addresses. + var newIP string + if node2ndaryIPs[serverNodeInfo.name] == nil { + node2ndaryIPs[serverNodeInfo.name] = make(map[int]string) + } + if utilnet.IsIPv6String(e2enode.GetAddresses(&nodes.Items[1], v1.NodeInternalIP)[0]) { + newIP = "fc00:f853:ccd:e794::" + strconv.Itoa(12) + framework.Logf("Secondary nodeIP %s for node %s", serverNodeInfo.name, newIP) + node2ndaryIPs[serverNodeInfo.name][6] = newIP + } else { + newIP = "172.18.1." + strconv.Itoa(13) + framework.Logf("Secondary nodeIP %s for node %s", serverNodeInfo.name, newIP) + node2ndaryIPs[serverNodeInfo.name][4] = newIP + } + + ginkgo.By("Adding additional IP addresses to node on which source pod lives") + for nodeName, ipFamilies := range node2ndaryIPs { + for _, ip := range ipFamilies { + // manually add the a secondary IP to each node + framework.Logf("Adding IP %s to node %s", ip, nodeName) + _, err = runCommand(containerRuntime, "exec", nodeName, "ip", "addr", "add", ip, "dev", "breth0") + if err != nil && !strings.Contains(err.Error(), "Address already assigned") { + framework.Failf("failed to add new IP address %s to node %s: %v", ip, nodeName, err) + } + toCurlSecondaryNodeIPAddresses.Insert(ip) + } + } + + // Verify basic external connectivity to ensure egress firewall is working for normal conditions + ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpDnsDest, "53") + if err != nil { + framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, srcPodName, serverNodeInfo.name, err) + } + // Verify the remote host/port as implicitly denied by the firewall policy is not reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest)) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWDenyTcpDnsDest, "53") + if err == nil { + framework.Failf("Succeeded in connecting the implicitly denied remote host %s from container %s on node %s", exFWDenyTcpDnsDest, ovnContainer, serverNodeInfo.name) + } + // Verify the explicitly allowed host/port tcp port 80 rule is functional + ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "80") + if err != nil { + framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) + } + // Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable + ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "443") + if err == nil { + framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) + } + + ginkgo.By("Should NOT be able to reach each host networked pod via node selector") + for _, node := range nodes.Items { + path := fmt.Sprintf("http://%s:%d/hostname", node.Status.Addresses[0].Address, hostNetworkPort) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) + if err == nil { + framework.Failf("Was able to curl node %s from container %s on node %s with no allow rule for egress firewall", node.Name, srcPodName, serverNodeInfo.name) + } + } + + ginkgo.By("Should NOT be able to reach each secondary hostIP via node selector") + for _, address := range toCurlSecondaryNodeIPAddresses.List() { + if !IsIPv6Cluster(f.ClientSet) && utilnet.IsIPv6String(address) || IsIPv6Cluster(f.ClientSet) && !utilnet.IsIPv6String(address) { + continue + } + path := fmt.Sprintf("http://%s:%d/hostname", address, hostNetworkPort) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) + if err == nil { + framework.Failf("Was able to curl node %s from container %s on nodeIP %s with no allow rule for egress firewall", address, srcPodName, serverNodeInfo.name) + } + } + + ginkgo.By("Apply label to nodes " + f.Namespace.Name + ":" + labelMatch) + patch := struct { + Metadata map[string]interface{} `json:"metadata"` + }{ + Metadata: map[string]interface{}{ + "labels": map[string]string{f.Namespace.Name: labelMatch}, + }, + } + for _, node := range nodes.Items { + patchData, err := json.Marshal(&patch) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + f.ClientSet.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.MergePatchType, patchData, metav1.PatchOptions{}) + } + + ginkgo.By("Should be able to reach each host networked pod via node selector") + for _, node := range nodes.Items { + path := fmt.Sprintf("http://%s:%d/hostname", node.Status.Addresses[0].Address, hostNetworkPort) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) + if err != nil { + framework.Failf("Failed to curl node %s from container %s on node %s: %v", node.Name, srcPodName, serverNodeInfo.name, err) + } + } + + ginkgo.By("Should be able to reach secondary hostIP via node selector") + for _, address := range toCurlSecondaryNodeIPAddresses.List() { + if !IsIPv6Cluster(f.ClientSet) && utilnet.IsIPv6String(address) || IsIPv6Cluster(f.ClientSet) && !utilnet.IsIPv6String(address) { + continue + } + path := fmt.Sprintf("http://%s:%d/hostname", address, hostNetworkPort) + _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) + if err != nil { + framework.Failf("Failed to curl node %s from container %s on nodeIP %s", address, srcPodName, serverNodeInfo.name) + } + } + }, + table.Entry("", false), + table.Entry("with chaos testing using many dnsNames", true), + ) }) From 230ea3dda6c9132b93e6c6f8f1e3fd6ab0f537e1 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:38:31 +0200 Subject: [PATCH 09/14] e2e: move node2ndaryIPs-related functionality to the only test case that needs it. Use defer to cleanup instead of afterEach, as afterEach should cleanup resources created by beforeEach. Signed-off-by: Nadia Pinaeva (cherry picked from commit 217ec65c6c67775f34116b60730297f024f1eab2) --- test/e2e/egress_firewall.go | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index 3f5a33d84f4..e77a47d3bf0 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -89,11 +89,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { f := wrappedTestFramework(svcname) - // node2ndaryIPs holds the nodeName as the key and the value is - // a map with ipFamily(v4 or v6) as the key and the secondaryIP as the value - // This is defined here globally to allow us to cleanup in AfterEach - node2ndaryIPs := make(map[string]map[int]string) - // Determine what mode the CI is running in and get relevant endpoint information for the tests ginkgo.BeforeEach(func() { nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 2) @@ -132,20 +127,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } }) - ginkgo.AfterEach(func() { - ginkgo.By("Deleting additional IP addresses from nodes") - for nodeName, ipFamilies := range node2ndaryIPs { - for _, ip := range ipFamilies { - _, err := runCommand(containerRuntime, "exec", nodeName, "ip", "addr", "delete", - fmt.Sprintf("%s/32", ip), "dev", "breth0") - if err != nil && !strings.Contains(err.Error(), - "RTNETLINK answers: Cannot assign requested address") { - framework.Failf("failed to remove ip address %s from node %s, err: %q", ip, nodeName, err) - } - } - } - }) - ginkgo.Context("with external containers", func() { const ( ciNetworkName = "kind" @@ -538,7 +519,9 @@ spec: // add new secondary IP from node subnet to the node where the source pod lives on, // if the cluster is v6 add an ipv6 address toCurlSecondaryNodeIPAddresses := sets.NewString() - // Calculate and store for AfterEach new target IP addresses. + // node2ndaryIPs holds the nodeName as the key and the value is + // a map with ipFamily(v4 or v6) as the key and the secondaryIP as the value + node2ndaryIPs := make(map[string]map[int]string) var newIP string if node2ndaryIPs[serverNodeInfo.name] == nil { node2ndaryIPs[serverNodeInfo.name] = make(map[int]string) @@ -565,6 +548,18 @@ spec: toCurlSecondaryNodeIPAddresses.Insert(ip) } } + defer func() { + for nodeName, ipFamilies := range node2ndaryIPs { + for _, ip := range ipFamilies { + // manually add the a secondary IP to each node + framework.Logf("Deleting IP %s from node %s", ip, nodeName) + _, err = runCommand(containerRuntime, "exec", nodeName, "ip", "addr", "del", ip, "dev", "breth0") + if err != nil { + framework.Logf("failed to delete secondary ip from the node %s: %v", nodeName, err) + } + } + } + }() // Verify basic external connectivity to ensure egress firewall is working for normal conditions ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) From 34ac5fc623c059643d4a742e516174cb23de2334 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:44:19 +0200 Subject: [PATCH 10/14] e2e: remove duplicated functionality from the node-selector tests. Remove unneeded external IPs from the deadlock test, as multiple unresolvable ds names is the main ingredient. Fix ip:port formatting for ipv6. Signed-off-by: Nadia Pinaeva (cherry picked from commit 3fac0f31ced22c6bb9b0c9431b93be355c2f83b8) --- test/e2e/egress_firewall.go | 86 ++++++------------------------------- 1 file changed, 13 insertions(+), 73 deletions(-) diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index e77a47d3bf0..12d2c45a1f2 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "math/rand" + "net" "os" "strconv" "strings" @@ -29,9 +30,7 @@ import ( // is properly handled as defined in the crd configuration in the test. var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { const ( - svcname string = "egress-firewall-policy" - - ovnContainer string = "ovnkube-node" + svcname string = "egress-firewall-policy" egressFirewallYamlFile string = "egress-fw.yml" testTimeout int = 3 retryInterval = 1 * time.Second @@ -44,14 +43,8 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { } var ( - serverNodeInfo nodeInfo - exFWPermitTcpDnsDest string - exFWDenyTcpDnsDest string - exFWPermitTcpWwwDest string - exFWPermitCIDR string - exFWDenyCIDR string - denyAllCIDR string - singleIPMask string + serverNodeInfo nodeInfo + denyAllCIDR string ) waitForEFApplied := func(namespace string) { @@ -106,21 +99,6 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() { nodeIP: ips[1], } - exFWPermitTcpDnsDest = "8.8.8.8" - exFWDenyTcpDnsDest = "8.8.4.4" - exFWPermitTcpWwwDest = "1.1.1.1" - exFWPermitCIDR = "1.1.1.0/24" - exFWDenyCIDR = "0.0.0.0/0" - singleIPMask = "32" - if IsIPv6Cluster(f.ClientSet) { - exFWPermitTcpDnsDest = "2001:4860:4860::8888" - exFWDenyTcpDnsDest = "2001:4860:4860::8844" - exFWPermitTcpWwwDest = "2606:4700:4700::1111" - exFWPermitCIDR = "2606:4700:4700::/64" - exFWDenyCIDR = "::/0" - singleIPMask = "128" - } - denyAllCIDR = "0.0.0.0/0" if IsIPv6Cluster(f.ClientSet) { denyAllCIDR = "::/0" @@ -391,9 +369,6 @@ metadata: namespace: %s spec: egress: - - type: Allow - to: - cidrSelector: %s/%s - type: Allow to: dnsName: www.test1.com @@ -430,16 +405,13 @@ spec: - type: Allow to: dnsName: www.test12.com - - type: Allow - to: - cidrSelector: %s ports: - protocol: TCP port: 80 - type: Deny to: cidrSelector: %s -`, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR) +`, f.Namespace.Name, denyAllCIDR) applyEF(egressFirewallConfig, f.Namespace.Name) framework.Logf("Deleting EgressFirewall in namespace %s", f.Namespace.Name) e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "delete", "egressfirewall", "default") @@ -458,15 +430,6 @@ metadata: namespace: %s spec: egress: - - type: Allow - to: - cidrSelector: %s/%s - - type: Allow - to: - cidrSelector: %s - ports: - - protocol: TCP - port: 80 - type: Allow to: nodeSelector: @@ -475,9 +438,11 @@ spec: - type: Deny to: cidrSelector: %s -`, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, f.Namespace.Name, labelMatch, exFWDenyCIDR) +`, f.Namespace.Name, f.Namespace.Name, labelMatch, denyAllCIDR) framework.Logf("Egress Firewall CR generated: %s", egressFirewallConfig) + applyEF(egressFirewallConfig, f.Namespace.Name) + // create the pod that will be used as the source for the connectivity test createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f) // create host networked pod @@ -512,7 +477,6 @@ spec: }) framework.ExpectNoError(err) - } ginkgo.By("Selecting additional IP addresses for serverNode on which source pod lives (networking routing to secondaryIP address on other nodes is harder to achieve)") @@ -561,34 +525,10 @@ spec: } }() - // Verify basic external connectivity to ensure egress firewall is working for normal conditions - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpDnsDest, "53") - if err != nil { - framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, srcPodName, serverNodeInfo.name, err) - } - // Verify the remote host/port as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWDenyTcpDnsDest, "53") - if err == nil { - framework.Failf("Succeeded in connecting the implicitly denied remote host %s from container %s on node %s", exFWDenyTcpDnsDest, ovnContainer, serverNodeInfo.name) - } - // Verify the explicitly allowed host/port tcp port 80 rule is functional - ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "80") - if err != nil { - framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) - } - // Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable - ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", fmt.Sprint(testTimeout), exFWPermitTcpWwwDest, "443") - if err == nil { - framework.Failf("Failed to curl the remote host %s from container %s on node %s: %v", exFWPermitTcpWwwDest, ovnContainer, serverNodeInfo.name, err) - } - ginkgo.By("Should NOT be able to reach each host networked pod via node selector") + hostNetworkPortStr := fmt.Sprint(hostNetworkPort) for _, node := range nodes.Items { - path := fmt.Sprintf("http://%s:%d/hostname", node.Status.Addresses[0].Address, hostNetworkPort) + path := fmt.Sprintf("http://%s/hostname", net.JoinHostPort(node.Status.Addresses[0].Address, hostNetworkPortStr)) _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) if err == nil { framework.Failf("Was able to curl node %s from container %s on node %s with no allow rule for egress firewall", node.Name, srcPodName, serverNodeInfo.name) @@ -600,7 +540,7 @@ spec: if !IsIPv6Cluster(f.ClientSet) && utilnet.IsIPv6String(address) || IsIPv6Cluster(f.ClientSet) && !utilnet.IsIPv6String(address) { continue } - path := fmt.Sprintf("http://%s:%d/hostname", address, hostNetworkPort) + path := fmt.Sprintf("http://%s/hostname", net.JoinHostPort(address, hostNetworkPortStr)) _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) if err == nil { framework.Failf("Was able to curl node %s from container %s on nodeIP %s with no allow rule for egress firewall", address, srcPodName, serverNodeInfo.name) @@ -623,7 +563,7 @@ spec: ginkgo.By("Should be able to reach each host networked pod via node selector") for _, node := range nodes.Items { - path := fmt.Sprintf("http://%s:%d/hostname", node.Status.Addresses[0].Address, hostNetworkPort) + path := fmt.Sprintf("http://%s/hostname", net.JoinHostPort(node.Status.Addresses[0].Address, hostNetworkPortStr)) _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) if err != nil { framework.Failf("Failed to curl node %s from container %s on node %s: %v", node.Name, srcPodName, serverNodeInfo.name, err) @@ -635,7 +575,7 @@ spec: if !IsIPv6Cluster(f.ClientSet) && utilnet.IsIPv6String(address) || IsIPv6Cluster(f.ClientSet) && !utilnet.IsIPv6String(address) { continue } - path := fmt.Sprintf("http://%s:%d/hostname", address, hostNetworkPort) + path := fmt.Sprintf("http://%s/hostname", net.JoinHostPort(address, hostNetworkPortStr)) _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "curl", "-g", "--max-time", "2", path) if err != nil { framework.Failf("Failed to curl node %s from container %s on nodeIP %s", address, srcPodName, serverNodeInfo.name) From f32c68f5bb16ba36c361634f18ed2691f70af2fa Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 22 May 2024 22:45:33 +0200 Subject: [PATCH 11/14] e2e: remove egressfirewall from ipv6 exceptions Signed-off-by: Nadia Pinaeva (cherry picked from commit 243c48d818cf37e01b5fb4c96e780a6ab60e7ed7) --- test/scripts/e2e-cp.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/scripts/e2e-cp.sh b/test/scripts/e2e-cp.sh index 4f93075b422..8a4fa231d1a 100755 --- a/test/scripts/e2e-cp.sh +++ b/test/scripts/e2e-cp.sh @@ -25,8 +25,6 @@ should provide Internet connection continuously when ovnkube-node pod is killed| should provide Internet connection continuously when pod running master instance of ovnkube-control-plane is killed|\ should provide Internet connection continuously when all pods are killed on node running master instance of ovnkube-control-plane|\ should provide Internet connection continuously when all ovnkube-control-plane pods are killed|\ -Should validate the egress firewall policy functionality against remote hosts|\ -Should validate the egress firewall policy functionality against cluster nodes by using node selector|\ Should validate ICMP connectivity to multiple external gateways for an ECMP scenario|\ Should validate ICMP connectivity to an external gateway\'s loopback address via a pod with external gateway annotations enabled|\ Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario|\ @@ -36,7 +34,6 @@ Should validate flow data of br-int is sent to an external gateway with netflow can retrieve multicast IGMP query|\ test node readiness according to its defaults interface MTU size|\ egress IP validation|\ -e2e egress firewall policy validation|\ Pod to pod TCP with low MTU|\ queries to the hostNetworked server pod on another node shall work for TCP|\ queries to the hostNetworked server pod on another node shall work for UDP|\ From b09216d8423ca9fb28a5bdd48604a7ccf171f85b Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Thu, 23 May 2024 10:45:58 +0200 Subject: [PATCH 12/14] e2e: go mod tidy Signed-off-by: Nadia Pinaeva (cherry picked from commit 8c1e9edd2f0455fb619ee76089edb0aa3adfeaed) --- test/e2e/go.mod | 1 + test/e2e/go.sum | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 7f8ad2d1fb8..e0d9d3cbd9d 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -152,6 +152,7 @@ require ( github.com/coreos/butane v0.18.0 github.com/docker/docker v24.0.9+incompatible github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f + github.com/onsi/ginkgo v1.16.5 github.com/openshift-kni/k8sreporter v1.0.4 github.com/ovn-org/ovn-kubernetes/go-controller v0.0.0-20230914190234-9cbdd02db9c6 go.universe.tf/metallb v0.0.0-00010101000000-000000000000 diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 5958f10a15e..bc916728610 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -319,12 +319,15 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= @@ -747,6 +750,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 514d5cfc02010e9089641368af0ed2c47106691c Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Tue, 10 Sep 2024 15:49:37 -0400 Subject: [PATCH 13/14] Adds e2e test: conntrack flush after ovnkube delete Test opens a TCP connection that simulates a GCP LB environment where the packet is redirected via iptables to a local server on a node. Note, in GCP the LB does not DNAT the VIP, so the packet arrives to the node with the GCP VIP on it. In OCP, we then redirect that packet to the local kapi server running on the node. Once the test opens the TCP connection, it leaves it open for 2 minutes while ovnkube-node is then deleted. Post ovn-controller starting it should not flush the conntrack in zone 0, and the test ensures that the conntrack entry still exists. Recent OVN regression that prompted this E2E: https://issues.redhat.com/browse/FDP-773 NOTE: The release-1.0 backport generated conflicts because on that branch the test didn't check IPv6. Signed-off-by: Tim Rozet (cherry picked from commit 45abadba6b42558d699e1d97604c6124430344b0) Signed-off-by: Dumitru Ceara --- test/e2e/e2e.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ test/e2e/util.go | 16 ++++----- 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 41a9591c075..956b56c8fed 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -47,10 +47,70 @@ const ( agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.26" iperf3Image = "quay.io/sronanrh/iperf" ovnNs = "ovn-kubernetes" //OVN kubernetes namespace + redirectIP = "123.123.123.123" + redirectPort = "13337" + exContainerName = "tcp-continuous-client" ) type podCondition = func(pod *v1.Pod) (bool, error) +// setupHostRedirectPod +func setupHostRedirectPod(f *framework.Framework, node *v1.Node, exContainerName string, isIPv6 bool) error { + _, _ = createClusterExternalContainer(exContainerName, externalContainerImage, []string{"-itd", "--privileged", "--network", externalContainerNetwork}, []string{}) + nodeV4, nodeV6 := getContainerAddressesForNetwork(node.Name, externalContainerNetwork) + mask := 32 + ipCmd := []string{"ip"} + nodeIP := nodeV4 + if isIPv6 { + mask = 128 + ipCmd = []string{"ip", "-6"} + nodeIP = nodeV6 + } + cmd := []string{"docker", "exec", exContainerName} + cmd = append(cmd, ipCmd...) + cmd = append(cmd, "route", "add", fmt.Sprintf("%s/%d", redirectIP, mask), "via", nodeIP) + _, err := runCommand(cmd...) + if err != nil { + return err + } + + // setup redirect iptables rule in node + ipTablesArgs := []string{"PREROUTING", "-t", "nat", "--dst", redirectIP, "-j", "REDIRECT"} + updateIPTablesRulesForNode("insert", node.Name, ipTablesArgs) + + command := []string{ + "bash", "-c", + fmt.Sprintf("set -xe; while true; do nc -l -p %s; done", + redirectPort), + } + tcpServer := "tcp-continuous-server" + // setup host networked pod to act as server + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: tcpServer, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: tcpServer, + Image: agnhostImage, + Command: command, + }, + }, + NodeName: node.Name, + RestartPolicy: v1.RestartPolicyNever, + HostNetwork: true, + }, + } + podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name) + _, err = podClient.Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + return err + } + err = e2epod.WaitForPodNotPending(context.TODO(), f.ClientSet, f.Namespace.Name, tcpServer) + return err +} + // checkContinuousConnectivity creates a pod and checks that it can connect to the given host over tries*2 seconds. // The created pod object is sent to the podChan while any errors along the way are sent to the errChan. // Callers are expected to read the errChan and verify that they received a nil before fetching @@ -649,6 +709,7 @@ var _ = ginkgo.Describe("e2e control plane", func() { numControlPlanePods int controlPlanePodName string controlPlaneLeaseName string + nodes []v1.Node ) ginkgo.BeforeEach(func() { @@ -683,7 +744,14 @@ var _ = ginkgo.Describe("e2e control plane", func() { if IsIPv6Cluster(f.ClientSet) { extDNSIP = "2001:4860:4860::8888" } + n, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) + framework.ExpectNoError(err) + nodes = n.Items + + }) + ginkgo.AfterEach(func() { + deleteClusterExternalContainer(exContainerName) }) ginkgo.It("should provide Internet connection continuously when ovnkube-node pod is killed", func() { @@ -701,6 +769,26 @@ var _ = ginkgo.Describe("e2e control plane", func() { testPod := <-podChan nodeName := testPod.Spec.NodeName framework.Logf("Test pod running on %q", nodeName) + var targetNode *v1.Node + for _, node := range nodes { + if node.Name == nodeName { + targetNode = &node + } + } + gomega.Expect(targetNode).ToNot(gomega.BeNil()) + err = setupHostRedirectPod(f, targetNode, exContainerName, IsIPv6Cluster(f.ClientSet)) + framework.ExpectNoError(err) + + // start TCP client + go func() { + defer ginkgo.GinkgoRecover() + _, _ = runCommand(containerRuntime, "exec", exContainerName, "nc", "--idle-timeout", "120s", redirectIP, redirectPort) + }() + + ginkgo.By("Checking that TCP redirect connection entry in conntrack before ovnkube-node restart") + gomega.Eventually(func() int { + return pokeConntrackEntries(nodeName, redirectIP, "tcp", nil) + }, "10s", "1s").ShouldNot(gomega.Equal(0)) ginkgo.By("Deleting ovn-kube pod on node " + nodeName) err = restartOVNKubeNodePod(f.ClientSet, "ovn-kubernetes", nodeName) @@ -711,6 +799,11 @@ var _ = ginkgo.Describe("e2e control plane", func() { err = waitClusterHealthy(f, numControlPlanePods, controlPlanePodName) framework.ExpectNoError(err, "one or more nodes failed to go back ready, schedulable, and untainted") + + ginkgo.By("Checking that TCP redirect connection entry in conntrack remained after ovnkube-node restart") + gomega.Consistently(func() int { + return pokeConntrackEntries(nodeName, redirectIP, "tcp", nil) + }, "5s", "500ms").ShouldNot(gomega.Equal(0)) }) ginkgo.It("should provide Internet connection continuously when pod running master instance of ovnkube-control-plane is killed", func() { diff --git a/test/e2e/util.go b/test/e2e/util.go index 361cde5d6b1..16ea4e5ec27 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1047,6 +1047,7 @@ func setUnsetTemplateContainerEnv(c kubernetes.Interface, namespace, resource, c // allowOrDropNodeInputTrafficOnPort ensures or deletes a drop iptables // input rule for the specified node, protocol and port func allowOrDropNodeInputTrafficOnPort(op, nodeName, protocol, port string) { + ipTablesArgs := []string{"INPUT", "-p", protocol, "--dport", port, "-j", "DROP"} switch op { case "Allow": op = "delete" @@ -1055,33 +1056,32 @@ func allowOrDropNodeInputTrafficOnPort(op, nodeName, protocol, port string) { default: framework.Failf("unsupported op %s", op) } + updateIPTablesRulesForNode(op, nodeName, ipTablesArgs) + updateIPTablesRulesForNode(op, nodeName, ipTablesArgs) +} +func updateIPTablesRulesForNode(op, nodeName string, ipTablesArgs []string) { args := []string{"get", "pods", "--selector=app=ovnkube-node", "--field-selector", fmt.Sprintf("spec.nodeName=%s", nodeName), "-o", "jsonpath={.items..metadata.name}"} ovnKubePodName := e2ekubectl.RunKubectlOrDie(ovnNamespace, args...) - ipTablesArgs := []string{"INPUT", "-p", protocol, "--dport", port, "-j", "DROP"} - args = []string{"exec", ovnKubePodName, "-c", getNodeContainerName(), "--", "iptables", "--check"} _, err := e2ekubectl.RunKubectl(ovnNamespace, append(args, ipTablesArgs...)...) - // errors known to be equivalent to not found notFound1 := "No chain/target/match by that name" notFound2 := "does a matching rule exist in that chain?" notFound := err != nil && (strings.Contains(err.Error(), notFound1) || strings.Contains(err.Error(), notFound2)) if err != nil && !notFound { - framework.Failf("failed to check existance of iptables rule on node %s: %v", nodeName, err) + framework.Failf("failed to check existance of %s rule on node %s: %v", "iptables", nodeName, err) } - if op == "delete" && notFound { // rule is not there return - } else if op == "append" && err == nil { + } else if op == "insert" && err == nil { // rule is already there return } - args = []string{"exec", ovnKubePodName, "-c", getNodeContainerName(), "--", "iptables", "--" + op} - framework.Logf("%s iptables input rule for protocol %s port %s action DROP on node %s", op, protocol, port, nodeName) + framework.Logf("%s iptables rule: %q on node %s", op, strings.Join(ipTablesArgs, ","), nodeName) e2ekubectl.RunKubectlOrDie(ovnNamespace, append(args, ipTablesArgs...)...) } From 10fed55bf7f4dbad2c81460bd3d5f9973391706d Mon Sep 17 00:00:00 2001 From: Patryk Diak Date: Mon, 17 Jun 2024 20:01:08 +0200 Subject: [PATCH 14/14] e2e: Fix secondaryIPV6Subnet mask The previous mask was invalid and docker was failing with: invalid subnet 2001:db8:abcd:1234:c000::/64: it should be 2001:db8:abcd:1234::/64 Signed-off-by: Patryk Diak (cherry picked from commit 511e9c699ada9a8fe9f846823b2a8917cff51aa2) --- test/e2e/egressip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/egressip.go b/test/e2e/egressip.go index 688b5a1b6ab..c623a8cfc18 100644 --- a/test/e2e/egressip.go +++ b/test/e2e/egressip.go @@ -34,7 +34,7 @@ const ( OVN_EGRESSIP_LEGACY_HEALTHCHECK_PORT = "9" // the actual port used by legacy health check primaryNetworkName = "kind" secondaryIPV4Subnet = "10.10.10.0/24" - secondaryIPV6Subnet = "2001:db8:abcd:1234:c000::/64" + secondaryIPV6Subnet = "2001:db8:abcd:1234::/64" secondaryNetworkName = "secondary-network" )