Skip to content

Commit

Permalink
Enable access to default network services from user-defined networks
Browse files Browse the repository at this point in the history 
Add OVS flows that redirect traffic from the masquerade
subnet towards UDN enabled services to the default gateway router.
For shared gateway mode a static route is added to redirect
the service traffic to the management port. Modified the
staticRouteCleanup to ensure that the new routes are not
getting removed.

Signed-off-by: Patryk Diak <[email protected]>
  • Loading branch information
@kyrtapz
kyrtapz committed Sep 20, 2024
1 parent d89b8e9 commit 354fa00
Show file tree
Hide file tree
Showing 12 changed files with 279 additions and 36 deletions.
39 changes: 26 additions & 13 deletions go-controller/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,20 @@ var (

// Default holds parsed config file parameters and command-line overrides
Default = DefaultConfig{
MTU: 1400,
ConntrackZone: 64000,
EncapType: "geneve",
EncapIP: "",
EncapPort: DefaultEncapPort,
InactivityProbe: 100000, // in Milliseconds
OpenFlowProbe: 180, // in Seconds
OfctrlWaitBeforeClear: 0, // in Milliseconds
MonitorAll: true,
OVSDBTxnTimeout: DefaultDBTxnTimeout,
LFlowCacheEnable: true,
RawClusterSubnets: "10.128.0.0/14/23",
Zone: types.OvnDefaultZone,
MTU: 1400,
ConntrackZone: 64000,
EncapType: "geneve",
EncapIP: "",
EncapPort: DefaultEncapPort,
InactivityProbe: 100000, // in Milliseconds
OpenFlowProbe: 180, // in Seconds
OfctrlWaitBeforeClear: 0, // in Milliseconds
MonitorAll: true,
OVSDBTxnTimeout: DefaultDBTxnTimeout,
LFlowCacheEnable: true,
RawClusterSubnets: "10.128.0.0/14/23",
Zone: types.OvnDefaultZone,
UDNAllowedDefaultServices: *cli.NewStringSlice("default/kubernetes", "kube-system/kube-dns"),
}

// Logging holds logging-related parsed config file parameters and command-line overrides
Expand Down Expand Up @@ -280,6 +281,10 @@ type DefaultConfig struct {

// Zone name to which ovnkube-node/ovnkube-controller belongs to
Zone string `gcfg:"zone"`

// UDNAllowedDefaultServices holds a list of namespaced names of
// default cluster network services accessible from primary user-defined networks
UDNAllowedDefaultServices cli.StringSlice `gcfg:"udn-allowed-default-services"`
}

// LoggingConfig holds logging-related parsed config file parameters and command-line overrides
Expand Down Expand Up @@ -921,6 +926,14 @@ var CommonFlags = []cli.Flag{
Value: Default.Zone,
Destination: &cliConfig.Default.Zone,
},
&cli.StringSliceFlag{
Name: "udn-allowed-default-services",
Usage: "a list of namespaced names of default cluster network services accessible from primary" +
"user-defined networks. If not specified defaults to [\"default/kubernetes\", \"kube-system/kube-dns\"]." +
"Only used when enable-network-segmentation is set",
Value: &Default.UDNAllowedDefaultServices,
Destination: &cliConfig.Default.UDNAllowedDefaultServices,
},
}

// MonitoringFlags capture monitoring-related options
Expand Down
13 changes: 7 additions & 6 deletions go-controller/pkg/factory/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@ import (
anpinformerfactory "sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions"
anpinformer "sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/apis/v1alpha1"

certificatesinformers "k8s.io/client-go/informers/certificates/v1"

ocpnetworkapiv1alpha1 "github.com/openshift/api/network/v1alpha1"
ocpnetworkscheme "github.com/openshift/client-go/network/clientset/versioned/scheme"
ocpnetworkinformerfactory "github.com/openshift/client-go/network/informers/externalversions"
ocpnetworkinformerv1alpha1 "github.com/openshift/client-go/network/informers/externalversions/network/v1alpha1"

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
egressfirewallapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1"
egressfirewallscheme "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/clientset/versioned/scheme"
Expand All @@ -20,12 +27,6 @@ import (
egressfirewalllister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/listers/egressfirewall/v1"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
certificatesinformers "k8s.io/client-go/informers/certificates/v1"

ocpnetworkapiv1alpha1 "github.com/openshift/api/network/v1alpha1"
ocpnetworkscheme "github.com/openshift/client-go/network/clientset/versioned/scheme"
ocpnetworkinformerfactory "github.com/openshift/client-go/network/informers/externalversions"
ocpnetworkinformerv1alpha1 "github.com/openshift/client-go/network/informers/externalversions/network/v1alpha1"

egressipapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1"
egressipscheme "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/clientset/versioned/scheme"
Expand Down
44 changes: 39 additions & 5 deletions go-controller/pkg/node/gateway_shared_intf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import (
"strings"
"sync"

"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
Expand All @@ -19,8 +22,6 @@ import (
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"

kapi "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
Expand Down Expand Up @@ -275,6 +276,31 @@ func (npw *nodePortWatcher) updateServiceFlowCache(service *kapi.Service, netInf
errors = append(errors, err)
}
}

// Add flows for default network services that are accessible from UDN networks
if util.IsNetworkSegmentationSupportEnabled() {
// The flow added below has a higher priority than the default network service flow:
// priority=500,ip,in_port=LOCAL,nw_dst=10.96.0.0/16 actions=ct(commit,table=2,zone=64001,nat(src=169.254.0.2))
// This ordering ensures that there is no SNAT for UDN originated traffic.

if util.IsUDNEnabledService(ktypes.NamespacedName{Namespace: service.Namespace, Name: service.Name}.String()) {
key = strings.Join([]string{"UDNAllowedSVC", service.Namespace, service.Name}, "_")
if !add {
npw.ofm.deleteFlowsByKey(key)
}

ipPrefix := "ip"
masqueradeSubnet := config.Gateway.V4MasqueradeSubnet
if !utilnet.IsIPv4String(service.Spec.ClusterIP) {
ipPrefix = "ipv6"
masqueradeSubnet = config.Gateway.V6MasqueradeSubnet
}
// table 0, user-defined network host -> OVN towards default cluster network services
npw.ofm.updateFlowCacheEntry(key, []string{fmt.Sprintf("cookie=%s, priority=600, in_port=%s, %s, %s_src=%s, %s_dst=%s,"+
"actions=ct(commit,zone=%d,table=2)",
defaultOpenFlowCookie, npw.ofm.defaultBridge.ofPortHost, ipPrefix, ipPrefix, masqueradeSubnet, ipPrefix, service.Spec.ClusterIP, config.Default.HostMasqConntrackZone)})
}
}
return utilerrors.Join(errors...)
}

Expand Down Expand Up @@ -1276,17 +1302,18 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
defaultOpenFlowCookie, ofPortHost, config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP.String(), config.Default.OVNMasqConntrackZone))
}

var protoPrefix string
var masqIP string
var protoPrefix, masqIP, masqSubnet string

// table 0, packets coming from Host -> Service
for _, svcCIDR := range config.Kubernetes.ServiceCIDRs {
if utilnet.IsIPv4CIDR(svcCIDR) {
protoPrefix = "ip"
masqIP = config.Gateway.MasqueradeIPs.V4HostMasqueradeIP.String()
masqSubnet = config.Gateway.V4MasqueradeSubnet
} else {
protoPrefix = "ipv6"
masqIP = config.Gateway.MasqueradeIPs.V6HostMasqueradeIP.String()
masqSubnet = config.Gateway.V6MasqueradeSubnet
}

// table 0, Host -> OVN towards SVC, SNAT to special IP
Expand All @@ -1295,13 +1322,18 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
"actions=ct(commit,zone=%d,nat(src=%s),table=2)",
defaultOpenFlowCookie, ofPortHost, protoPrefix, protoPrefix, svcCIDR, config.Default.HostMasqConntrackZone, masqIP))

masqDst := masqIP
if util.IsNetworkSegmentationSupportEnabled() {
// In UDN match on the whole masquerade subnet to handle replies from UDN enabled services
masqDst = masqSubnet
}
for _, netConfig := range bridge.patchedNetConfigs() {
// table 0, Reply hairpin traffic to host, coming from OVN, unSNAT
dftFlows = append(dftFlows,
fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_src=%s, %s_dst=%s,"+
"actions=ct(zone=%d,nat,table=3)",
defaultOpenFlowCookie, netConfig.ofPortPatch, protoPrefix, protoPrefix, svcCIDR,
protoPrefix, masqIP, config.Default.HostMasqConntrackZone))
protoPrefix, masqDst, config.Default.HostMasqConntrackZone))
// table 0, Reply traffic coming from OVN to outside, drop it if the DNAT wasn't done either
// at the GR load balancer or switch load balancer. It means the correct port wasn't provided.
// nodeCIDR->serviceCIDR traffic flow is internal and it shouldn't be carried to outside the cluster
Expand Down Expand Up @@ -1383,7 +1415,9 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
fmt.Sprintf("cookie=%s, priority=10, table=1, dl_dst=%s, actions=output:%s",
defaultOpenFlowCookie, bridgeMacAddress, ofPortHost))
}

defaultNetConfig := bridge.netConfig[types.DefaultNetworkName]

// table 2, dispatch from Host -> OVN
dftFlows = append(dftFlows,
fmt.Sprintf("cookie=%s, table=2, "+
Expand Down
1 change: 1 addition & 0 deletions go-controller/pkg/ovn/base_network_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

libovsdbclient "github.com/ovn-org/libovsdb/client"
"github.com/ovn-org/libovsdb/ovsdb"

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/pod"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
Expand Down
40 changes: 39 additions & 1 deletion go-controller/pkg/ovn/controller/services/repair.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"time"

libovsdbclient "github.com/ovn-org/libovsdb/client"
libovsdb "github.com/ovn-org/libovsdb/ovsdb"

libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
Expand Down Expand Up @@ -51,7 +53,7 @@ func newRepair(serviceLister corelisters.ServiceLister, nbClient libovsdbclient.
}

// runBeforeSync performs some cleanup of stale LBs and other miscellaneous setup.
func (r *repair) runBeforeSync(useTemplates bool, netInfo util.NetInfo) {
func (r *repair) runBeforeSync(useTemplates bool, netInfo util.NetInfo, nodes map[string]nodeInfo) {
// no need to lock, single-threaded.

startTime := time.Now()
Expand Down Expand Up @@ -151,6 +153,42 @@ func (r *repair) runBeforeSync(useTemplates bool, netInfo util.NetInfo) {
klog.Errorf("Failed to purge existing reject rules: %v", err)
}
}

// remove static routes for UDN enabled services that are no longer valid
udnDelPredicate := func(route *nbdb.LogicalRouterStaticRoute) bool {
if route.ExternalIDs[types.NetworkExternalID] == netInfo.GetNetworkName() &&
route.ExternalIDs[types.TopologyExternalID] == netInfo.TopologyType() {
if serviceKey, exists := route.ExternalIDs[types.UDNEnabledServiceExternalID]; exists {
if !r.unsyncedServices.Has(serviceKey) {
// the service doesn't exist
return true
}
if !util.IsUDNEnabledService(serviceKey) {
// the service is not a part of UDNAllowedDefaultServices anymore
return true
}
}
}
return false
}

if netInfo.IsPrimaryNetwork() {
var ops []libovsdb.Operation
if netInfo.TopologyType() == types.Layer2Topology {
for _, node := range nodes {
if ops, err = libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicateOps(r.nbClient, ops, netInfo.GetNetworkScopedGWRouterName(node.name), udnDelPredicate); err != nil {
klog.Errorf("Failed to create a delete logical router static route op: %v", err)
}
}
} else {
if ops, err = libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicateOps(r.nbClient, ops, netInfo.GetNetworkScopedClusterRouterName(), udnDelPredicate); err != nil {
klog.Errorf("Failed to create a delete logical router static route op: %v", err)
}
}
if _, err = libovsdbops.TransactAndCheck(r.nbClient, ops); err != nil {
klog.Errorf("Failed to delete logical router static routes: %v", err)
}
}
}

// serviceSynced is called by a ServiceController worker when it has successfully
Expand Down
Loading

0 comments on commit 354fa00

Please sign in to comment.