Skip to content

Commit

Permalink
fix(MeshCircuitBreaker): apply to real resource targeted policies wit…
Browse files Browse the repository at this point in the history 
…h MeshGateway (#11557)

Signed-off-by: Mike Beaumont <[email protected]>
  • Loading branch information
@michaelbeaumont
michaelbeaumont committed Sep 25, 2024
1 parent 412c77c commit 43bdc8e
Show file tree
Hide file tree
Showing 3 changed files with 195 additions and 26 deletions.
69 changes: 53 additions & 16 deletions pkg/plugins/policies/meshcircuitbreaker/plugin/v1alpha1/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
core_plugins "github.com/kumahq/kuma/pkg/core/plugins"
core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
meshexternalservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshexternalservice/api/v1alpha1"
core_model "github.com/kumahq/kuma/pkg/core/resources/model"
core_xds "github.com/kumahq/kuma/pkg/core/xds"
xds_types "github.com/kumahq/kuma/pkg/core/xds/types"
"github.com/kumahq/kuma/pkg/plugins/policies/core/matchers"
Expand Down Expand Up @@ -63,11 +64,11 @@ func (p plugin) Apply(
return err
}

if err := applyToGateways(policies.GatewayRules, clusters.Gateway, proxy); err != nil {
if err := applyToGateways(ctx.Mesh, proxy, rs, policies.GatewayRules, clusters.Gateway); err != nil {
return err
}

if err := applyToRealResources(rs, policies.ToRules.ResourceRules, ctx.Mesh); err != nil {
if err := applyToRealResources(ctx.Mesh, rs, policies.ToRules.ResourceRules); err != nil {
return err
}

Expand Down Expand Up @@ -127,10 +128,14 @@ func applyToOutbounds(
}

func applyToGateways(
meshCtx xds_context.MeshContext,
proxy *core_xds.Proxy,
rs *core_xds.ResourceSet,
gatewayRules core_rules.GatewayRules,
gatewayClusters map[string]*envoy_cluster.Cluster,
proxy *core_xds.Proxy,
) error {
resourcesByOrigin := rs.IndexByOrigin(core_xds.NonMeshExternalService)

for _, listenerInfo := range gateway.ExtractGatewayListeners(proxy) {
rules, ok := gatewayRules.ToRules.ByListener[core_rules.InboundListener{
Address: proxy.Dataplane.Spec.GetNetworking().Address,
Expand Down Expand Up @@ -161,6 +166,22 @@ func applyToGateways(
); err != nil {
return err
}

// This happens when using MeshGatewayRoutes
if dest.BackendRef == nil {
continue
}
if realRef := dest.BackendRef.ResourceOrNil(); realRef != nil {
resources := resourcesByOrigin[*realRef]
if err := applyToRealResource(
meshCtx,
rules.ResourceRules,
*realRef,
resources,
); err != nil {
return err
}
}
}
}
}
Expand Down Expand Up @@ -216,26 +237,42 @@ func applyToEgressRealResources(rs *core_xds.ResourceSet, proxy *core_xds.Proxy)
return nil
}

func applyToRealResources(rs *core_xds.ResourceSet, rules core_rules.ResourceRules, meshCtx xds_context.MeshContext) error {
for uri, resType := range rs.IndexByOrigin(core_xds.NonMeshExternalService) {
conf := rules.Compute(uri, meshCtx.Resources)
if conf == nil {
continue
}
func applyToRealResource(
meshCtx xds_context.MeshContext,
rules core_rules.ResourceRules,
uri core_model.TypedResourceIdentifier,
resourcesByType core_xds.ResourcesByType,
) error {
conf := rules.Compute(uri, meshCtx.Resources)
if conf == nil {
return nil
}

for typ, resources := range resType {
switch typ {
case envoy_resource.ClusterType:
err := configureClusters(resources, conf.Conf[0].(api.Conf))
if err != nil {
return err
}
for typ, resources := range resourcesByType {
switch typ {
case envoy_resource.ClusterType:
err := configureClusters(resources, conf.Conf[0].(api.Conf))
if err != nil {
return err
}
}
}
return nil
}

func applyToRealResources(
meshCtx xds_context.MeshContext,
rs *core_xds.ResourceSet,
rules core_rules.ResourceRules,
) error {
for uri, resType := range rs.IndexByOrigin(core_xds.NonMeshExternalService) {
if err := applyToRealResource(meshCtx, rules, uri, resType); err != nil {
return err
}
}
return nil
}

func configureClusters(resources []*core_xds.Resource, conf api.Conf) error {
for _, resource := range resources {
configurer := plugin_xds.Configurer{
Expand Down
128 changes: 118 additions & 10 deletions pkg/plugins/policies/meshcircuitbreaker/plugin/v1alpha1/plugin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,22 @@ import (
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/util/intstr"

common_api "github.com/kumahq/kuma/api/common/v1alpha1"
mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
core_plugins "github.com/kumahq/kuma/pkg/core/plugins"
core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
meshexternalservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshexternalservice/api/v1alpha1"
meshservice_api "github.com/kumahq/kuma/pkg/core/resources/apis/meshservice/api/v1alpha1"
core_model "github.com/kumahq/kuma/pkg/core/resources/model"
core_xds "github.com/kumahq/kuma/pkg/core/xds"
xds_types "github.com/kumahq/kuma/pkg/core/xds/types"
core_rules "github.com/kumahq/kuma/pkg/plugins/policies/core/rules"
api "github.com/kumahq/kuma/pkg/plugins/policies/meshcircuitbreaker/api/v1alpha1"
plugin "github.com/kumahq/kuma/pkg/plugins/policies/meshcircuitbreaker/plugin/v1alpha1"
meshhttproute_api "github.com/kumahq/kuma/pkg/plugins/policies/meshhttproute/api/v1alpha1"
meshhttproute_plugin "github.com/kumahq/kuma/pkg/plugins/policies/meshhttproute/plugin/v1alpha1"
meshtcproute_api "github.com/kumahq/kuma/pkg/plugins/policies/meshtcproute/api/v1alpha1"
meshtcproute_plugin "github.com/kumahq/kuma/pkg/plugins/policies/meshtcproute/plugin/v1alpha1"
gateway_plugin "github.com/kumahq/kuma/pkg/plugins/runtime/gateway"
"github.com/kumahq/kuma/pkg/test"
"github.com/kumahq/kuma/pkg/test/matchers"
Expand All @@ -42,10 +48,8 @@ import (
var _ = Describe("MeshCircuitBreaker", func() {
backendMeshServiceIdentifier := core_model.TypedResourceIdentifier{
ResourceIdentifier: core_model.ResourceIdentifier{
Name: "backend",
Mesh: "default",
Namespace: "backend-ns",
Zone: "zone-1",
Name: "backend",
Mesh: "default",
},
ResourceType: "MeshService",
SectionName: "",
Expand Down Expand Up @@ -564,8 +568,12 @@ var _ = Describe("MeshCircuitBreaker", func() {
})

type gatewayTestCase struct {
name string
rules core_rules.GatewayRules
name string
gatewayRoutes []*core_mesh.MeshGatewayRouteResource
meshhttproutes core_rules.GatewayRules
meshtcproutes core_rules.GatewayRules
meshservices []*meshservice_api.MeshServiceResource
rules core_rules.GatewayRules
}
DescribeTable("should generate proper Envoy config for MeshGateways",
func(given gatewayTestCase) {
Expand All @@ -574,8 +582,15 @@ var _ = Describe("MeshCircuitBreaker", func() {
resources.MeshLocalResources[core_mesh.MeshGatewayType] = &core_mesh.MeshGatewayResourceList{
Items: []*core_mesh.MeshGatewayResource{samples.GatewayResource()},
}
resources.MeshLocalResources[core_mesh.MeshGatewayRouteType] = &core_mesh.MeshGatewayRouteResourceList{
Items: []*core_mesh.MeshGatewayRouteResource{samples.BackendGatewayRoute()},
if len(given.gatewayRoutes) > 0 {
resources.MeshLocalResources[core_mesh.MeshGatewayRouteType] = &core_mesh.MeshGatewayRouteResourceList{
Items: given.gatewayRoutes,
}
}
if len(given.meshservices) > 0 {
resources.MeshLocalResources[meshservice_api.MeshServiceType] = &meshservice_api.MeshServiceResourceList{
Items: given.meshservices,
}
}

xdsCtx := *xds_builders.Context().
Expand All @@ -585,7 +600,11 @@ var _ = Describe("MeshCircuitBreaker", func() {
Build()
proxy := xds_builders.Proxy().
WithDataplane(samples.GatewayDataplaneBuilder()).
WithPolicies(xds_builders.MatchedPolicies().WithGatewayPolicy(api.MeshCircuitBreakerType, given.rules)).
WithPolicies(xds_builders.MatchedPolicies().
WithGatewayPolicy(api.MeshCircuitBreakerType, given.rules).
WithGatewayPolicy(meshhttproute_api.MeshHTTPRouteType, given.meshhttproutes).
WithGatewayPolicy(meshtcproute_api.MeshTCPRouteType, given.meshtcproutes),
).
Build()
for n, p := range core_plugins.Plugins().ProxyPlugins() {
Expect(p.Apply(context.Background(), xdsCtx.Mesh, proxy)).To(Succeed(), n)
Expand All @@ -594,6 +613,12 @@ var _ = Describe("MeshCircuitBreaker", func() {
generatedResources, err := gatewayGenerator.Generate(context.Background(), nil, xdsCtx, proxy)
Expect(err).NotTo(HaveOccurred())

httpRoutePlugin := meshhttproute_plugin.NewPlugin().(core_plugins.PolicyPlugin)
Expect(httpRoutePlugin.Apply(generatedResources, xdsCtx, proxy)).To(Succeed())

tcpRoutePlugin := meshtcproute_plugin.NewPlugin().(core_plugins.PolicyPlugin)
Expect(tcpRoutePlugin.Apply(generatedResources, xdsCtx, proxy)).To(Succeed())

// when
plugin := plugin.NewPlugin().(core_plugins.PolicyPlugin)
Expect(plugin.Apply(generatedResources, xdsCtx, proxy)).To(Succeed())
Expand All @@ -603,7 +628,8 @@ var _ = Describe("MeshCircuitBreaker", func() {
To(matchers.MatchGoldenYAML(filepath.Join("testdata", fmt.Sprintf("%s.gateway_cluster.golden.yaml", given.name))))
},
Entry("basic outbound cluster with connection limits", gatewayTestCase{
name: "basic",
name: "basic",
gatewayRoutes: []*core_mesh.MeshGatewayRouteResource{samples.BackendGatewayRoute()},
rules: core_rules.GatewayRules{
ToRules: core_rules.GatewayToRules{
ByListener: map[core_rules.InboundListener]core_rules.ToRules{
Expand All @@ -622,5 +648,87 @@ var _ = Describe("MeshCircuitBreaker", func() {
},
},
}),
Entry("real MeshService targeted to real MeshService", gatewayTestCase{
name: "real-MeshService-targeted-to-real-MeshService",
meshservices: []*meshservice_api.MeshServiceResource{
{
Meta: &test_model.ResourceMeta{Name: "backend", Mesh: "default"},
Spec: &meshservice_api.MeshService{
Selector: meshservice_api.Selector{},
Ports: []meshservice_api.Port{{
Port: 80,
TargetPort: intstr.FromInt(8084),
AppProtocol: core_mesh.ProtocolHTTP,
}},
Identities: []meshservice_api.MeshServiceIdentity{
{
Type: meshservice_api.MeshServiceIdentityServiceTagType,
Value: "backend",
},
{
Type: meshservice_api.MeshServiceIdentityServiceTagType,
Value: "other-backend",
},
},
},
Status: &meshservice_api.MeshServiceStatus{
VIPs: []meshservice_api.VIP{{
IP: "10.0.0.1",
}},
},
},
},
meshhttproutes: core_rules.GatewayRules{
ToRules: core_rules.GatewayToRules{
ByListenerAndHostname: map[core_rules.InboundListenerHostname]core_rules.ToRules{
core_rules.NewInboundListenerHostname("192.168.0.1", 8080, "*"): {
Rules: core_rules.Rules{{
Subset: core_rules.MeshSubset(),
Conf: meshhttproute_api.PolicyDefault{
Rules: []meshhttproute_api.Rule{{
Matches: []meshhttproute_api.Match{{
Path: &meshhttproute_api.PathMatch{
Type: meshhttproute_api.Exact,
Value: "/",
},
}},
Default: meshhttproute_api.RuleConf{
BackendRefs: &[]common_api.BackendRef{{
TargetRef: builders.TargetRefService("backend"),
Port: pointer.To(uint32(80)),
Weight: pointer.To(uint(100)),
}},
},
}},
},
Origin: []core_model.ResourceMeta{
&test_model.ResourceMeta{Mesh: "default", Name: "http-route"},
},
BackendRefOriginIndex: core_rules.BackendRefOriginIndex{
meshhttproute_api.HashMatches([]meshhttproute_api.Match{{Path: &meshhttproute_api.PathMatch{Type: meshhttproute_api.Exact, Value: "/"}}}): 0,
},
}},
},
},
},
},
rules: core_rules.GatewayRules{
ToRules: core_rules.GatewayToRules{
ByListener: map[core_rules.InboundListener]core_rules.ToRules{
{Address: "192.168.0.1", Port: 8080}: {
ResourceRules: map[core_model.TypedResourceIdentifier]core_rules.ResourceRule{
backendMeshServiceIdentifier: {
Conf: []interface{}{
api.Conf{
ConnectionLimits: genConnectionLimits(),
},
},
},
},
},
},
},
},
}),
)
})
24 changes: 24 additions & 0 deletions ...alpha1/testdata/real-MeshService-targeted-to-real-MeshService.gateway_cluster.golden.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
resources:
- name: default_backend___msvc_80-01804e3659fbd290
resource:
'@type': type.googleapis.com/envoy.config.cluster.v3.Cluster
circuitBreakers:
thresholds:
- maxConnectionPools: 1111
maxConnections: 2222
maxPendingRequests: 3333
maxRequests: 4444
maxRetries: 5555
edsClusterConfig:
edsConfig:
ads: {}
initialFetchTimeout: 0s
resourceApiVersion: V3
name: default_backend___msvc_80-01804e3659fbd290
perConnectionBufferLimitBytes: 32768
type: EDS
typedExtensionProtocolOptions:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
'@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicitHttpConfig:
httpProtocolOptions: {}

0 comments on commit 43bdc8e

Please sign in to comment.