fix: sort resources when building MeshContext (#5391)

Signed-off-by: Ilya Lobkov <ilya.lobkov@konghq.com>

pkg/core/resources/store/options.go

@@ -158,6 +158,7 @@ type ListOptions struct {
 	PageOffset string
 	FilterFunc ListFilterFunc
 	NamePrefix string
+	Ordered    bool
 }
 
 type ListOptionsFunc func(*ListOptions)
@@ -204,6 +205,12 @@ func ListByFilterFunc(filterFunc ListFilterFunc) ListOptionsFunc {
 	}
 }
 
+func ListOrdered() ListOptionsFunc {
+	return func(opts *ListOptions) {
+		opts.Ordered = true
+	}
+}
+
 func (l *ListOptions) HashCode() string {
 	return l.Mesh
 }

pkg/core/resources/store/pagination_store.go

@@ -45,8 +45,9 @@ func (p *paginationStore) Get(ctx context.Context, resource model.Resource, opti
 func (p *paginationStore) List(ctx context.Context, list model.ResourceList, optionsFunc ...ListOptionsFunc) error {
 	opts := NewListOptions(optionsFunc...)
 
-	// Performance optimization
-	if opts.FilterFunc == nil && opts.PageSize == 0 && opts.PageOffset == "" {
+	// At least one of the following options is required to trigger the paginationStore to do work.
+	// Otherwise, it delegates the request and returns early.
+	if opts.FilterFunc == nil && opts.PageSize == 0 && opts.PageOffset == "" && !opts.Ordered {
 		return p.delegate.List(ctx, list, optionsFunc...)
 	}
 

pkg/plugins/runtime/gateway/gateway_route_generator_test.go

@@ -2,6 +2,7 @@ package gateway_test
 
 import (
 	"context"
+	"math/rand"
 	"path"
 
 	"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
@@ -1736,11 +1737,41 @@ conf:
 `, `
 type: ExternalService
 mesh: default
-name: external-httpbin
+name: external-httpbin-1
 tags:
   kuma.io/service: external-httpbin
 networking:
-  address: httpbin.com:443
+  address: httpbin-1.com:443
+  tls:
+    enabled: true
+`, `
+type: ExternalService
+mesh: default
+name: external-httpbin-2
+tags:
+  kuma.io/service: external-httpbin
+networking:
+  address: httpbin-2.com:443
+  tls:
+    enabled: true
+`, `
+type: ExternalService
+mesh: default
+name: external-httpbin-3
+tags:
+  kuma.io/service: external-httpbin
+networking:
+  address: httpbin-3.com:443
+  tls:
+    enabled: true
+`, `
+type: ExternalService
+mesh: default
+name: external-httpbin-4
+tags:
+  kuma.io/service: external-httpbin
+networking:
+  address: httpbin-4.com:443
   tls:
     enabled: true
 `, `
@@ -1844,6 +1875,10 @@ conf:
 		DescribeTable("generating xDS resources",
 			func(goldenFileName string, fixtureResources ...string) {
 				// given
+				rand.Seed(GinkgoRandomSeed())
+				rand.Shuffle(len(fixtureResources), func(i, j int) {
+					fixtureResources[i], fixtureResources[j] = fixtureResources[j], fixtureResources[i]
+				})
 				for _, resource := range fixtureResources {
 					Expect(StoreInlineFixture(rt, []byte(resource))).To(Succeed())
 				}

pkg/plugins/runtime/gateway/generator.go

@@ -148,7 +148,7 @@ func GatewayListenerInfoFromProxy(
 	var listenerInfos []GatewayListenerInfo
 
 	matchedExternalServices, err := permissions.MatchExternalServicesTrafficPermissions(
-		proxy.Dataplane, meshCtx.Resources.ExternalServices(), meshCtx.Resources.TrafficPermissions(),
+		proxy.Dataplane, externalServices, meshCtx.Resources.TrafficPermissions(),
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to find external services matched by traffic permissions")

pkg/plugins/runtime/gateway/testdata/tcp/tcp-route-no-egress.yaml

@@ -16,15 +16,51 @@ Clusters:
           - endpoint:
               address:
                 socketAddress:
-                  address: httpbin.com
+                  address: httpbin-1.com
                   portValue: 443
             loadBalancingWeight: 1
             metadata:
               filterMetadata:
                 envoy.lb:
-                  kuma.io/external-service-name: external-httpbin
+                  kuma.io/external-service-name: external-httpbin-1
                 envoy.transport_socket_match:
-                  kuma.io/external-service-name: external-httpbin
+                  kuma.io/external-service-name: external-httpbin-1
+          - endpoint:
+              address:
+                socketAddress:
+                  address: httpbin-2.com
+                  portValue: 443
+            loadBalancingWeight: 1
+            metadata:
+              filterMetadata:
+                envoy.lb:
+                  kuma.io/external-service-name: external-httpbin-2
+                envoy.transport_socket_match:
+                  kuma.io/external-service-name: external-httpbin-2
+          - endpoint:
+              address:
+                socketAddress:
+                  address: httpbin-3.com
+                  portValue: 443
+            loadBalancingWeight: 1
+            metadata:
+              filterMetadata:
+                envoy.lb:
+                  kuma.io/external-service-name: external-httpbin-3
+                envoy.transport_socket_match:
+                  kuma.io/external-service-name: external-httpbin-3
+          - endpoint:
+              address:
+                socketAddress:
+                  address: httpbin-4.com
+                  portValue: 443
+            loadBalancingWeight: 1
+            metadata:
+              filterMetadata:
+                envoy.lb:
+                  kuma.io/external-service-name: external-httpbin-4
+                envoy.transport_socket_match:
+                  kuma.io/external-service-name: external-httpbin-4
       name: external-httpbin-8490df2e58a77ae0
       outlierDetection:
         enforcingConsecutive5xx: 0
@@ -35,13 +71,37 @@ Clusters:
       perConnectionBufferLimitBytes: 32768
       transportSocketMatches:
       - match:
-          kuma.io/external-service-name: external-httpbin
-        name: httpbin.com
+          kuma.io/external-service-name: external-httpbin-1
+        name: httpbin-1.com
+        transportSocket:
+          name: envoy.transport_sockets.tls
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+            sni: httpbin-1.com
+      - match:
+          kuma.io/external-service-name: external-httpbin-2
+        name: httpbin-2.com
+        transportSocket:
+          name: envoy.transport_sockets.tls
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+            sni: httpbin-2.com
+      - match:
+          kuma.io/external-service-name: external-httpbin-3
+        name: httpbin-3.com
+        transportSocket:
+          name: envoy.transport_sockets.tls
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+            sni: httpbin-3.com
+      - match:
+          kuma.io/external-service-name: external-httpbin-4
+        name: httpbin-4.com
         transportSocket:
           name: envoy.transport_sockets.tls
           typedConfig:
             '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
-            sni: httpbin.com
+            sni: httpbin-4.com
       type: STRICT_DNS
 Endpoints:
   Resources: {}

pkg/test/resources/builders/dataplane_builder.go

@@ -103,11 +103,16 @@ func (d *DataplaneBuilder) AddInboundOfTags(tags ...string) *DataplaneBuilder {
 }
 
 func (d *DataplaneBuilder) AddInboundOfTagsMap(tags map[string]string) *DataplaneBuilder {
-	d.res.Spec.Networking.Inbound = append(d.res.Spec.Networking.Inbound, &mesh_proto.Dataplane_Networking_Inbound{
-		Port:        FirstInboundPort + uint32(len(d.res.Spec.Networking.Inbound)),
-		ServicePort: FirstInboundServicePort + uint32(len(d.res.Spec.Networking.Inbound)),
-		Tags:        tags,
-	})
+	return d.AddInbound(
+		Inbound().
+			WithPort(FirstInboundPort + uint32(len(d.res.Spec.Networking.Inbound))).
+			WithServicePort(FirstInboundServicePort + uint32(len(d.res.Spec.Networking.Inbound))).
+			WithTags(tags),
+	)
+}
+
+func (d *DataplaneBuilder) AddInbound(inbound *InboundBuilder) *DataplaneBuilder {
+	d.res.Spec.Networking.Inbound = append(d.res.Spec.Networking.Inbound, inbound.Build())
 	return d
 }
 
@@ -128,6 +133,14 @@ func (d *DataplaneBuilder) AddOutboundsToServices(services ...string) *Dataplane
 	return d
 }
 
+func (d *DataplaneBuilder) WithTransparentProxying(redirectPortOutbound, redirectPortInbound uint32) *DataplaneBuilder {
+	d.res.Spec.Networking.TransparentProxying = &mesh_proto.Dataplane_Networking_TransparentProxying{
+		RedirectPortInbound:  redirectPortInbound,
+		RedirectPortOutbound: redirectPortOutbound,
+	}
+	return d
+}
+
 func tagsKVToMap(tagsKV []string) map[string]string {
 	if len(tagsKV)%2 == 1 {
 		panic("tagsKV has to have even number of arguments")
@@ -147,3 +160,46 @@ func (d *DataplaneBuilder) WithPrometheusMetrics(config *mesh_proto.PrometheusMe
 	}
 	return d
 }
+
+type InboundBuilder struct {
+	res *mesh_proto.Dataplane_Networking_Inbound
+}
+
+func Inbound() *InboundBuilder {
+	return &InboundBuilder{
+		res: &mesh_proto.Dataplane_Networking_Inbound{
+			Tags: map[string]string{},
+		},
+	}
+}
+
+func (b *InboundBuilder) WithAddress(addr string) *InboundBuilder {
+	b.res.Address = addr
+	return b
+}
+
+func (b *InboundBuilder) WithPort(port uint32) *InboundBuilder {
+	b.res.Port = port
+	return b
+}
+
+func (b *InboundBuilder) WithServicePort(port uint32) *InboundBuilder {
+	b.res.ServicePort = port
+	return b
+}
+
+func (b *InboundBuilder) WithTags(tags map[string]string) *InboundBuilder {
+	for k, v := range tags {
+		b.res.Tags[k] = v
+	}
+	return b
+}
+
+func (b *InboundBuilder) WithService(name string) *InboundBuilder {
+	b.WithTags(map[string]string{mesh_proto.ServiceTag: name})
+	return b
+}
+
+func (b *InboundBuilder) Build() *mesh_proto.Dataplane_Networking_Inbound {
+	return b.res
+}

pkg/test/runtime/runtime.go

@@ -20,6 +20,7 @@ import (
 	core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
 	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
 	"github.com/kumahq/kuma/pkg/core/resources/registry"
+	"github.com/kumahq/kuma/pkg/core/resources/store"
 	core_runtime "github.com/kumahq/kuma/pkg/core/runtime"
 	"github.com/kumahq/kuma/pkg/core/runtime/component"
 	secret_cipher "github.com/kumahq/kuma/pkg/core/secrets/cipher"
@@ -75,7 +76,7 @@ func BuilderFor(appCtx context.Context, cfg kuma_cp.Config) (*core_runtime.Build
 
 	builder.
 		WithComponentManager(component.NewManager(leader_memory.NewAlwaysLeaderElector())).
-		WithResourceStore(resources_memory.NewStore()).
+		WithResourceStore(store.NewPaginationStore(resources_memory.NewStore())).
 		WithSecretStore(secret_store.NewSecretStore(builder.ResourceStore())).
 		WithResourceValidators(core_runtime.ResourceValidators{
 			Dataplane: dataplane.NewMembershipValidator(),

pkg/xds/context/mesh_context_builder.go

@@ -154,7 +154,7 @@ func (m *meshContextBuilder) fetchCrossMesh(
 		return err
 	}
 
-	if err := m.rm.List(ctx, local, core_store.ListByMesh(localMesh)); err != nil {
+	if err := m.rm.List(ctx, local, core_store.ListByMesh(localMesh), core_store.ListOrdered()); err != nil {
 		return err
 	}
 
@@ -169,7 +169,7 @@ func (m *meshContextBuilder) fetchCrossMesh(
 		if err != nil {
 			return err
 		}
-		if err := m.rm.List(ctx, allOtherMeshItems, core_store.ListByMesh(otherMesh)); err != nil {
+		if err := m.rm.List(ctx, allOtherMeshItems, core_store.ListByMesh(otherMesh), core_store.ListOrdered()); err != nil {
 			return err
 		}
 
@@ -209,7 +209,7 @@ func (m *meshContextBuilder) fetchResources(ctx context.Context, mesh *core_mesh
 		switch typ {
 		case core_mesh.MeshType:
 			meshes := &core_mesh.MeshResourceList{}
-			if err := m.rm.List(ctx, meshes); err != nil {
+			if err := m.rm.List(ctx, meshes, core_store.ListOrdered()); err != nil {
 				return Resources{}, err
 			}
 			otherMeshes := &core_mesh.MeshResourceList{}
@@ -223,20 +223,20 @@ func (m *meshContextBuilder) fetchResources(ctx context.Context, mesh *core_mesh
 			resources.MeshLocalResources[typ] = otherMeshes
 		case core_mesh.ZoneIngressType:
 			zoneIngresses := &core_mesh.ZoneIngressResourceList{}
-			if err := m.rm.List(ctx, zoneIngresses); err != nil {
+			if err := m.rm.List(ctx, zoneIngresses, core_store.ListOrdered()); err != nil {
 				return Resources{}, err
 			}
 			resources.MeshLocalResources[typ] = zoneIngresses
 		case core_mesh.ZoneEgressType:
 			zoneEgresses := &core_mesh.ZoneEgressResourceList{}
-			if err := m.rm.List(ctx, zoneEgresses); err != nil {
+			if err := m.rm.List(ctx, zoneEgresses, core_store.ListOrdered()); err != nil {
 				return Resources{}, err
 			}
 			resources.MeshLocalResources[typ] = zoneEgresses
 		case system.ConfigType:
 			configs := &system.ConfigResourceList{}
 			var items []*system.ConfigResource
-			if err := m.rm.List(ctx, configs); err != nil {
+			if err := m.rm.List(ctx, configs, core_store.ListOrdered()); err != nil {
 				return Resources{}, err
 			}
 			for _, config := range configs.Items {
@@ -255,7 +255,7 @@ func (m *meshContextBuilder) fetchResources(ctx context.Context, mesh *core_mesh
 			}
 
 			insights := &core_mesh.ServiceInsightResourceList{}
-			if err := m.rm.List(ctx, insights, core_store.ListByMesh(mesh.Meta.GetName())); err != nil {
+			if err := m.rm.List(ctx, insights, core_store.ListByMesh(mesh.Meta.GetName()), core_store.ListOrdered()); err != nil {
 				return Resources{}, err
 			}
 
@@ -265,7 +265,7 @@ func (m *meshContextBuilder) fetchResources(ctx context.Context, mesh *core_mesh
 			if err != nil {
 				return Resources{}, err
 			}
-			if err := m.rm.List(ctx, rlist, core_store.ListByMesh(mesh.Meta.GetName())); err != nil {
+			if err := m.rm.List(ctx, rlist, core_store.ListByMesh(mesh.Meta.GetName()), core_store.ListOrdered()); err != nil {
 				return Resources{}, err
 			}
 			resources.MeshLocalResources[typ] = rlist

pkg/xds/server/v3/components.go

@@ -85,7 +85,7 @@ func DefaultReconciler(
 	)
 
 	return &reconciler{
-		generator: &templateSnapshotGenerator{
+		generator: &TemplateSnapshotGenerator{
 			ResourceSetHooks:      rt.XDSHooks().ResourceSetHooks(),
 			ProxyTemplateResolver: resolver,
 		},
@@ -110,7 +110,7 @@ func DefaultIngressReconciler(
 	}
 
 	return &reconciler{
-		generator: &templateSnapshotGenerator{
+		generator: &TemplateSnapshotGenerator{
 			ResourceSetHooks:      rt.XDSHooks().ResourceSetHooks(),
 			ProxyTemplateResolver: resolver,
 		},
@@ -135,7 +135,7 @@ func DefaultEgressReconciler(
 	}
 
 	return &reconciler{
-		generator: &templateSnapshotGenerator{
+		generator: &TemplateSnapshotGenerator{
 			ResourceSetHooks:      rt.XDSHooks().ResourceSetHooks(),
 			ProxyTemplateResolver: resolver,
 		},