From bcb6b15b72a0d77c13042526467caa1ca8e990a8 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Thu, 26 Jan 2023 17:05:31 +0100 Subject: [PATCH 1/5] set maximum number of connections to 60000 Signed-off-by: Felix Breuer --- image/install-alpine.yaml | 2 +- image/sysctl-yawollet.conf | 2 +- internal/helper/yawollet.go | 38 ++++++++++++++++++++++++++++--------- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/image/install-alpine.yaml b/image/install-alpine.yaml index 70b26230..2910ba9a 100644 --- a/image/install-alpine.yaml +++ b/image/install-alpine.yaml @@ -209,7 +209,7 @@ lineinfile: path: /etc/rc.conf regexp: "rc_ulimit=" - line: "rc_ulimit='-n 100000'" + line: "rc_ulimit='-n 130000'" - name: Set boot timout to 1 lineinfile: diff --git a/image/sysctl-yawollet.conf b/image/sysctl-yawollet.conf index 3f9655a5..35ce25af 100644 --- a/image/sysctl-yawollet.conf +++ b/image/sysctl-yawollet.conf @@ -1,3 +1,3 @@ net.core.somaxconn = 65534 net.ipv4.tcp_max_syn_backlog = 100000 -net.core.netdev_max_backlog = 100000 \ No newline at end of file +net.core.netdev_max_backlog = 100000 diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 3616c14c..2677bea0 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -14,6 +14,7 @@ import ( "github.com/golang/protobuf/ptypes/wrappers" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" corev1 "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,6 +36,7 @@ import ( envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoylistener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoyrbacconfig "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + envoyconnection "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/connection_limit/v3" envoyrbac "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/rbac/v3" envoytcp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" envoyudp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3" @@ -305,6 +307,16 @@ func createEnvoyTCPListener( idleTimeout = durationpb.New(lb.Spec.Options.TCPIdleTimeout.Duration) } + connectionLimit, err := anypb.New(&envoyconnection.ConnectionLimit{ + StatPrefix: "envoyconnectionlimit", + // this is limited by the number of available ports on the machine + MaxConnections: wrapperspb.UInt64(60000), + }) + + if err != nil { + panic(err) + } + listenPort, err := anypb.New(&envoytcp.TcpProxy{ StatPrefix: "envoytcp", ClusterSpecifier: &envoytcp.TcpProxy_Cluster{Cluster: fmt.Sprintf("%v-%v", port.Protocol, port.Port)}, @@ -344,17 +356,25 @@ func createEnvoyTCPListener( }, }, FilterChains: []*envoylistener.FilterChain{{ - // proxy filter has to be the last in the chain - Filters: append(filters, &envoylistener.Filter{ - Name: envoywellknown.TCPProxy, - ConfigType: &envoylistener.Filter_TypedConfig{ - TypedConfig: listenPort, + Filters: append(filters, + &envoylistener.Filter{ + // NOTE: this is not present in envoywellknown + Name: "extensions.filters.network.connection_limit", + ConfigType: &envoylistener.Filter_TypedConfig{ + TypedConfig: connectionLimit, + }, }, - }), + // proxy filter has to be the last in the chain + &envoylistener.Filter{ + Name: envoywellknown.TCPProxy, + ConfigType: &envoylistener.Filter_TypedConfig{ + TypedConfig: listenPort, + }, + }), }}, - Freebind: &wrappers.BoolValue{Value: true}, - EnableReusePort: &wrappers.BoolValue{Value: true}, - PerConnectionBufferLimitBytes: &wrappers.UInt32Value{Value: 32768}, // 32 Kib + Freebind: wrapperspb.Bool(true), + EnableReusePort: wrapperspb.Bool(true), + PerConnectionBufferLimitBytes: wrapperspb.UInt32(32768), // 32 Kib } } From 78ba511edbacc7443f14125a96d43fd25486f926 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Mon, 30 Jan 2023 11:45:45 +0100 Subject: [PATCH 2/5] extract filter names into constants Signed-off-by: Felix Breuer --- internal/helper/yawollet.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 2677bea0..bedb64fa 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -58,6 +58,12 @@ type LoadbalancerConditionStatus string // LoadbalancerMetric metric name for lbm type LoadbalancerMetric string +// TODO: replace with constants from envoywellknown when available +const ( + FilterConnectionLimit string = "extensions.filters.network.connection_limit" + FilterUDPProxy string = "envoy.filters.udp_listener.udp_proxy" +) + // Condition status const const ( ConditionTrue LoadbalancerConditionStatus = "True" @@ -358,8 +364,7 @@ func createEnvoyTCPListener( FilterChains: []*envoylistener.FilterChain{{ Filters: append(filters, &envoylistener.Filter{ - // NOTE: this is not present in envoywellknown - Name: "extensions.filters.network.connection_limit", + Name: FilterConnectionLimit, ConfigType: &envoylistener.Filter_TypedConfig{ TypedConfig: connectionLimit, }, @@ -433,8 +438,7 @@ func createEnvoyUDPListener( }, ListenerFilters: []*envoylistener.ListenerFilter{ { - // TODO this is not in envoywellknown - Name: "envoy.filters.udp_listener.udp_proxy", + Name: FilterUDPProxy, ConfigType: &envoylistener.ListenerFilter_TypedConfig{ TypedConfig: listenPort, }, From f26d4a25b17accbcfee73c2e50c6b2f5adca20d2 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Mon, 30 Jan 2023 16:07:39 +0100 Subject: [PATCH 3/5] change filter order Signed-off-by: Felix Breuer --- internal/helper/errors.go | 1 + internal/helper/yawollet.go | 50 ++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/internal/helper/errors.go b/internal/helper/errors.go index b98f5fb5..7438b4ce 100644 --- a/internal/helper/errors.go +++ b/internal/helper/errors.go @@ -63,6 +63,7 @@ var ( ErrCANotFoundInSecret = errors.New("ca in secret not found") ErrNoNetworkID = errors.New("cant get networkID for loadbalancer") ErrCouldNotParseSourceRange = errors.New("could not parse LoadBalancerSourceRange") + ErrCouldNotMarshalConnectionLimit = errors.New("could not marshal ConnectionLimit") ErrListingChildLBMs = errors.New("unable to list child loadbalancerMachines") ErrUnsupportedProtocol = errors.New("unsupported protocol used (TCP and UDP is supported)") ErrProjectIsImmutable = errors.New("project id is immutable, cant be changed after initial creation") diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index bedb64fa..35a077f0 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -313,16 +313,6 @@ func createEnvoyTCPListener( idleTimeout = durationpb.New(lb.Spec.Options.TCPIdleTimeout.Duration) } - connectionLimit, err := anypb.New(&envoyconnection.ConnectionLimit{ - StatPrefix: "envoyconnectionlimit", - // this is limited by the number of available ports on the machine - MaxConnections: wrapperspb.UInt64(60000), - }) - - if err != nil { - panic(err) - } - listenPort, err := anypb.New(&envoytcp.TcpProxy{ StatPrefix: "envoytcp", ClusterSpecifier: &envoytcp.TcpProxy_Cluster{Cluster: fmt.Sprintf("%v-%v", port.Protocol, port.Port)}, @@ -348,6 +338,23 @@ func createEnvoyTCPListener( } } + connectionLimit, err := anypb.New(&envoyconnection.ConnectionLimit{ + StatPrefix: "envoyconnectionlimit", + // this is limited by the number of available ports on the machine + MaxConnections: wrapperspb.UInt64(60000), + }) + + if err == nil { + filters = append(filters, &envoylistener.Filter{ + Name: FilterConnectionLimit, + ConfigType: &envoylistener.Filter_TypedConfig{ + TypedConfig: connectionLimit, + }, + }) + } else { + _ = kubernetes.SendErrorAsEvent(r, ErrCouldNotMarshalConnectionLimit, lb) + } + return &envoylistener.Listener{ Name: fmt.Sprintf("%v-%v", port.Protocol, port.Port), Address: &envoycore.Address{ @@ -362,20 +369,13 @@ func createEnvoyTCPListener( }, }, FilterChains: []*envoylistener.FilterChain{{ - Filters: append(filters, - &envoylistener.Filter{ - Name: FilterConnectionLimit, - ConfigType: &envoylistener.Filter_TypedConfig{ - TypedConfig: connectionLimit, - }, - }, + Filters: append(filters, &envoylistener.Filter{ // proxy filter has to be the last in the chain - &envoylistener.Filter{ - Name: envoywellknown.TCPProxy, - ConfigType: &envoylistener.Filter_TypedConfig{ - TypedConfig: listenPort, - }, - }), + Name: envoywellknown.TCPProxy, + ConfigType: &envoylistener.Filter_TypedConfig{ + TypedConfig: listenPort, + }, + }), }}, Freebind: wrapperspb.Bool(true), EnableReusePort: wrapperspb.Bool(true), @@ -495,13 +495,13 @@ func createEnvoyRBACRules( split := strings.Split(sourceRange, "/") if err != nil || len(split) != 2 { - _ = kubernetes.SendErrorAsEvent(r, fmt.Errorf("%w: SourceRage: %s", ErrCouldNotParseSourceRange, sourceRange), lb) + _ = kubernetes.SendErrorAsEvent(r, fmt.Errorf("%w: SourceRange: %s", ErrCouldNotParseSourceRange, sourceRange), lb) continue } prefix, err := strconv.ParseUint(split[1], 10, 32) if err != nil { - _ = kubernetes.SendErrorAsEvent(r, fmt.Errorf("%w: SourceRage: %s", ErrCouldNotParseSourceRange, sourceRange), lb) + _ = kubernetes.SendErrorAsEvent(r, fmt.Errorf("%w: SourceRange: %s", ErrCouldNotParseSourceRange, sourceRange), lb) continue } From 28477925cbf38e74e4ceb4885e7b9508331a736b Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Wed, 1 Feb 2023 14:22:31 +0100 Subject: [PATCH 4/5] use overlaod manager to limit connections Signed-off-by: Felix Breuer --- image/envoy-config.yaml | 3 ++- internal/helper/yawollet.go | 37 +++++++++---------------------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/image/envoy-config.yaml b/image/envoy-config.yaml index c52b11bf..3f59de00 100644 --- a/image/envoy-config.yaml +++ b/image/envoy-config.yaml @@ -2,6 +2,7 @@ # https://www.envoyproxy.io/docs/envoy/latest/start/quick-start/configuration-dynamic-control-plane admin: + ignore_global_conn_limit: true address: socket_address: address: 127.0.0.1 @@ -51,4 +52,4 @@ layered_runtime: - name: static_layer_0 static_layer: overload: - global_downstream_max_connections: 90000 + global_downstream_max_connections: 60000 diff --git a/internal/helper/yawollet.go b/internal/helper/yawollet.go index 35a077f0..3b93c923 100644 --- a/internal/helper/yawollet.go +++ b/internal/helper/yawollet.go @@ -36,7 +36,6 @@ import ( envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoylistener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoyrbacconfig "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" - envoyconnection "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/connection_limit/v3" envoyrbac "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/rbac/v3" envoytcp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" envoyudp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3" @@ -60,8 +59,7 @@ type LoadbalancerMetric string // TODO: replace with constants from envoywellknown when available const ( - FilterConnectionLimit string = "extensions.filters.network.connection_limit" - FilterUDPProxy string = "envoy.filters.udp_listener.udp_proxy" + FilterUDPProxy string = "envoy.filters.udp_listener.udp_proxy" ) // Condition status const @@ -264,14 +262,14 @@ func createEnvoyCluster(lb *yawolv1beta1.LoadBalancer) []envoytypes.Resource { DnsLookupFamily: envoycluster.Cluster_AUTO, HealthChecks: healthChecks, TransportSocket: transportSocket, - PerConnectionBufferLimitBytes: &wrappers.UInt32Value{Value: 32768}, // 32Kib + PerConnectionBufferLimitBytes: wrapperspb.UInt32(32 * 1024), CircuitBreakers: &envoycluster.CircuitBreakers{ Thresholds: []*envoycluster.CircuitBreakers_Thresholds{ { Priority: envoycore.RoutingPriority_DEFAULT, - MaxConnections: &wrappers.UInt32Value{Value: 10000 * uint32(hostmetrics.GetCPUNum())}, - MaxRequests: &wrappers.UInt32Value{Value: 8000 * uint32(hostmetrics.GetCPUNum())}, - MaxPendingRequests: &wrappers.UInt32Value{Value: 2000 * uint32(hostmetrics.GetCPUNum())}, + MaxConnections: wrapperspb.UInt32(uint32(10000 * hostmetrics.GetCPUNum())), + MaxRequests: wrapperspb.UInt32(uint32(8000 * hostmetrics.GetCPUNum())), + MaxPendingRequests: wrapperspb.UInt32(uint32(2000 * hostmetrics.GetCPUNum())), }, }, }, @@ -338,23 +336,6 @@ func createEnvoyTCPListener( } } - connectionLimit, err := anypb.New(&envoyconnection.ConnectionLimit{ - StatPrefix: "envoyconnectionlimit", - // this is limited by the number of available ports on the machine - MaxConnections: wrapperspb.UInt64(60000), - }) - - if err == nil { - filters = append(filters, &envoylistener.Filter{ - Name: FilterConnectionLimit, - ConfigType: &envoylistener.Filter_TypedConfig{ - TypedConfig: connectionLimit, - }, - }) - } else { - _ = kubernetes.SendErrorAsEvent(r, ErrCouldNotMarshalConnectionLimit, lb) - } - return &envoylistener.Listener{ Name: fmt.Sprintf("%v-%v", port.Protocol, port.Port), Address: &envoycore.Address{ @@ -379,7 +360,7 @@ func createEnvoyTCPListener( }}, Freebind: wrapperspb.Bool(true), EnableReusePort: wrapperspb.Bool(true), - PerConnectionBufferLimitBytes: wrapperspb.UInt32(32768), // 32 Kib + PerConnectionBufferLimitBytes: wrapperspb.UInt32(32 * 1024), } } @@ -444,9 +425,9 @@ func createEnvoyUDPListener( }, }, }, - Freebind: &wrappers.BoolValue{Value: true}, - EnableReusePort: &wrappers.BoolValue{Value: true}, - PerConnectionBufferLimitBytes: &wrappers.UInt32Value{Value: 32768}, // 32 Kib + Freebind: wrapperspb.Bool(true), + EnableReusePort: wrapperspb.Bool(true), + PerConnectionBufferLimitBytes: wrapperspb.UInt32(32 * 1024), } } From 38fa2c8b6c11949ebe1484ae6a5acf51bcbcb98e Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Wed, 1 Feb 2023 14:27:04 +0100 Subject: [PATCH 5/5] remove unneeded error Signed-off-by: Felix Breuer --- internal/helper/errors.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/helper/errors.go b/internal/helper/errors.go index 7438b4ce..b98f5fb5 100644 --- a/internal/helper/errors.go +++ b/internal/helper/errors.go @@ -63,7 +63,6 @@ var ( ErrCANotFoundInSecret = errors.New("ca in secret not found") ErrNoNetworkID = errors.New("cant get networkID for loadbalancer") ErrCouldNotParseSourceRange = errors.New("could not parse LoadBalancerSourceRange") - ErrCouldNotMarshalConnectionLimit = errors.New("could not marshal ConnectionLimit") ErrListingChildLBMs = errors.New("unable to list child loadbalancerMachines") ErrUnsupportedProtocol = errors.New("unsupported protocol used (TCP and UDP is supported)") ErrProjectIsImmutable = errors.New("project id is immutable, cant be changed after initial creation")