Merge pull request #1825 from panslava/rbs-ipv6

Add Dual-Stack support to L4 NetLB

cmd/glbc/main.go

@@ -188,19 +188,20 @@ func main() {
 	cloud := app.NewGCEClient()
 	defaultBackendServicePort := app.DefaultBackendServicePort(kubeClient)
 	ctxConfig := ingctx.ControllerContextConfig{
-		Namespace:             flags.F.WatchNamespace,
-		ResyncPeriod:          flags.F.ResyncPeriod,
-		NumL4Workers:          flags.F.NumL4Workers,
-		NumL4NetLBWorkers:     flags.F.NumL4NetLBWorkers,
-		DefaultBackendSvcPort: defaultBackendServicePort,
-		HealthCheckPath:       flags.F.HealthCheckPath,
-		FrontendConfigEnabled: flags.F.EnableFrontendConfig,
-		EnableASMConfigMap:    flags.F.EnableASMConfigMapBasedConfig,
-		ASMConfigMapNamespace: flags.F.ASMConfigMapBasedConfigNamespace,
-		ASMConfigMapName:      flags.F.ASMConfigMapBasedConfigCMName,
-		EndpointSlicesEnabled: flags.F.EnableEndpointSlices,
-		MaxIGSize:             flags.F.MaxIGSize,
-		EnableL4ILBDualStack:  flags.F.EnableL4ILBDualStack,
+		Namespace:              flags.F.WatchNamespace,
+		ResyncPeriod:           flags.F.ResyncPeriod,
+		NumL4Workers:           flags.F.NumL4Workers,
+		NumL4NetLBWorkers:      flags.F.NumL4NetLBWorkers,
+		DefaultBackendSvcPort:  defaultBackendServicePort,
+		HealthCheckPath:        flags.F.HealthCheckPath,
+		FrontendConfigEnabled:  flags.F.EnableFrontendConfig,
+		EnableASMConfigMap:     flags.F.EnableASMConfigMapBasedConfig,
+		ASMConfigMapNamespace:  flags.F.ASMConfigMapBasedConfigNamespace,
+		ASMConfigMapName:       flags.F.ASMConfigMapBasedConfigCMName,
+		EndpointSlicesEnabled:  flags.F.EnableEndpointSlices,
+		MaxIGSize:              flags.F.MaxIGSize,
+		EnableL4ILBDualStack:   flags.F.EnableL4ILBDualStack,
+		EnableL4NetLBDualStack: flags.F.EnableL4NetLBDualStack,
 	}
 	ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, svcNegClient, ingParamsClient, svcAttachmentClient, cloud, namer, kubeSystemUID, ctxConfig)
 	go app.RunHTTPServer(ctx.HealthCheck)

pkg/context/context.go

@@ -118,15 +118,16 @@ type ControllerContextConfig struct {
 	NumL4Workers      int
 	NumL4NetLBWorkers int
 	// DefaultBackendSvcPortID is the ServicePort for the system default backend.
-	DefaultBackendSvcPort utils.ServicePort
-	HealthCheckPath       string
-	FrontendConfigEnabled bool
-	EnableASMConfigMap    bool
-	ASMConfigMapNamespace string
-	ASMConfigMapName      string
-	EndpointSlicesEnabled bool
-	MaxIGSize             int
-	EnableL4ILBDualStack  bool
+	DefaultBackendSvcPort  utils.ServicePort
+	HealthCheckPath        string
+	FrontendConfigEnabled  bool
+	EnableASMConfigMap     bool
+	ASMConfigMapNamespace  string
+	ASMConfigMapName       string
+	EndpointSlicesEnabled  bool
+	MaxIGSize              int
+	EnableL4ILBDualStack   bool
+	EnableL4NetLBDualStack bool
 }
 
 // NewControllerContext returns a new shared set of informers.

pkg/flags/flags.go

@@ -111,6 +111,7 @@ var (
 		EnableEndpointSlices           bool
 		EnablePinhole                  bool
 		EnableL4ILBDualStack           bool
+		EnableL4NetLBDualStack         bool
 		EnableMultipleIGs              bool
 		MaxIGSize                      int
 	}{
@@ -255,6 +256,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
 	flag.BoolVar(&F.EnableEndpointSlices, "enable-endpoint-slices", false, "Enable using Endpoint Slices API instead of Endpoints API")
 	flag.BoolVar(&F.EnablePinhole, "enable-pinhole", false, "Enable Pinhole firewall feature")
 	flag.BoolVar(&F.EnableL4ILBDualStack, "enable-l4ilb-dual-stack", false, "Enable Dual-Stack handling for L4 Internal Load Balancers")
+	flag.BoolVar(&F.EnableL4NetLBDualStack, "enable-l4netlb-dual-stack", false, "Enable Dual-Stack handling for L4 External Load Balancers")
 	flag.BoolVar(&F.EnableMultipleIGs, "enable-multiple-igs", false, "Enable using multiple unmanaged instance groups")
 	flag.IntVar(&F.MaxIGSize, "max-ig-size", 1000, "Max number of instances in Instance Group")
 	flag.DurationVar(&F.MetricsExportInterval, "metrics-export-interval", 10*time.Minute, `Period for calculating and exporting metrics related to state of managed objects.`)

pkg/healthchecksl4/healthchecksl4.go

@@ -47,6 +47,7 @@ const (
 	gceSharedHcUnhealthyThreshold = int64(3) // 3  * 8 = 24 seconds before the LB will steer traffic away
 	gceLocalHcUnhealthyThreshold  = int64(2) // 2  * 3 = 6 seconds before the LB will steer traffic away
 	L4ILBIPv6HCRange              = "2600:2d00:1:b029::/64"
+	L4NetLBIPv6HCRange            = "2600:1901:8001::/48"
 )
 
 var (
@@ -150,7 +151,7 @@ func (l4hc *l4HealthChecks) EnsureHealthCheckWithDualStackFirewalls(svc *corev1.
 
 	if needsIPv6 {
 		klog.V(3).Infof("Ensuring IPv6 firewall rule for health check %s for service %s", hcName, namespacedName.String())
-		l4hc.ensureIPv6Firewall(svc, namer, hcPort, sharedHC, nodeNames, hcResult)
+		l4hc.ensureIPv6Firewall(svc, namer, hcPort, sharedHC, nodeNames, l4Type, hcResult)
 	}
 
 	return hcResult
@@ -207,9 +208,10 @@ func (l4hc *l4HealthChecks) ensureHealthCheck(hcName string, svcName types.Names
 // L4 ILB and L4 NetLB Services with ExternalTrafficPolicy=Cluster use the same firewall
 // rule at global scope.
 func (l4hc *l4HealthChecks) ensureIPv4Firewall(svc *corev1.Service, namer namer.L4ResourcesNamer, hcPort int32, isSharedHC bool, nodeNames []string, hcResult *EnsureHealthCheckResult) {
+	start := time.Now()
+
 	hcFwName := namer.L4HealthCheckFirewall(svc.Namespace, svc.Name, isSharedHC)
 
-	start := time.Now()
 	klog.V(2).Infof("Ensuring IPv4 Firewall for health check %s for service %s/%s, health check port %d, shared health check: %t, len(nodeNames): %d", hcFwName, svc.Namespace, svc.Name, hcPort, isSharedHC, len(nodeNames))
 	defer func() {
 		klog.V(2).Infof("Finished ensuring IPv4 firewall for health check %s for service %s/%s, time taken %v", hcFwName, svc.Namespace, svc.Name, time.Since(start))
@@ -232,7 +234,7 @@ func (l4hc *l4HealthChecks) ensureIPv4Firewall(svc *corev1.Service, namer namer.
 	hcResult.HCFirewallRuleName = hcFwName
 }
 
-func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, namer namer.L4ResourcesNamer, hcPort int32, isSharedHC bool, nodeNames []string, hcResult *EnsureHealthCheckResult) {
+func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, namer namer.L4ResourcesNamer, hcPort int32, isSharedHC bool, nodeNames []string, l4Type utils.L4LBType, hcResult *EnsureHealthCheckResult) {
 	ipv6HCFWName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, isSharedHC)
 
 	start := time.Now()
@@ -243,7 +245,7 @@ func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, namer namer.
 
 	hcFWRParams := firewalls.FirewallParams{
 		PortRanges:   []string{strconv.Itoa(int(hcPort))},
-		SourceRanges: []string{L4ILBIPv6HCRange},
+		SourceRanges: getIPv6HCFirewallSourceRanges(l4Type),
 		Protocol:     string(corev1.ProtocolTCP),
 		Name:         ipv6HCFWName,
 		NodeNames:    nodeNames,
@@ -301,9 +303,10 @@ func (l4hc *l4HealthChecks) deleteHealthCheckWithDualStackFirewalls(svc *corev1.
 }
 
 func (l4hc *l4HealthChecks) deleteHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType) (bool, error) {
+	start := time.Now()
+
 	hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
 
-	start := time.Now()
 	klog.V(3).Infof("Deleting L4 healthcheck %s for service %s/%s, shared: %v, scope: %v", hcName, svc.Namespace, svc.Name, sharedHC, scope)
 	defer func() {
 		klog.V(3).Infof("Finished deleting L4 healthcheck %s for service %s/%s, time taken: %v", hcName, svc.Namespace, svc.Name, time.Since(start))
@@ -336,10 +339,11 @@ func (l4hc *l4HealthChecks) deleteIPv4HealthCheckFirewall(svc *corev1.Service, n
 }
 
 func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, namer namer.L4ResourcesNamer, isSharedHC bool, l4type utils.L4LBType) (string, error) {
+	start := time.Now()
+
 	hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, isSharedHC)
 	ipv6hcFwName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, isSharedHC)
 
-	start := time.Now()
 	klog.V(3).Infof("Deleting IPv6 Firewall %s for health check %s", ipv6hcFwName, hcName)
 	defer func() {
 		klog.V(3).Infof("Finished deleting IPv6 Firewall %s for health check %s, time taken: %v", ipv6hcFwName, hcName, time.Since(start))
@@ -463,3 +467,10 @@ func needToUpdateHealthChecks(hc, newHC *composite.HealthCheck) bool {
 		hc.UnhealthyThreshold < newHC.UnhealthyThreshold ||
 		hc.HealthyThreshold < newHC.HealthyThreshold
 }
+
+func getIPv6HCFirewallSourceRanges(l4Type utils.L4LBType) []string {
+	if l4Type == utils.XLB {
+		return []string{L4NetLBIPv6HCRange}
+	}
+	return []string{L4ILBIPv6HCRange}
+}

pkg/l4lb/l4lbcommon.go

@@ -95,6 +95,16 @@ func deleteL4RBSAnnotations(ctx *context.ControllerContext, svc *v1.Service) err
 	return patch.PatchServiceObjectMetadata(ctx.KubeClient.CoreV1(), svc, *newObjectMeta)
 }
 
+// deleteL4RBSDualStackAnnotations deletes all annotations which could be added by L4 ELB RBS controller
+func deleteL4RBSDualStackAnnotations(ctx *context.ControllerContext, svc *v1.Service) error {
+	newObjectMeta := computeNewAnnotationsIfNeeded(svc, nil, loadbalancers.L4DualStackResourceRBSAnnotationKeys)
+	if newObjectMeta == nil {
+		return nil
+	}
+	klog.V(3).Infof("Deleting all DualStack annotations for L4 ELB RBS service %v/%v", svc.Namespace, svc.Name)
+	return patch.PatchServiceObjectMetadata(ctx.KubeClient.CoreV1(), svc, *newObjectMeta)
+}
+
 func deleteAnnotation(ctx *context.ControllerContext, svc *v1.Service, annotationKey string) error {
 	newObjectMeta := svc.ObjectMeta.DeepCopy()
 	if _, ok := newObjectMeta.Annotations[annotationKey]; !ok {

pkg/l4lb/l4netlbcontroller.go

@@ -19,6 +19,7 @@ package l4lb
 import (
 	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
@@ -61,6 +62,7 @@ type L4NetLBController struct {
 	instancePool    instancegroups.Manager
 	igLinker        *backends.RegionalInstanceGroupLinker
 	forwardingRules ForwardingRulesGetter
+	enableDualStack bool
 }
 
 // NewL4NetLBController creates a controller for l4 external loadbalancer.
@@ -84,6 +86,7 @@ func NewL4NetLBController(
 		instancePool:    ctx.InstancePool,
 		igLinker:        backends.NewRegionalInstanceGroupLinker(ctx.InstancePool, backendPool),
 		forwardingRules: forwardingrules.New(ctx.Cloud, meta.VersionGA, meta.Regional),
+		enableDualStack: ctx.EnableL4NetLBDualStack,
 	}
 	l4netLBc.svcQueue = utils.NewPeriodicTaskQueueWithMultipleWorkers("l4netLB", "services", ctx.NumL4NetLBWorkers, l4netLBc.sync)
 
@@ -213,6 +216,11 @@ func (lc *L4NetLBController) needsUpdate(newSvc, oldSvc *v1.Service) bool {
 			oldSvc.Spec.HealthCheckNodePort, newSvc.Spec.HealthCheckNodePort)
 		return true
 	}
+	if lc.enableDualStack && !reflect.DeepEqual(oldSvc.Spec.IPFamilies, newSvc.Spec.IPFamilies) {
+		recorder.Eventf(newSvc, v1.EventTypeNormal, "IPFamilies", "%v -> %v",
+			oldSvc.Spec.IPFamilies, newSvc.Spec.IPFamilies)
+		return true
+	}
 	return false
 }
 
@@ -417,10 +425,11 @@ func (lc *L4NetLBController) sync(key string) error {
 // Returns an error if processing the service update failed.
 func (lc *L4NetLBController) syncInternal(service *v1.Service) *loadbalancers.L4NetLBSyncResult {
 	l4NetLBParams := &loadbalancers.L4NetLBParams{
-		Service:  service,
-		Cloud:    lc.ctx.Cloud,
-		Namer:    lc.namer,
-		Recorder: lc.ctx.Recorder(service.Namespace),
+		Service:          service,
+		Cloud:            lc.ctx.Cloud,
+		Namer:            lc.namer,
+		Recorder:         lc.ctx.Recorder(service.Namespace),
+		DualStackEnabled: lc.enableDualStack,
 	}
 	l4netlb := loadbalancers.NewL4NetLB(l4NetLBParams)
 	// check again that rbs is enabled.
@@ -473,18 +482,39 @@ func (lc *L4NetLBController) syncInternal(service *v1.Service) *loadbalancers.L4
 		syncResult.Error = err
 		return syncResult
 	}
-	lc.ctx.Recorder(service.Namespace).Eventf(service, v1.EventTypeNormal, "SyncLoadBalancerSuccessful",
-		"Successfully ensured L4 External LoadBalancer resources")
-	if err = updateL4ResourcesAnnotations(lc.ctx, service, syncResult.Annotations); err != nil {
-		lc.ctx.Recorder(service.Namespace).Eventf(service, v1.EventTypeWarning, "SyncExternalLoadBalancerFailed",
-			"Failed to update annotations for load balancer, err: %v", err)
-		syncResult.Error = fmt.Errorf("failed to set resource annotations, err: %w", err)
-		return syncResult
+	if lc.enableDualStack {
+		lc.emitEnsuredDualStackEvent(service)
+
+		if err = updateL4DualStackResourcesAnnotations(lc.ctx, service, syncResult.Annotations); err != nil {
+			lc.ctx.Recorder(service.Namespace).Eventf(service, v1.EventTypeWarning, "SyncExternalLoadBalancerFailed",
+				"Failed to update annotations for load balancer, err: %v", err)
+			syncResult.Error = fmt.Errorf("failed to set resource annotations, err: %w", err)
+			return syncResult
+		}
+	} else {
+		lc.ctx.Recorder(service.Namespace).Eventf(service, v1.EventTypeNormal, "SyncLoadBalancerSuccessful",
+			"Successfully ensured L4 External LoadBalancer resources")
+
+		if err = updateL4ResourcesAnnotations(lc.ctx, service, syncResult.Annotations); err != nil {
+			lc.ctx.Recorder(service.Namespace).Eventf(service, v1.EventTypeWarning, "SyncExternalLoadBalancerFailed",
+				"Failed to update annotations for load balancer, err: %v", err)
+			syncResult.Error = fmt.Errorf("failed to set resource annotations, err: %w", err)
+			return syncResult
+		}
 	}
 	syncResult.SetMetricsForSuccessfulServiceSync()
 	return syncResult
 }
 
+func (lc *L4NetLBController) emitEnsuredDualStackEvent(service *v1.Service) {
+	var ipFamilies []string
+	for _, ipFamily := range service.Spec.IPFamilies {
+		ipFamilies = append(ipFamilies, string(ipFamily))
+	}
+	lc.ctx.Recorder(service.Namespace).Eventf(service, v1.EventTypeNormal, "SyncLoadBalancerSuccessful",
+		"Successfully ensured %v External LoadBalancer resources", strings.Join(ipFamilies, " "))
+}
+
 func (lc *L4NetLBController) ensureBackendLinking(service *v1.Service) error {
 	start := time.Now()
 	klog.V(2).Infof("Linking backend service with instance groups for service %s/%s", service.Namespace, service.Name)
@@ -537,10 +567,11 @@ func (lc *L4NetLBController) garbageCollectRBSNetLB(key string, svc *v1.Service)
 	}()
 
 	l4NetLBParams := &loadbalancers.L4NetLBParams{
-		Service:  svc,
-		Cloud:    lc.ctx.Cloud,
-		Namer:    lc.namer,
-		Recorder: lc.ctx.Recorder(svc.Namespace),
+		Service:          svc,
+		Cloud:            lc.ctx.Cloud,
+		Namer:            lc.namer,
+		Recorder:         lc.ctx.Recorder(svc.Namespace),
+		DualStackEnabled: lc.enableDualStack,
 	}
 	l4netLB := loadbalancers.NewL4NetLB(l4NetLBParams)
 	lc.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer",
@@ -577,11 +608,20 @@ func (lc *L4NetLBController) garbageCollectRBSNetLB(key string, svc *v1.Service)
 	}
 
 	// IMPORTANT: Remove LB annotations from the Service LAST, cause changing service fields are emitted as service update to other controllers
-	if err := deleteL4RBSAnnotations(lc.ctx, svc); err != nil {
-		lc.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeWarning, "DeleteLoadBalancer",
-			"Error removing resource annotations: %v: %v", err)
-		result.Error = fmt.Errorf("Failed to reset resource annotations, err: %w", err)
-		return result
+	if lc.enableDualStack {
+		if err := deleteL4RBSDualStackAnnotations(lc.ctx, svc); err != nil {
+			lc.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeWarning, "DeleteLoadBalancer",
+				"Error removing Dual Stack resource annotations: %v: %v", err)
+			result.Error = fmt.Errorf("failed to reset Dual Stack resource annotations, err: %w", err)
+			return result
+		}
+	} else {
+		if err := deleteL4RBSAnnotations(lc.ctx, svc); err != nil {
+			lc.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeWarning, "DeleteLoadBalancer",
+				"Error removing resource annotations: %v: %v", err)
+			result.Error = fmt.Errorf("failed to reset resource annotations, err: %w", err)
+			return result
+		}
 	}
 
 	lc.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletedLoadBalancer", "Deleted L4 External LoadBalancer")