use global loadbalancers

pkg/apis/kops/cluster.go

@@ -505,8 +505,10 @@ const (
 type LoadBalancerClass string
 
 const (
-	LoadBalancerClassClassic LoadBalancerClass = "Classic"
-	LoadBalancerClassNetwork LoadBalancerClass = "Network"
+	LoadBalancerClassClassic  LoadBalancerClass = "Classic"
+	LoadBalancerClassNetwork  LoadBalancerClass = "Network"
+	LoadBalancerClassRegional LoadBalancerClass = "Regional"
+	LoadBalancerClassGlobal   LoadBalancerClass = "Global"
 )
 
 type AccessLogSpec struct {

pkg/apis/kops/validation/validation.go

@@ -224,9 +224,6 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		lbSpec := spec.API.LoadBalancer
 		lbPath := fieldPath.Child("api", "loadBalancer")
 		if spec.GetCloudProvider() != kops.CloudProviderAWS {
-			if lbSpec.Class != "" {
-				allErrs = append(allErrs, field.Forbidden(lbPath.Child("class"), "class is only supported on AWS"))
-			}
 			if lbSpec.IdleTimeoutSeconds != nil {
 				allErrs = append(allErrs, field.Forbidden(lbPath.Child("idleTimeoutSeconds"), "idleTimeoutSeconds is only supported on AWS"))
 			}

pkg/model/context.go

@@ -301,6 +301,11 @@ func (b *KopsModelContext) UseNetworkLoadBalancer() bool {
 	return b.Cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork
 }
 
+// UseRegionalLoadBalancer checks if we are using Regional LoadBalancer
+func (b *KopsModelContext) UseRegionalLoadBalancer() bool {
+	return b.Cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassRegional
+}
+
 // UseSSHKey returns true if SSHKeyName from the cluster spec is set to a nonempty string
 // or there is an SSH public key provisioned in the key store.
 func (b *KopsModelContext) UseSSHKey() bool {

pkg/model/gcemodel/api_loadbalancer.go

@@ -39,34 +39,60 @@ var _ fi.CloudupModelBuilder = &APILoadBalancerBuilder{}
 // createPublicLB validates the existence of a target pool with the given name,
 // and creates an IP address and forwarding rule pointing to that target pool.
 func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext) error {
-	healthCheck := &gcetasks.HTTPHealthcheck{
-		Name:        s(b.NameForHealthcheck("api")),
-		Port:        i64(wellknownports.KubeAPIServerHealthCheck),
-		RequestPath: s("/healthz"),
-		Lifecycle:   b.Lifecycle,
+	// healthCheck := &gcetasks.HTTPHealthcheck{
+	// 	Name:        s(b.NameForHealthcheck("api")),
+	// 	Port:        i64(wellknownports.KubeAPIServerHealthCheck),
+	// 	RequestPath: s("/healthz"),
+	// 	Lifecycle:   b.Lifecycle,
+	// }
+	// c.AddTask(healthCheck)
+
+	hc := &gcetasks.HealthCheck{
+		Name:      s(b.NameForHealthCheck("api")),
+		Port:      wellknownports.KubeAPIServer,
+		Lifecycle: b.Lifecycle,
 	}
-	c.AddTask(healthCheck)
+	c.AddTask(hc)
 
-	// TODO: point target pool to instance group managers, as done in internal LB.
-	targetPool := &gcetasks.TargetPool{
-		Name:        s(b.NameForTargetPool("api")),
-		HealthCheck: healthCheck,
-		Lifecycle:   b.Lifecycle,
+	region := ""
+	if b.UseRegionalLoadBalancer() {
+		for _, subnet := range b.Cluster.Spec.Networking.Subnets {
+			if subnet.Region != "" {
+				region = subnet.Region
+			}
+		}
+	}
+	var igms []*gcetasks.InstanceGroupManager
+	for _, ig := range b.InstanceGroups {
+		if ig.Spec.Role != kops.InstanceGroupRoleControlPlane {
+			continue
+		}
+		if len(ig.Spec.Zones) > 1 {
+			return fmt.Errorf("instance group %q has %d zones, which is not yet supported for GCP", ig.GetName(), len(ig.Spec.Zones))
+		}
+		if len(ig.Spec.Zones) == 0 {
+			return fmt.Errorf("instance group %q must specify exactly one zone", ig.GetName())
+		}
+		zone := ig.Spec.Zones[0]
+		igms = append(igms, &gcetasks.InstanceGroupManager{Name: s(gce.NameForInstanceGroupManager(b.Cluster.ObjectMeta.Name, ig.ObjectMeta.Name, zone)), Zone: s(zone)})
 	}
-	c.AddTask(targetPool)
 
-	poolHealthCheck := &gcetasks.PoolHealthCheck{
-		Name:        s(b.NameForPoolHealthcheck("api")),
-		Healthcheck: healthCheck,
-		Pool:        targetPool,
-		Lifecycle:   b.Lifecycle,
+	bs := &gcetasks.BackendService{
+		Name:                  s(b.NameForBackendService("api")),
+		Protocol:              s("TCP"),
+		HealthChecks:          []*gcetasks.HealthCheck{hc},
+		Lifecycle:             b.Lifecycle,
+		LoadBalancingScheme:   s("EXTERNAL_MANAGED"),
+		Region:                region,
+		InstanceGroupManagers: igms,
 	}
-	c.AddTask(poolHealthCheck)
+	c.AddTask(bs)
 
 	ipAddress := &gcetasks.Address{
 		Name:         s(b.NameForIPAddress("api")),
 		ForAPIServer: true,
 		Lifecycle:    b.Lifecycle,
+		Region:       region,
 	}
 	c.AddTask(ipAddress)
 
@@ -75,11 +101,12 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
 	c.AddTask(&gcetasks.ForwardingRule{
 		Name:                s(b.NameForForwardingRule("api")),
 		Lifecycle:           b.Lifecycle,
-		PortRange:           s(strconv.Itoa(wellknownports.KubeAPIServer) + "-" + strconv.Itoa(wellknownports.KubeAPIServer)),
-		TargetPool:          targetPool,
+		Ports:               []string{strconv.Itoa(wellknownports.KubeAPIServer)},
+		BackendService:      bs,
 		IPAddress:           ipAddress,
+		Region:              region,
 		IPProtocol:          "TCP",
-		LoadBalancingScheme: s("EXTERNAL"),
+		LoadBalancingScheme: s("EXTERNAL_MANAGED"),
 		Labels: map[string]string{
 			clusterLabel.Key: clusterLabel.Value,
 			"name":           "api",
@@ -89,11 +116,12 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
 		c.AddTask(&gcetasks.ForwardingRule{
 			Name:                s(b.NameForForwardingRule("kops-controller")),
 			Lifecycle:           b.Lifecycle,
-			PortRange:           s(strconv.Itoa(wellknownports.KopsControllerPort) + "-" + strconv.Itoa(wellknownports.KopsControllerPort)),
-			TargetPool:          targetPool,
+			Ports:               []string{strconv.Itoa(wellknownports.KopsControllerPort)},
+			BackendService:      bs,
 			IPAddress:           ipAddress,
 			IPProtocol:          "TCP",
-			LoadBalancingScheme: s("EXTERNAL"),
+			Region:              region,
+			LoadBalancingScheme: s("EXTERNAL_MANAGED"),
 			Labels: map[string]string{
 				clusterLabel.Key: clusterLabel.Value,
 				"name":           "kops-controller",
@@ -151,6 +179,14 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
 func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderContext) error {
 	clusterLabel := gce.LabelForCluster(b.ClusterName())
 
+	// internal Loadbalancers are always regional
+	region := ""
+	for _, subnet := range b.Cluster.Spec.Networking.Subnets {
+		if subnet.Region != "" {
+			region = subnet.Region
+		}
+	}
+
 	hc := &gcetasks.HealthCheck{
 		Name:      s(b.NameForHealthCheck("api")),
 		Port:      wellknownports.KubeAPIServer,
@@ -177,6 +213,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 		HealthChecks:          []*gcetasks.HealthCheck{hc},
 		Lifecycle:             b.Lifecycle,
 		LoadBalancingScheme:   s("INTERNAL"),
+		Region:                region,
 		InstanceGroupManagers: igms,
 	}
 	c.AddTask(bs)

pkg/model/gcemodel/autoscalinggroup.go

@@ -316,10 +316,9 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) e
 				if b.UseLoadBalancerForAPI() {
 					lbSpec := b.Cluster.Spec.API.LoadBalancer
 					if lbSpec != nil {
-						switch lbSpec.Type {
-						case kops.LoadBalancerTypePublic:
+						if !b.UseRegionalLoadBalancer() && lbSpec.Type == kops.LoadBalancerTypePublic {
 							t.TargetPools = append(t.TargetPools, b.LinkToTargetPool("api"))
-						case kops.LoadBalancerTypeInternal:
+						} else {
 							klog.Warningf("Not hooking the instance group manager up to anything.")
 						}
 					}

pkg/resources/gce/gce.go

@@ -509,7 +509,11 @@ func deleteForwardingRule(cloud fi.Cloud, r *resources.Resource) error {
 		return err
 	}
 
-	op, err := c.Compute().ForwardingRules().Delete(u.Project, u.Region, u.Name)
+	region := ""
+	if !u.Global {
+		region = u.Region
+	}
+	op, err := c.Compute().ForwardingRules().Delete(u.Project, region, u.Name)
 	if err != nil {
 		if gce.IsNotFound(err) {
 			klog.Infof("ForwardingRule not found, assuming deleted: %q", t.SelfLink)
@@ -826,9 +830,14 @@ func (d *clusterDiscoveryGCE) listAddresses() ([]*resources.Resource, error) {
 
 	addrs, err := c.Compute().Addresses().List(ctx, c.Project(), c.Region())
 	if err != nil {
-		return nil, fmt.Errorf("error listing Addresses: %v", err)
+		return nil, fmt.Errorf("error listing regional Addresses: %v", err)
+	}
+	globalAddrs, err := c.Compute().Addresses().List(ctx, c.Project(), "")
+	if err != nil {
+		return nil, fmt.Errorf("error listing global Addresses: %v", err)
 	}
 
+	addrs = append(addrs, globalAddrs...)
 	for _, a := range addrs {
 		if !d.matchesClusterName(a.Name) {
 			klog.V(8).Infof("Skipping Address with name %q", a.Name)
@@ -859,8 +868,12 @@ func deleteAddress(cloud fi.Cloud, r *resources.Resource) error {
 	if err != nil {
 		return err
 	}
+	region := ""
+	if !u.Global {
+		region = u.Region
+	}
 
-	op, err := c.Compute().Addresses().Delete(u.Project, u.Region, u.Name)
+	op, err := c.Compute().Addresses().Delete(u.Project, region, u.Name)
 	if err != nil {
 		if gce.IsNotFound(err) {
 			klog.Infof("Address not found, assuming deleted: %q", t.SelfLink)
@@ -1077,14 +1090,22 @@ func containsOnlyListedIGMs(svc *compute.BackendService, igms []*resources.Resou
 
 func (d *clusterDiscoveryGCE) listBackendServices() ([]*resources.Resource, error) {
 	c := d.gceCloud
-
-	svcs, err := c.Compute().RegionBackendServices().List(context.Background(), c.Project(), c.Region())
+	// list global backendservices first
+	svcs, err := c.Compute().BackendServices().List(context.Background(), c.Project(), "")
 	if err != nil {
 		if gce.IsNotFound(err) {
-			klog.Infof("backend services not found, assuming none exist in project: %q region: %q", c.Project(), c.Region())
-			return nil, nil
+			// list regional backendservices as well
+			regionalsvcs, err := c.Compute().BackendServices().List(context.Background(), c.Project(), c.Region())
+			if err != nil {
+				if gce.IsNotFound(err) {
+					klog.Infof("backend services not found, assuming none exist in project: %q region: %q", c.Project(), c.Region())
+					return nil, nil
+				}
+			}
+			svcs = append(svcs, regionalsvcs...)
+		} else {
+			return nil, fmt.Errorf("failed to list backend services: %w", err)
 		}
-		return nil, fmt.Errorf("Failed to list backend services: %w", err)
 	}
 	// TODO: cache, for efficiency, if needed.
 	// Find all relevant backend services by finding all the cluster's IGMs, and then
@@ -1102,7 +1123,15 @@ func (d *clusterDiscoveryGCE) listBackendServices() ([]*resources.Resource, erro
 				ID:   svc.Name,
 				Type: typeBackendService,
 				Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
-					op, err := c.Compute().RegionBackendServices().Delete(c.Project(), c.Region(), svc.Name)
+					u, err := gce.ParseGoogleCloudURL(svc.SelfLink)
+					if err != nil {
+						return err
+					}
+					region := ""
+					if !u.Global {
+						region = u.Region
+					}
+					op, err := c.Compute().BackendServices().Delete(c.Project(), region, svc.Name)
 					if err != nil {
 						return err
 					}