Skip to content

Commit

Permalink
refactor regional/global GCE API methods
Browse files Browse the repository at this point in the history
  • Loading branch information
@upodroid
upodroid committed Feb 19, 2024
1 parent 3c3188e commit 4d4e6e8
Show file tree
Hide file tree
Showing 15 changed files with 387 additions and 166 deletions.
4 changes: 2 additions & 2 deletions cloudmock/gce/mockcompute/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,11 @@ func (c *MockClient) HTTPHealthChecks() gce.HttpHealthChecksClient {
return c.httpHealthChecksClient
}

func (c *MockClient) RegionHealthChecks() gce.RegionHealthChecksClient {
func (c *MockClient) HealthChecks() gce.HealthChecksClient {
return c.healthCheckClient
}

func (c *MockClient) RegionBackendServices() gce.RegionBackendServiceClient {
func (c *MockClient) BackendServices() gce.BackendServiceClient {
return c.backendServiceClient
}

Expand Down
2 changes: 1 addition & 1 deletion cloudmock/gce/mockcompute/backend_services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type backendServiceClient struct {
sync.Mutex
}

var _ gce.RegionBackendServiceClient = &backendServiceClient{}
var _ gce.BackendServiceClient = &backendServiceClient{}

func newBackendServiceClient() *backendServiceClient {
return &backendServiceClient{
Expand Down
2 changes: 1 addition & 1 deletion cloudmock/gce/mockcompute/health_checks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type healthCheckClient struct {
sync.Mutex
}

var _ gce.RegionHealthChecksClient = &healthCheckClient{}
var _ gce.HealthChecksClient = &healthCheckClient{}

func newHealthCheckClient() *healthCheckClient {
return &healthCheckClient{
Expand Down
29 changes: 26 additions & 3 deletions pkg/model/gcemodel/api_loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ var _ fi.CloudupModelBuilder = &APILoadBalancerBuilder{}
// createPublicLB validates the existence of a target pool with the given name,
// and creates an IP address and forwarding rule pointing to that target pool.
func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext) error {
region := ""
for _, subnet := range b.Cluster.Spec.Networking.Subnets {
if subnet.Region != "" {
region = subnet.Region
}
}
healthCheck := &gcetasks.HTTPHealthcheck{
Name: s(b.NameForHealthcheck("api")),
Port: i64(wellknownports.KubeAPIServerHealthCheck),
Expand Down Expand Up @@ -68,6 +74,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
Name: s(b.NameForIPAddress("api")),

Lifecycle: b.Lifecycle,
Region: region,
WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
}
c.AddTask(ipAddress)
Expand All @@ -80,6 +87,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
PortRange: s(strconv.Itoa(wellknownports.KubeAPIServer) + "-" + strconv.Itoa(wellknownports.KubeAPIServer)),
TargetPool: targetPool,
IPAddress: ipAddress,
Region: region,
IPProtocol: "TCP",
LoadBalancingScheme: s("EXTERNAL"),
Labels: map[string]string{
Expand All @@ -97,6 +105,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext
TargetPool: targetPool,
IPAddress: ipAddress,
IPProtocol: "TCP",
Region: region,
LoadBalancingScheme: s("EXTERNAL"),
Labels: map[string]string{
clusterLabel.Key: clusterLabel.Value,
Expand Down Expand Up @@ -155,10 +164,20 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderContext) error {
clusterLabel := gce.LabelForCluster(b.ClusterName())

// internal Loadbalancers are always regional
region := ""
for _, subnet := range b.Cluster.Spec.Networking.Subnets {
if subnet.Region != "" {
region = subnet.Region
}
}

hc := &gcetasks.HealthCheck{
Name: s(b.NameForHealthCheck("api")),
Port: wellknownports.KubeAPIServer,
Lifecycle: b.Lifecycle,
Name: s(b.NameForHealthCheck("api")),
Port: i64(wellknownports.KubeAPIServer),
RequestPath: s("/healthz"),
Region: region,
Lifecycle: b.Lifecycle,
}
c.AddTask(hc)
var igms []*gcetasks.InstanceGroupManager
Expand All @@ -181,6 +200,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
HealthChecks: []*gcetasks.HealthCheck{hc},
Lifecycle: b.Lifecycle,
LoadBalancingScheme: s("INTERNAL"),
Region: region,
InstanceGroupManagers: igms,
}
c.AddTask(bs)
Expand Down Expand Up @@ -209,6 +229,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
Subnetwork: subnet,

WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
Region: region,
Lifecycle: b.Lifecycle,
}
c.AddTask(ipAddress)
Expand All @@ -222,6 +243,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
IPProtocol: "TCP",
LoadBalancingScheme: s("INTERNAL"),
Network: network,
Region: region,
Subnetwork: subnet,
Labels: map[string]string{
clusterLabel.Key: clusterLabel.Value,
Expand All @@ -240,6 +262,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
IPProtocol: "TCP",
LoadBalancingScheme: s("INTERNAL"),
Network: network,
Region: region,
Subnetwork: subnet,
Labels: map[string]string{
clusterLabel.Key: clusterLabel.Value,
Expand Down
58 changes: 49 additions & 9 deletions pkg/resources/gce/gce.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,11 @@ func deleteForwardingRule(cloud fi.Cloud, r *resources.Resource) error {
return err
}

op, err := c.Compute().ForwardingRules().Delete(ctx, u.Project, u.Region, u.Name)
region := ""
if !u.Global {
region = u.Region
}
op, err := c.Compute().ForwardingRules().Delete(ctx, u.Project, region, u.Name)
if err != nil {
if gce.IsNotFound(err) {
klog.Infof("ForwardingRule not found, assuming deleted: %q", t.SelfLink)
Expand Down Expand Up @@ -828,9 +832,14 @@ func (d *clusterDiscoveryGCE) listAddresses() ([]*resources.Resource, error) {

addrs, err := c.Compute().Addresses().List(ctx, c.Project(), c.Region())
if err != nil {
return nil, fmt.Errorf("error listing Addresses: %v", err)
return nil, fmt.Errorf("error listing regional Addresses: %v", err)
}
globalAddrs, err := c.Compute().Addresses().List(ctx, c.Project(), "")
if err != nil {
return nil, fmt.Errorf("error listing global Addresses: %v", err)
}

addrs = append(addrs, globalAddrs...)
for _, a := range addrs {
if !d.matchesClusterName(a.Name) {
klog.V(8).Infof("Skipping Address with name %q", a.Name)
Expand Down Expand Up @@ -861,8 +870,12 @@ func deleteAddress(cloud fi.Cloud, r *resources.Resource) error {
if err != nil {
return err
}
region := ""
if !u.Global {
region = u.Region
}

op, err := c.Compute().Addresses().Delete(u.Project, u.Region, u.Name)
op, err := c.Compute().Addresses().Delete(u.Project, region, u.Name)
if err != nil {
if gce.IsNotFound(err) {
klog.Infof("Address not found, assuming deleted: %q", t.SelfLink)
Expand Down Expand Up @@ -1079,15 +1092,26 @@ func containsOnlyListedIGMs(svc *compute.BackendService, igms []*resources.Resou

func (d *clusterDiscoveryGCE) listBackendServices() ([]*resources.Resource, error) {
c := d.gceCloud
// list global backendservices first
svcs, err := c.Compute().BackendServices().List(context.Background(), c.Project(), "")
if err != nil {
if gce.IsNotFound(err) {
klog.Infof("backend services not found, assuming none exist in project: %q region: %q", c.Project(), c.Region())
return nil, nil
}
return nil, fmt.Errorf("failed to list global backend services: %w", err)
}

svcs, err := c.Compute().RegionBackendServices().List(context.Background(), c.Project(), c.Region())
// list regional backendservices as well
regionalsvcs, err := c.Compute().BackendServices().List(context.Background(), c.Project(), c.Region())
if err != nil {
if gce.IsNotFound(err) {
klog.Infof("backend services not found, assuming none exist in project: %q region: %q", c.Project(), c.Region())
return nil, nil
}
return nil, fmt.Errorf("Failed to list backend services: %w", err)
return nil, fmt.Errorf("failed to list regional backend services: %w", err)
}
svcs = append(svcs, regionalsvcs...)
// TODO: cache, for efficiency, if needed.
// Find all relevant backend services by finding all the cluster's IGMs, and then
// listing all backend services in the project / region, then selecting
Expand All @@ -1104,7 +1128,15 @@ func (d *clusterDiscoveryGCE) listBackendServices() ([]*resources.Resource, erro
ID: svc.Name,
Type: typeBackendService,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
op, err := c.Compute().RegionBackendServices().Delete(c.Project(), c.Region(), svc.Name)
u, err := gce.ParseGoogleCloudURL(svc.SelfLink)
if err != nil {
return err
}
region := ""
if !u.Global {
region = u.Region
}
op, err := c.Compute().BackendServices().Delete(c.Project(), region, svc.Name)
if err != nil {
return err
}
Expand Down Expand Up @@ -1139,12 +1171,20 @@ func (d *clusterDiscoveryGCE) listHealthchecks() ([]*resources.Resource, error)
}
var hcResources []*resources.Resource
for hc := range hcs {
u, err := gce.ParseGoogleCloudURL(hc)
if err != nil {
return nil, err
}
region := ""
if !u.Global {
region = u.Region
}
hcResources = append(hcResources, &resources.Resource{
Name: gce.LastComponent(hc),
ID: gce.LastComponent(hc),
Name: u.Name,
ID: u.Name,
Type: typeHealthcheck,
Deleter: func(cloud fi.Cloud, r *resources.Resource) error {
op, err := c.Compute().RegionHealthChecks().Delete(c.Project(), c.Region(), gce.LastComponent(hc))
op, err := c.Compute().HealthChecks().Delete(u.Project, region, u.Name)
if err != nil {
return err
}
Expand Down
44 changes: 25 additions & 19 deletions tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,19 +174,10 @@ resource "google_compute_address" "api-us-test1-minimal-gce-example-com" {
address_type = "INTERNAL"
name = "api-us-test1-minimal-gce-example-com"
purpose = "SHARED_LOADBALANCER_VIP"
region = "us-test1"
subnetwork = google_compute_subnetwork.us-test1-minimal-gce-example-com.name
}

resource "google_compute_backend_service" "api-minimal-gce-example-com" {
backend {
group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-example-com.instance_group
}
health_checks = [google_compute_health_check.api-minimal-gce-example-com.id]
load_balancing_scheme = "INTERNAL_SELF_MANAGED"
name = "api-minimal-gce-example-com"
protocol = "TCP"
}

resource "google_compute_disk" "a-etcd-events-minimal-gce-example-com" {
labels = {
"k8s-io-cluster-name" = "minimal-gce-example-com"
Expand Down Expand Up @@ -449,7 +440,7 @@ resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-example-com
}

resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-example-com" {
backend_service = google_compute_backend_service.api-minimal-gce-example-com.id
backend_service = google_compute_region_backend_service.api-minimal-gce-example-com.id
ip_address = google_compute_address.api-us-test1-minimal-gce-example-com.address
ip_protocol = "TCP"
labels = {
Expand All @@ -460,11 +451,12 @@ resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-example-com"
name = "api-us-test1-minimal-gce-example-com"
network = google_compute_network.minimal-gce-example-com.name
ports = ["443"]
region = "us-test1"
subnetwork = google_compute_subnetwork.us-test1-minimal-gce-example-com.name
}

resource "google_compute_forwarding_rule" "kops-controller-us-test1-minimal-gce-example-com" {
backend_service = google_compute_backend_service.api-minimal-gce-example-com.id
backend_service = google_compute_region_backend_service.api-minimal-gce-example-com.id
ip_address = google_compute_address.api-us-test1-minimal-gce-example-com.address
ip_protocol = "TCP"
labels = {
Expand All @@ -475,16 +467,10 @@ resource "google_compute_forwarding_rule" "kops-controller-us-test1-minimal-gce-
name = "kops-controller-us-test1-minimal-gce-example-com"
network = google_compute_network.minimal-gce-example-com.name
ports = ["3988"]
region = "us-test1"
subnetwork = google_compute_subnetwork.us-test1-minimal-gce-example-com.name
}

resource "google_compute_health_check" "api-minimal-gce-example-com" {
name = "api-minimal-gce-example-com"
tcp_health_check {
port = 443
}
}

resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-example-com" {
base_instance_name = "master-us-test1-a"
list_managed_instances_results = "PAGINATED"
Expand Down Expand Up @@ -612,6 +598,26 @@ resource "google_compute_network" "minimal-gce-example-com" {
name = "minimal-gce-example-com"
}

resource "google_compute_region_backend_service" "api-minimal-gce-example-com" {
backend {
group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-example-com.instance_group
}
health_checks = [google_compute_region_health_check.api-minimal-gce-example-com.id]
load_balancing_scheme = "INTERNAL_SELF_MANAGED"
name = "api-minimal-gce-example-com"
protocol = "TCP"
region = "us-test1"
}

resource "google_compute_region_health_check" "api-minimal-gce-example-com" {
http_health_check {
port = 443
request_path = "/healthz"
}
name = "api-minimal-gce-example-com"
region = "us-test1"
}

resource "google_compute_router" "nat-minimal-gce-example-com" {
name = "nat-minimal-gce-example-com"
network = google_compute_network.minimal-gce-example-com.name
Expand Down
Loading

0 comments on commit 4d4e6e8

Please sign in to comment.