Remove metadata-proxy from GCE Clusters from 1.29+ onwards

cmd/kops/create_cluster_integration_test.go

@@ -113,6 +113,7 @@ func TestCreateClusterHA(t *testing.T) {
 func TestCreateClusterMinimalGCE(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-gce", "v1alpha2")
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-gce-dns-none", "v1alpha2")
+	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.29-gce", "v1alpha2")
 }
 
 // TestCreateClusterHAGCE runs kops create cluster ha-gce.example.com --cloud gce --zones us-test1-a,us-test1-b,us-test1-c --master-zones us-test1-a,us-test1-b,us-test1-c

docs/releases/1.28-NOTES.md

@@ -12,6 +12,8 @@ This is a document to gather the release notes prior to the release.
 
 ## GCP
 
+* [metadata-proxy](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy) is no longer deployed on GCP clusters for Kubernetes 1.29+.
+
 ## Openstack
 
 # Breaking changes

tests/integration/create_cluster/minimal-1.29-gce/expected-v1alpha2.yaml

@@ -0,0 +1,96 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal.example.com
+spec:
+  api:
+    dns: {}
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudConfig: {}
+  cloudProvider: gce
+  configBase: memfs://tests/minimal.example.com
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: control-plane-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: control-plane-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: events
+  iam:
+    allowContainerRegistry: true
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  - ::/0
+  kubernetesVersion: v1.29.0
+  masterPublicName: api.minimal.example.com
+  networking:
+    cni: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  - ::/0
+  subnets:
+  - cidr: 10.0.16.0/20
+    name: us-test1
+    region: us-test1
+    type: Public
+  topology:
+    dns:
+      type: Public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+  name: control-plane-us-test1-a
+spec:
+  image: ubuntu-os-cloud/ubuntu-2204-jammy-v20230714
+  machineType: e2-medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+  name: nodes-us-test1-a
+spec:
+  image: ubuntu-os-cloud/ubuntu-2204-jammy-v20230714
+  machineType: e2-medium
+  maxSize: 1
+  minSize: 1
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a

tests/integration/create_cluster/minimal-1.29-gce/options.yaml

@@ -0,0 +1,7 @@
+CloudProvider: gce
+ClusterName: minimal.example.com
+KubernetesVersion: v1.29.0
+Networking: cni
+Project: testproject
+Zones:
+  - us-test1-a

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -849,7 +849,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 
 	// The metadata-proxy daemonset conceals node metadata endpoints in GCE.
 	// It will land on nodes labeled cloud.google.com/metadata-proxy-ready=true
-	if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE {
+	if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE && b.Cluster.IsKubernetesLT("1.29") {
 		key := "metadata-proxy.addons.k8s.io"
 
 		{
@@ -863,61 +863,61 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 				Id:       id,
 			})
 		}
+	}
 
-		if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE {
-			{
-				key := "gcp-cloud-controller.addons.k8s.io"
-				useBuiltin := !b.hasExternalAddon(key)
-
-				if !useBuiltin {
-					klog.Infof("Found cloud-controller-manager in addons; won't use builtin")
-
-					// Until we make the manifest extensible, we still need to inject our arguments.
-					// TODO(justinsb): we don't really want to do this, it limits the ability for users to override things.
-					// However, this is behind a feature flag at the moment, and this way we can work towards something better.
-					gkDaemonset := schema.GroupKind{Group: "apps", Kind: "DaemonSet"}
-					for _, addon := range b.ClusterAddons {
-						if addon.GroupVersionKind().GroupKind() == gkDaemonset &&
-							addon.GetName() == "cloud-controller-manager" &&
-							addon.GetNamespace() == "kube-system" {
-
-							klog.Infof("replacing arguments in externally provided cloud-controller-manager")
-
-							fnAny, ok := b.templates.TemplateFunctions["CloudControllerConfigArgv"]
-							if !ok {
-								return nil, nil, fmt.Errorf("unable to find TemplateFunction CloudControllerConfigArgv")
-							}
-							fn, ok := fnAny.(func() ([]string, error))
-							if !ok {
-								return nil, nil, fmt.Errorf("unexpected type for TemplateFunction CloudControllerConfigArgv: %T", fnAny)
-							}
-							args, err := fn()
-							if err != nil {
-								return nil, nil, fmt.Errorf("in TemplateFunction CloudControllerConfigArgv: %w", err)
-							}
-
-							if err := addon.VisitContainers(func(container map[string]interface{}) error {
-								// TODO: Check name?
-								container["args"] = args
-								return nil
-							}); err != nil {
-								return nil, nil, fmt.Errorf("error visiting containers: %w", err)
-							}
+	if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE {
+		{
+			key := "gcp-cloud-controller.addons.k8s.io"
+			useBuiltin := !b.hasExternalAddon(key)
+
+			if !useBuiltin {
+				klog.Infof("Found cloud-controller-manager in addons; won't use builtin")
+
+				// Until we make the manifest extensible, we still need to inject our arguments.
+				// TODO(justinsb): we don't really want to do this, it limits the ability for users to override things.
+				// However, this is behind a feature flag at the moment, and this way we can work towards something better.
+				gkDaemonset := schema.GroupKind{Group: "apps", Kind: "DaemonSet"}
+				for _, addon := range b.ClusterAddons {
+					if addon.GroupVersionKind().GroupKind() == gkDaemonset &&
+						addon.GetName() == "cloud-controller-manager" &&
+						addon.GetNamespace() == "kube-system" {
+
+						klog.Infof("replacing arguments in externally provided cloud-controller-manager")
+
+						fnAny, ok := b.templates.TemplateFunctions["CloudControllerConfigArgv"]
+						if !ok {
+							return nil, nil, fmt.Errorf("unable to find TemplateFunction CloudControllerConfigArgv")
+						}
+						fn, ok := fnAny.(func() ([]string, error))
+						if !ok {
+							return nil, nil, fmt.Errorf("unexpected type for TemplateFunction CloudControllerConfigArgv: %T", fnAny)
+						}
+						args, err := fn()
+						if err != nil {
+							return nil, nil, fmt.Errorf("in TemplateFunction CloudControllerConfigArgv: %w", err)
+						}
+
+						if err := addon.VisitContainers(func(container map[string]interface{}) error {
+							// TODO: Check name?
+							container["args"] = args
+							return nil
+						}); err != nil {
+							return nil, nil, fmt.Errorf("error visiting containers: %w", err)
 						}
 					}
 				}
+			}
 
-				if useBuiltin {
-					id := "k8s-1.23"
-					location := key + "/" + id + ".yaml"
-					addon := addons.Add(&channelsapi.AddonSpec{
-						Name:     fi.PtrTo(key),
-						Manifest: fi.PtrTo(location),
-						Selector: map[string]string{"k8s-addon": key},
-						Id:       id,
-					})
-					addon.BuildPrune = true
-				}
+			if useBuiltin {
+				id := "k8s-1.23"
+				location := key + "/" + id + ".yaml"
+				addon := addons.Add(&channelsapi.AddonSpec{
+					Name:     fi.PtrTo(key),
+					Manifest: fi.PtrTo(location),
+					Selector: map[string]string{"k8s-addon": key},
+					Id:       id,
+				})
+				addon.BuildPrune = true
 			}
 		}
 	}

upup/pkg/fi/cloudup/new_cluster.go

@@ -1063,7 +1063,9 @@ func setupNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetsMap m
 			if g.Spec.NodeLabels == nil {
 				g.Spec.NodeLabels = make(map[string]string)
 			}
-			g.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true"
+			if cluster.IsKubernetesLT("1.29") {
+				g.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true"
+			}
 		}
 
 		for i, size := range opt.NodeSizes {