diff --git a/cmd/kops/create_cluster_integration_test.go b/cmd/kops/create_cluster_integration_test.go index 36f8e125a0ea7..54c7ebdd5437b 100644 --- a/cmd/kops/create_cluster_integration_test.go +++ b/cmd/kops/create_cluster_integration_test.go @@ -113,6 +113,7 @@ func TestCreateClusterHA(t *testing.T) { func TestCreateClusterMinimalGCE(t *testing.T) { runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-gce", "v1alpha2") runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-gce-dns-none", "v1alpha2") + runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.29-gce", "v1alpha2") } // TestCreateClusterHAGCE runs kops create cluster ha-gce.example.com --cloud gce --zones us-test1-a,us-test1-b,us-test1-c --master-zones us-test1-a,us-test1-b,us-test1-c diff --git a/docs/releases/1.28-NOTES.md b/docs/releases/1.28-NOTES.md index 88b023a0d1c5c..37dc2a24223e3 100644 --- a/docs/releases/1.28-NOTES.md +++ b/docs/releases/1.28-NOTES.md @@ -12,6 +12,8 @@ This is a document to gather the release notes prior to the release. ## GCP +* [metadata-proxy](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy) is no longer deployed on GCP clusters for Kubernetes 1.29+. + ## Openstack # Breaking changes diff --git a/tests/integration/create_cluster/minimal-1.29-gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.29-gce/expected-v1alpha2.yaml new file mode 100644 index 0000000000000..e67d06c88a013 --- /dev/null +++ b/tests/integration/create_cluster/minimal-1.29-gce/expected-v1alpha2.yaml @@ -0,0 +1,96 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal.example.com +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudConfig: {} + cloudProvider: gce + configBase: memfs://tests/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: control-plane-us-test1-a + name: a + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: control-plane-us-test1-a + name: a + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: v1.29.0 + masterPublicName: api.minimal.example.com + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + project: testproject + sshAccess: + - 0.0.0.0/0 + - ::/0 + subnets: + - cidr: 10.0.16.0/20 + name: us-test1 + region: us-test1 + type: Public + topology: + dns: + type: Public + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal.example.com + name: control-plane-us-test1-a +spec: + image: ubuntu-os-cloud/ubuntu-2204-jammy-v20230714 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test1 + zones: + - us-test1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal.example.com + name: nodes-us-test1-a +spec: + image: ubuntu-os-cloud/ubuntu-2204-jammy-v20230714 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Node + subnets: + - us-test1 + zones: + - us-test1-a diff --git a/tests/integration/create_cluster/minimal-1.29-gce/options.yaml b/tests/integration/create_cluster/minimal-1.29-gce/options.yaml new file mode 100644 index 0000000000000..321bcad03e159 --- /dev/null +++ b/tests/integration/create_cluster/minimal-1.29-gce/options.yaml @@ -0,0 +1,7 @@ +CloudProvider: gce +ClusterName: minimal.example.com +KubernetesVersion: v1.29.0 +Networking: cni +Project: testproject +Zones: + - us-test1-a diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 8cccc9f59bde0..27b99a4f523d2 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -849,7 +849,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) // The metadata-proxy daemonset conceals node metadata endpoints in GCE. // It will land on nodes labeled cloud.google.com/metadata-proxy-ready=true - if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE { + if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE && b.Cluster.IsKubernetesLT("1.29") { key := "metadata-proxy.addons.k8s.io" { @@ -863,61 +863,61 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) Id: id, }) } + } - if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE { - { - key := "gcp-cloud-controller.addons.k8s.io" - useBuiltin := !b.hasExternalAddon(key) - - if !useBuiltin { - klog.Infof("Found cloud-controller-manager in addons; won't use builtin") - - // Until we make the manifest extensible, we still need to inject our arguments. - // TODO(justinsb): we don't really want to do this, it limits the ability for users to override things. - // However, this is behind a feature flag at the moment, and this way we can work towards something better. - gkDaemonset := schema.GroupKind{Group: "apps", Kind: "DaemonSet"} - for _, addon := range b.ClusterAddons { - if addon.GroupVersionKind().GroupKind() == gkDaemonset && - addon.GetName() == "cloud-controller-manager" && - addon.GetNamespace() == "kube-system" { - - klog.Infof("replacing arguments in externally provided cloud-controller-manager") - - fnAny, ok := b.templates.TemplateFunctions["CloudControllerConfigArgv"] - if !ok { - return nil, nil, fmt.Errorf("unable to find TemplateFunction CloudControllerConfigArgv") - } - fn, ok := fnAny.(func() ([]string, error)) - if !ok { - return nil, nil, fmt.Errorf("unexpected type for TemplateFunction CloudControllerConfigArgv: %T", fnAny) - } - args, err := fn() - if err != nil { - return nil, nil, fmt.Errorf("in TemplateFunction CloudControllerConfigArgv: %w", err) - } - - if err := addon.VisitContainers(func(container map[string]interface{}) error { - // TODO: Check name? - container["args"] = args - return nil - }); err != nil { - return nil, nil, fmt.Errorf("error visiting containers: %w", err) - } + if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE { + { + key := "gcp-cloud-controller.addons.k8s.io" + useBuiltin := !b.hasExternalAddon(key) + + if !useBuiltin { + klog.Infof("Found cloud-controller-manager in addons; won't use builtin") + + // Until we make the manifest extensible, we still need to inject our arguments. + // TODO(justinsb): we don't really want to do this, it limits the ability for users to override things. + // However, this is behind a feature flag at the moment, and this way we can work towards something better. + gkDaemonset := schema.GroupKind{Group: "apps", Kind: "DaemonSet"} + for _, addon := range b.ClusterAddons { + if addon.GroupVersionKind().GroupKind() == gkDaemonset && + addon.GetName() == "cloud-controller-manager" && + addon.GetNamespace() == "kube-system" { + + klog.Infof("replacing arguments in externally provided cloud-controller-manager") + + fnAny, ok := b.templates.TemplateFunctions["CloudControllerConfigArgv"] + if !ok { + return nil, nil, fmt.Errorf("unable to find TemplateFunction CloudControllerConfigArgv") + } + fn, ok := fnAny.(func() ([]string, error)) + if !ok { + return nil, nil, fmt.Errorf("unexpected type for TemplateFunction CloudControllerConfigArgv: %T", fnAny) + } + args, err := fn() + if err != nil { + return nil, nil, fmt.Errorf("in TemplateFunction CloudControllerConfigArgv: %w", err) + } + + if err := addon.VisitContainers(func(container map[string]interface{}) error { + // TODO: Check name? + container["args"] = args + return nil + }); err != nil { + return nil, nil, fmt.Errorf("error visiting containers: %w", err) } } } + } - if useBuiltin { - id := "k8s-1.23" - location := key + "/" + id + ".yaml" - addon := addons.Add(&channelsapi.AddonSpec{ - Name: fi.PtrTo(key), - Manifest: fi.PtrTo(location), - Selector: map[string]string{"k8s-addon": key}, - Id: id, - }) - addon.BuildPrune = true - } + if useBuiltin { + id := "k8s-1.23" + location := key + "/" + id + ".yaml" + addon := addons.Add(&channelsapi.AddonSpec{ + Name: fi.PtrTo(key), + Manifest: fi.PtrTo(location), + Selector: map[string]string{"k8s-addon": key}, + Id: id, + }) + addon.BuildPrune = true } } } diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index ced5a3e620e0a..e6eda9913b3ec 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -1063,7 +1063,9 @@ func setupNodes(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubnetsMap m if g.Spec.NodeLabels == nil { g.Spec.NodeLabels = make(map[string]string) } - g.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true" + if cluster.IsKubernetesLT("1.29") { + g.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true" + } } for i, size := range opt.NodeSizes {