Support multiple service-account-issuers in apiserver

k8s/crds/kops.k8s.io_clusters.yaml

@@ -1620,6 +1620,12 @@ spec:
                 description: KubeAPIServerConfig defines the configuration for the
                   kube api
                 properties:
+                  additionalServiceAccountIssuers:
+                    description: AdditionalServiceAccountIssuers can contain additional
+                      service account token issuers.
+                    items:
+                      type: string
+                    type: array
                   address:
                     description: 'Address is the binding address for the kube api:
                       Deprecated - use insecure-bind-address and bind-address'

nodeup/pkg/model/kube_apiserver.go

@@ -678,6 +678,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
 			"/usr/local/bin/kube-apiserver",
 		}
 		container.Args = append(container.Args, sortedStrings(flags)...)
+		container.Args = sortServiceAccountIssuers(container.Args, fi.ValueOf(kubeAPIServer.ServiceAccountIssuer), kubeAPIServer.AdditionalServiceAccountIssuers)
 	}
 
 	for _, path := range b.SSLHostPaths() {
@@ -755,3 +756,26 @@ func (b *KubeAPIServerBuilder) buildAnnotations() map[string]string {
 
 	return annotations
 }
+
+func sortServiceAccountIssuers(in []string, currentIssuer string, oldIssuers []string) []string {
+	if len(oldIssuers) == 0 {
+		return in
+	}
+
+	positionMap := make(map[int]string)
+	positionMap[0] = fmt.Sprintf("--service-account-issuer=%s", currentIssuer)
+	for k, v := range oldIssuers {
+		positionMap[k+1] = fmt.Sprintf("--service-account-issuer=%s", v)
+	}
+
+	out := make([]string, 0, len(in))
+	issuerPosition := 0
+	for _, flag := range in {
+		if strings.HasPrefix(flag, "--service-account-issuer=") {
+			flag = positionMap[issuerPosition]
+			issuerPosition++
+		}
+		out = append(out, flag)
+	}
+	return out
+}

nodeup/pkg/model/kube_apiserver_test.go

@@ -19,6 +19,7 @@ package model
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
@@ -197,3 +198,37 @@ func TestKubeAPIServerBuilderARM64(t *testing.T) {
 		return builder.Build(target)
 	})
 }
+
+func TestSortServiceAccountIssuers(t *testing.T) {
+	type testCase struct {
+		name          string
+		flags         []string
+		currentIssuer string
+		oldIssuers    []string
+		expected      []string
+	}
+
+	testCases := []testCase{
+		{
+			name:          "flags without old issuers",
+			flags:         []string{"--bar=foo", "--service-account-issuer=https://foo"},
+			currentIssuer: "https://foo",
+			oldIssuers:    nil,
+			expected:      []string{"--bar=foo", "--service-account-issuer=https://foo"},
+		},
+		{
+			name:          "flags without with old issuers",
+			flags:         []string{"--bar=foo", "--service-account-issuer=https://aa", "--service-account-issuer=https://bar", "--service-account-issuer=https://foo", "--zz=zz"},
+			currentIssuer: "https://foo",
+			oldIssuers:    []string{"https://aa", "https://bar"},
+			expected:      []string{"--bar=foo", "--service-account-issuer=https://foo", "--service-account-issuer=https://aa", "--service-account-issuer=https://bar", "--zz=zz"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := sortServiceAccountIssuers(tc.flags, tc.currentIssuer, tc.oldIssuers)
+			assert.Equal(t, tc.expected, actual)
+		})
+	}
+}

nodeup/pkg/model/tests/golden/oidc/cluster.yaml

@@ -38,6 +38,12 @@ spec:
     - claim2=value2
     oidcUsernameClaim: user
     oidcUsernamePrefix: 'oidc:'
+    serviceAccountIssuer: https://huh.com/multisaissuer.example.com
+    additionalServiceAccountIssuers:
+    - https://foobar.com
+    - https://aaa.com
+    - https://hello.com
+    - https://dorld.com
   kubelet:
     anonymousAuth: false
   kubernetesVersion: v1.28.0

nodeup/pkg/model/tests/golden/oidc/tasks-kube-apiserver.yaml

@@ -52,8 +52,12 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-issuer=https://api.internal.minimal.example.com
-      - --service-account-jwks-uri=https://api.internal.minimal.example.com/openid/v1/jwks
+      - --service-account-issuer=https://huh.com/multisaissuer.example.com
+      - --service-account-issuer=https://foobar.com
+      - --service-account-issuer=https://aaa.com
+      - --service-account-issuer=https://hello.com
+      - --service-account-issuer=https://dorld.com
+      - --service-account-jwks-uri=https://huh.com/multisaissuer.example.com/openid/v1/jwks
       - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub
       - --service-account-signing-key-file=/srv/kubernetes/kube-apiserver/service-account.key
       - --service-cluster-ip-range=100.64.0.0/13

pkg/apis/kops/componentconfig.go

@@ -500,6 +500,9 @@ type KubeAPIServerConfig struct {
 	// in "iss" claim of issued tokens. This value is a string or URI.
 	ServiceAccountIssuer *string `json:"serviceAccountIssuer,omitempty" flag:"service-account-issuer"`
 
+	// AdditionalServiceAccountIssuers can contain additional service account token issuers.
+	AdditionalServiceAccountIssuers []string `json:"additionalServiceAccountIssuers,omitempty" flag:"service-account-issuer,repeat"`
+
 	// ServiceAccountJWKSURI overrides the path for the jwks document; this is useful when we are republishing the service account discovery information elsewhere.
 	ServiceAccountJWKSURI *string `json:"serviceAccountJWKSURI,omitempty" flag:"service-account-jwks-uri"`
 

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -507,6 +507,9 @@ type KubeAPIServerConfig struct {
 	// in "iss" claim of issued tokens. This value is a string or URI.
 	ServiceAccountIssuer *string `json:"serviceAccountIssuer,omitempty" flag:"service-account-issuer"`
 
+	// AdditionalServiceAccountIssuers can contain additional service account token issuers.
+	AdditionalServiceAccountIssuers []string `json:"additionalServiceAccountIssuers,omitempty" flag:"service-account-issuer,repeat"`
+
 	// ServiceAccountJWKSURI overrides the path for the jwks document; this is useful when we are republishing the service account discovery information elsewhere.
 	ServiceAccountJWKSURI *string `json:"serviceAccountJWKSURI,omitempty" flag:"service-account-jwks-uri"`
 

pkg/apis/kops/v1alpha3/componentconfig.go

@@ -498,6 +498,9 @@ type KubeAPIServerConfig struct {
 	// in "iss" claim of issued tokens. This value is a string or URI.
 	ServiceAccountIssuer *string `json:"serviceAccountIssuer,omitempty" flag:"service-account-issuer"`
 
+	// AdditionalServiceAccountIssuers can contain additional service account token issuers.
+	AdditionalServiceAccountIssuers []string `json:"additionalServiceAccountIssuers,omitempty" flag:"service-account-issuer,repeat"`
+
 	// ServiceAccountJWKSURI overrides the path for the jwks document; this is useful when we are republishing the service account discovery information elsewhere.
 	ServiceAccountJWKSURI *string `json:"serviceAccountJWKSURI,omitempty" flag:"service-account-jwks-uri"`
 

pkg/apis/kops/validation/validation.go

@@ -829,6 +829,9 @@ func validateKubeAPIServer(v *kops.KubeAPIServerConfig, c *kops.Cluster, fldPath
 		}
 	}
 
+	if c.IsKubernetesLT("1.22") && len(v.AdditionalServiceAccountIssuers) > 0 {
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("additionalServiceAccountIssuers"), "additionalServiceAccountIssuers requires Kubernetes 1.22+"))
+	}
 	return allErrs
 }
 

pkg/apis/kops/validation/validation_test.go

@@ -310,6 +310,23 @@ func TestValidateKubeAPIServer(t *testing.T) {
 			},
 			ExpectedErrors: []string{"Unsupported value::KubeAPIServer.logFormat"},
 		},
+		{
+			Input: kops.KubeAPIServerConfig{
+				AdditionalServiceAccountIssuers: []string{"https://foo.bar"},
+			},
+			Cluster: &kops.Cluster{
+				Spec: kops.ClusterSpec{
+					Authorization: &kops.AuthorizationSpec{
+						RBAC: &kops.RBACAuthorizationSpec{},
+					},
+					KubernetesVersion: "1.21.0",
+					CloudProvider: kops.CloudProviderSpec{
+						AWS: &kops.AWSSpec{},
+					},
+				},
+			},
+			ExpectedErrors: []string{"Forbidden::KubeAPIServer.additionalServiceAccountIssuers"},
+		},
 	}
 	for _, g := range grid {
 		if g.Cluster == nil {