From e1842606025b1f209f67abb8b581024f8abfea52 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Tue, 6 Dec 2022 12:01:48 +0000 Subject: [PATCH] feat: per cluster credentials This change allows you to optionally specify credentials to use for the cluster provisioning instead of using the controllers credentials. You specify a reference to a Secret via the `credentialsRef`. The secret must have the service account json added with a key called **credentials**. The e2e tests have been extended to include a test that uses the credentials. Signed-off-by: Richard Case --- api/v1alpha3/gcpcluster_conversion.go | 4 + api/v1alpha3/zz_generated.conversion.go | 1 + api/v1alpha4/conversion_test.go | 16 +- api/v1alpha4/gcpcluster_conversion.go | 9 ++ api/v1alpha4/gcpclustertemplate_conversion.go | 4 + api/v1alpha4/zz_generated.conversion.go | 16 +- api/v1beta1/gcpcluster_types.go | 5 + api/v1beta1/gcpcluster_webhook.go | 7 + api/v1beta1/types.go | 12 ++ api/v1beta1/zz_generated.deepcopy.go | 20 +++ cloud/scope/cluster.go | 40 ++++- .../compute/instances/reconcile_test.go | 4 +- .../compute/subnets/reconcile_test.go | 4 +- ...tructure.cluster.x-k8s.io_gcpclusters.yaml | 15 ++ ....cluster.x-k8s.io_gcpclustertemplates.yaml | 16 ++ controllers/gcpcluster_controller.go | 3 +- controllers/gcpmachine_controller.go | 2 +- test/e2e/common.go | 15 ++ test/e2e/config/gcp-ci.yaml | 1 + .../cluster-template-ci-with-creds.yaml | 142 ++++++++++++++++++ test/e2e/e2e_test.go | 37 +++++ 21 files changed, 347 insertions(+), 26 deletions(-) create mode 100644 test/e2e/data/infrastructure-gcp/cluster-template-ci-with-creds.yaml diff --git a/api/v1alpha3/gcpcluster_conversion.go b/api/v1alpha3/gcpcluster_conversion.go index b3632c1c5..5e4927e57 100644 --- a/api/v1alpha3/gcpcluster_conversion.go +++ b/api/v1alpha3/gcpcluster_conversion.go @@ -49,6 +49,10 @@ func (src *GCPCluster) ConvertTo(dstRaw conversion.Hub) error { // nolint } } + if restored.Spec.CredentialsRef != nil { + dst.Spec.CredentialsRef = restored.Spec.CredentialsRef + } + return nil } diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 02a3fa6ed..828acbc76 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -414,6 +414,7 @@ func autoConvert_v1beta1_GCPClusterSpec_To_v1alpha3_GCPClusterSpec(in *v1beta1.G } out.FailureDomains = *(*[]string)(unsafe.Pointer(&in.FailureDomains)) out.AdditionalLabels = *(*Labels)(unsafe.Pointer(&in.AdditionalLabels)) + // WARNING: in.CredentialsRef requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1alpha4/conversion_test.go b/api/v1alpha4/conversion_test.go index deebe8240..3317fb805 100644 --- a/api/v1alpha4/conversion_test.go +++ b/api/v1alpha4/conversion_test.go @@ -25,22 +25,22 @@ import ( func TestFuzzyConversion(t *testing.T) { t.Run("for GCPCluster", utilconversion.FuzzTestFunc(utilconversion.FuzzTestFuncInput{ - Hub: &v1beta1.GCPCluster{}, - Spoke: &GCPCluster{}, + Hub: &v1beta1.GCPCluster{}, + Spoke: &GCPCluster{}, })) t.Run("for GCPClusterTemplate", utilconversion.FuzzTestFunc(utilconversion.FuzzTestFuncInput{ - Hub: &v1beta1.GCPClusterTemplate{}, - Spoke: &GCPClusterTemplate{}, + Hub: &v1beta1.GCPClusterTemplate{}, + Spoke: &GCPClusterTemplate{}, })) t.Run("for GCPMachine", utilconversion.FuzzTestFunc(utilconversion.FuzzTestFuncInput{ - Hub: &v1beta1.GCPMachine{}, - Spoke: &GCPMachine{}, + Hub: &v1beta1.GCPMachine{}, + Spoke: &GCPMachine{}, })) t.Run("for GCPMachineTemplate", utilconversion.FuzzTestFunc(utilconversion.FuzzTestFuncInput{ - Hub: &v1beta1.GCPMachineTemplate{}, - Spoke: &GCPMachineTemplate{}, + Hub: &v1beta1.GCPMachineTemplate{}, + Spoke: &GCPMachineTemplate{}, })) } diff --git a/api/v1alpha4/gcpcluster_conversion.go b/api/v1alpha4/gcpcluster_conversion.go index 8461da083..f55cc12df 100644 --- a/api/v1alpha4/gcpcluster_conversion.go +++ b/api/v1alpha4/gcpcluster_conversion.go @@ -19,6 +19,7 @@ package v1alpha4 import ( apiconversion "k8s.io/apimachinery/pkg/conversion" infrav1beta1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + v1beta1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) @@ -47,6 +48,9 @@ func (src *GCPCluster) ConvertTo(dstRaw conversion.Hub) error { // nolint break } } + if restored.Spec.CredentialsRef != nil { + dst.Spec.CredentialsRef = restored.Spec.CredentialsRef.DeepCopy() + } return nil } @@ -83,3 +87,8 @@ func (dst *GCPClusterList) ConvertFrom(srcRaw conversion.Hub) error { // nolint func Convert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in *infrav1beta1.SubnetSpec, out *SubnetSpec, s apiconversion.Scope) error { return autoConvert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in, out, s) } + +// Convert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec is an autogenerated conversion function. +func Convert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(in *v1beta1.GCPClusterSpec, out *GCPClusterSpec, s apiconversion.Scope) error { + return autoConvert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(in, out, s) +} diff --git a/api/v1alpha4/gcpclustertemplate_conversion.go b/api/v1alpha4/gcpclustertemplate_conversion.go index 61114c95d..9c946103b 100644 --- a/api/v1alpha4/gcpclustertemplate_conversion.go +++ b/api/v1alpha4/gcpclustertemplate_conversion.go @@ -50,6 +50,10 @@ func (src *GCPClusterTemplate) ConvertTo(dstRaw conversion.Hub) error { // nolin } } + if restored.Spec.Template.Spec.CredentialsRef != nil { + dst.Spec.Template.Spec.CredentialsRef = restored.Spec.Template.Spec.CredentialsRef.DeepCopy() + } + return nil } diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index d508836e1..81e835a00 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -95,11 +95,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.GCPClusterSpec)(nil), (*GCPClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(a.(*v1beta1.GCPClusterSpec), b.(*GCPClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*GCPClusterStatus)(nil), (*v1beta1.GCPClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_GCPClusterStatus_To_v1beta1_GCPClusterStatus(a.(*GCPClusterStatus), b.(*v1beta1.GCPClusterStatus), scope) }); err != nil { @@ -260,6 +255,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.GCPClusterSpec)(nil), (*GCPClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(a.(*v1beta1.GCPClusterSpec), b.(*GCPClusterSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.GCPClusterTemplateResource)(nil), (*GCPClusterTemplateResource)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_GCPClusterTemplateResource_To_v1alpha4_GCPClusterTemplateResource(a.(*v1beta1.GCPClusterTemplateResource), b.(*GCPClusterTemplateResource), scope) }); err != nil { @@ -459,14 +459,10 @@ func autoConvert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(in *v1beta1.G } out.FailureDomains = *(*[]string)(unsafe.Pointer(&in.FailureDomains)) out.AdditionalLabels = *(*Labels)(unsafe.Pointer(&in.AdditionalLabels)) + // WARNING: in.CredentialsRef requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec is an autogenerated conversion function. -func Convert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(in *v1beta1.GCPClusterSpec, out *GCPClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta1_GCPClusterSpec_To_v1alpha4_GCPClusterSpec(in, out, s) -} - func autoConvert_v1alpha4_GCPClusterStatus_To_v1beta1_GCPClusterStatus(in *GCPClusterStatus, out *v1beta1.GCPClusterStatus, s conversion.Scope) error { if in.FailureDomains != nil { in, out := &in.FailureDomains, &out.FailureDomains diff --git a/api/v1beta1/gcpcluster_types.go b/api/v1beta1/gcpcluster_types.go index 75821bac7..920cda0e3 100644 --- a/api/v1beta1/gcpcluster_types.go +++ b/api/v1beta1/gcpcluster_types.go @@ -53,6 +53,11 @@ type GCPClusterSpec struct { // ones added by default. // +optional AdditionalLabels Labels `json:"additionalLabels,omitempty"` + + // CredentialsRef is a reference to a Secret that contains the credentials to use for provisioning this cluster. If not + // supplied then the credentials of the controller will be used. + // +optional + CredentialsRef *ObjectReference `json:"credentialsRef,omitempty"` } // GCPClusterStatus defines the observed state of GCPCluster. diff --git a/api/v1beta1/gcpcluster_webhook.go b/api/v1beta1/gcpcluster_webhook.go index dd6787aed..31000ce9d 100644 --- a/api/v1beta1/gcpcluster_webhook.go +++ b/api/v1beta1/gcpcluster_webhook.go @@ -75,6 +75,13 @@ func (c *GCPCluster) ValidateUpdate(oldRaw runtime.Object) error { ) } + if !reflect.DeepEqual(c.Spec.CredentialsRef, old.Spec.CredentialsRef) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "CredentialsRef"), + c.Spec.CredentialsRef, "field is immutable"), + ) + } + if len(allErrs) == 0 { return nil } diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 4676a1734..de44eacac 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -255,3 +255,15 @@ type ServiceAccount struct { // account. Scopes []string `json:"scopes,omitempty"` } + +// ObjectReference is a reference to another Kubernetes object instance. +type ObjectReference struct { + // Namespace of the referent. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + // +kubebuilder:validation:Required + Namespace string `json:"namespace"` + // Name of the referent. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + // +kubebuilder:validation:Required + Name string `json:"name"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 89e9b9219..6a4ca5792 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -176,6 +176,11 @@ func (in *GCPClusterSpec) DeepCopyInto(out *GCPClusterSpec) { (*out)[key] = val } } + if in.CredentialsRef != nil { + in, out := &in.CredentialsRef, &out.CredentialsRef + *out = new(ObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPClusterSpec. @@ -710,6 +715,21 @@ func (in *NetworkSpec) DeepCopy() *NetworkSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReference. +func (in *ObjectReference) DeepCopy() *ObjectReference { + if in == nil { + return nil + } + out := new(ObjectReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceAccount) DeepCopyInto(out *ServiceAccount) { *out = *in diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 129deb827..86665ce86 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -24,6 +24,9 @@ import ( "github.com/pkg/errors" "google.golang.org/api/compute/v1" + "google.golang.org/api/option" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/cloud" @@ -42,7 +45,7 @@ type ClusterScopeParams struct { // NewClusterScope creates a new Scope from the supplied parameters. // This is meant to be called for each reconcile iteration. -func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { +func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterScope, error) { if params.Cluster == nil { return nil, errors.New("failed to generate new scope from nil Cluster") } @@ -51,7 +54,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { } if params.GCPServices.Compute == nil { - computeSvc, err := compute.NewService(context.TODO()) + computeSvc, err := createComputeService(ctx, params) if err != nil { return nil, errors.Errorf("failed to create gcp compute client: %v", err) } @@ -73,6 +76,39 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { }, nil } +func createComputeService(ctx context.Context, params ClusterScopeParams) (*compute.Service, error) { + if params.GCPCluster.Spec.CredentialsRef == nil { + computeSvc, err := compute.NewService(ctx) + if err != nil { + return nil, errors.Errorf("failed to create gcp compute client: %v", err) + } + + return computeSvc, nil + } + + secretRefName := types.NamespacedName{ + Name: params.GCPCluster.Spec.CredentialsRef.Name, + Namespace: params.GCPCluster.Spec.CredentialsRef.Namespace, + } + + credSecret := &corev1.Secret{} + if err := params.Client.Get(ctx, secretRefName, credSecret); err != nil { + return nil, fmt.Errorf("getting credentials secret %s\\%s: %w", secretRefName.Namespace, secretRefName.Name, err) + } + + rawData, ok := credSecret.Data["credentials"] + if !ok { + return nil, errors.New("no credentials key in secret") + } + + computeSvc, err := compute.NewService(ctx, option.WithCredentialsJSON(rawData)) + if err != nil { + return nil, errors.Errorf("failed to create gcp compute client with credentials secret: %v", err) + } + + return computeSvc, nil +} + // ClusterScope defines the basic context for an actuator to operate upon. type ClusterScope struct { client client.Client diff --git a/cloud/services/compute/instances/reconcile_test.go b/cloud/services/compute/instances/reconcile_test.go index d9d2cc9f6..09cf43de2 100644 --- a/cloud/services/compute/instances/reconcile_test.go +++ b/cloud/services/compute/instances/reconcile_test.go @@ -133,7 +133,7 @@ func TestService_createOrGetInstance(t *testing.T) { WithObjects(fakeBootstrapSecret). Build() - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.TODO(), scope.ClusterScopeParams{ Client: fakec, Cluster: fakeCluster, GCPCluster: fakeGCPCluster, @@ -155,7 +155,7 @@ func TestService_createOrGetInstance(t *testing.T) { t.Fatal(err) } - clusterScopeWithoutFailureDomain, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScopeWithoutFailureDomain, err := scope.NewClusterScope(context.TODO(), scope.ClusterScopeParams{ Client: fakec, Cluster: fakeCluster, GCPCluster: fakeGCPClusterWithOutFailureDomain, diff --git a/cloud/services/compute/subnets/reconcile_test.go b/cloud/services/compute/subnets/reconcile_test.go index 8139916a6..90b44f11a 100644 --- a/cloud/services/compute/subnets/reconcile_test.go +++ b/cloud/services/compute/subnets/reconcile_test.go @@ -82,7 +82,7 @@ func TestService_Reconcile(t *testing.T) { WithScheme(scheme.Scheme). Build() - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.TODO(), scope.ClusterScopeParams{ Client: fakec, Cluster: fakeCluster, GCPCluster: fakeGCPCluster, @@ -179,7 +179,7 @@ func TestService_Delete(t *testing.T) { WithScheme(scheme.Scheme). Build() - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.TODO(), scope.ClusterScopeParams{ Client: fakec, Cluster: fakeCluster, GCPCluster: fakeGCPCluster, diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml index cf38f8cc1..b0402fc6e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml @@ -521,6 +521,21 @@ spec: - host - port type: object + credentialsRef: + description: CredentialsRef is a reference to a Secret that contains + the credentials to use for provisioning this cluster. If not supplied + then the credentials of the controller will be used. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + required: + - name + - namespace + type: object failureDomains: description: FailureDomains is an optional field which is used to assign selected availability zones to a cluster FailureDomains if diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml index 66d8a15a0..de4ff6bf8 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml @@ -235,6 +235,22 @@ spec: - host - port type: object + credentialsRef: + description: CredentialsRef is a reference to a Secret that + contains the credentials to use for provisioning this cluster. + If not supplied then the credentials of the controller will + be used. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + required: + - name + - namespace + type: object failureDomains: description: FailureDomains is an optional field which is used to assign selected availability zones to a cluster diff --git a/controllers/gcpcluster_controller.go b/controllers/gcpcluster_controller.go index 04da5ee66..0e53c08e2 100644 --- a/controllers/gcpcluster_controller.go +++ b/controllers/gcpcluster_controller.go @@ -57,6 +57,7 @@ type GCPClusterReconciler struct { // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=gcpclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=gcpclusters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch func (r *GCPClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { log := log.FromContext(ctx).WithValues("controller", "GCPCluster") @@ -133,7 +134,7 @@ func (r *GCPClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Cluster: cluster, GCPCluster: gcpCluster, diff --git a/controllers/gcpmachine_controller.go b/controllers/gcpmachine_controller.go index 0b220bf72..924faec8f 100644 --- a/controllers/gcpmachine_controller.go +++ b/controllers/gcpmachine_controller.go @@ -180,7 +180,7 @@ func (r *GCPMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Create the cluster scope - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Cluster: cluster, GCPCluster: gcpCluster, diff --git a/test/e2e/common.go b/test/e2e/common.go index 3b82892b3..57bf7574e 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/ginkgo" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/util" @@ -48,6 +49,20 @@ func setupSpecNamespace(ctx context.Context, specName string, clusterProxy frame return namespace, cancelWatches } +func createSecret(ctx context.Context, name, namespace string, data map[string][]byte, clusterProxy framework.ClusterProxy) error { + Byf("Creating secret %s in namespace %s", name, namespace) + + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } + + return clusterProxy.GetClient().Create(ctx, secret) +} + type cleanupInput struct { SpecName string ClusterProxy framework.ClusterProxy diff --git a/test/e2e/config/gcp-ci.yaml b/test/e2e/config/gcp-ci.yaml index 21e588d6e..e260a9f90 100644 --- a/test/e2e/config/gcp-ci.yaml +++ b/test/e2e/config/gcp-ci.yaml @@ -68,6 +68,7 @@ providers: - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-kcp-remediation.yaml" - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-topology.yaml" - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/clusterclass-quick-start.yaml" + - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-creds.yaml" variables: KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.23.8}" diff --git a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-creds.yaml b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-creds.yaml new file mode 100644 index 000000000..1aefa293f --- /dev/null +++ b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-creds.yaml @@ -0,0 +1,142 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cni: "${CLUSTER_NAME}-crs-cni" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + project: "${GCP_PROJECT}" + region: "${GCP_REGION}" + network: + name: "${GCP_NETWORK_NAME}" + credentialsRef: + name: test-creds + namespace: default +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: GCPMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + useExperimentalRetryJoin: true + initConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce + clusterConfiguration: + apiServer: + timeoutForControlPlane: 20m + extraArgs: + cloud-provider: gce + controllerManager: + extraArgs: + cloud-provider: gce + allocate-node-cidrs: "false" + kubernetesVersion: "${KUBERNETES_VERSION}" + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + instanceType: "${GCP_NODE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "${CLUSTER_NAME}-crs-cni" +data: ${CNI_RESOURCES} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: "${CLUSTER_NAME}-crs-cni" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-cni" + resources: + - name: "${CLUSTER_NAME}-crs-cni" + kind: ConfigMap diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index f3147cf68..220cb64db 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -147,4 +147,41 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) }) + + Context("Creating a single control-plane cluster with per cluster credentials", func() { + It("Should create a cluster with 1 worker node", func() { + By("Create the credentials secret") + + credsFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + Expect(credsFile).NotTo(BeEmpty()) + data, err := os.ReadFile(credsFile) + Expect(err).NotTo(HaveOccurred()) + secretData := map[string][]byte{ + "credentials": data, + } + err = createSecret(ctx, "test-creds", "default", secretData, bootstrapClusterProxy) + Expect(err).NotTo(HaveOccurred(), "failed creating credentials sercret") + + By("Initializes with 1 worker node") + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "ci-with-creds", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) })