From 5655b12d5e390e036dfc4a15ac3e5971fc7b11a7 Mon Sep 17 00:00:00 2001 From: Niki Dokovski Date: Thu, 15 Feb 2024 14:14:43 +0100 Subject: [PATCH] Remove deployment and statefulset mutating webhooks --- VERSION | 2 +- .../templates/clusterrole.yaml | 2 +- .../templates/webhook.yaml | 55 +---- charts/oidc-apps-controller/values.yaml | 11 +- go.mod | 2 +- pkg/controllers/deployment-reconciler.go | 53 +---- pkg/controllers/modifiers.go | 36 +--- pkg/controllers/oidc-apps-ingresses.go | 66 +++++- pkg/controllers/oidc-apps-secrets.go | 21 +- pkg/controllers/oidc-apps-services.go | 20 +- pkg/controllers/statefulset-reconciler.go | 34 +-- pkg/oidc-apps-controller/cmd.go | 21 +- pkg/webhook/deployment-webhook.go | 140 ------------ pkg/webhook/modifiers.go | 55 +++-- pkg/webhook/pod-webhook.go | 168 ++++++++++++--- pkg/webhook/statefulset-webhook.go | 152 ------------- pkg/webhook/test/deployment_webhook_test.go | 189 ---------------- pkg/webhook/test/mutating-webhook_test.go | 36 ++++ ..._suite_test.go => oisc-apps_suite_test.go} | 18 ++ test/e2e/deployment_test.go | 56 +++++ test/e2e/deployment_webhook.go | 204 ------------------ test/e2e/statefulset_test.go | 63 ++++++ test/e2e/suite_test.go | 16 ++ 23 files changed, 475 insertions(+), 945 deletions(-) delete mode 100644 pkg/webhook/deployment-webhook.go delete mode 100644 pkg/webhook/statefulset-webhook.go delete mode 100644 pkg/webhook/test/deployment_webhook_test.go create mode 100644 pkg/webhook/test/mutating-webhook_test.go rename pkg/webhook/test/{webhook_suite_test.go => oisc-apps_suite_test.go} (68%) create mode 100644 test/e2e/deployment_test.go delete mode 100644 test/e2e/deployment_webhook.go create mode 100644 test/e2e/statefulset_test.go diff --git a/VERSION b/VERSION index ca25b0a..3ea3eab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.1.8-dev +v0.2.0-dev diff --git a/charts/oidc-apps-controller/templates/clusterrole.yaml b/charts/oidc-apps-controller/templates/clusterrole.yaml index 329429c..ce30a86 100644 --- a/charts/oidc-apps-controller/templates/clusterrole.yaml +++ b/charts/oidc-apps-controller/templates/clusterrole.yaml @@ -10,7 +10,7 @@ rules: resources: [ "services","secrets","events" ] verbs: [ "*" ] - apiGroups: [ "apps" ] - resources: [ "deployments","statefulsets" ] + resources: [ "deployments","statefulsets", "replicasets" ] verbs: [ "*" ] - apiGroups: [ "networking.k8s.io" ] resources: [ "ingresses" ] diff --git a/charts/oidc-apps-controller/templates/webhook.yaml b/charts/oidc-apps-controller/templates/webhook.yaml index 91e9dde..28c65a3 100644 --- a/charts/oidc-apps-controller/templates/webhook.yaml +++ b/charts/oidc-apps-controller/templates/webhook.yaml @@ -10,56 +10,6 @@ metadata: cert-manager.io/inject-ca-from: {{ include "oidc-apps-extension.certificateRef" . }} {{- end }} webhooks: - - name: {{ include "oidc-apps-extension.fullname" . }}-deployments.gardener.cloud - clientConfig: - service: - name: {{ include "oidc-apps-extension.fullname" . }} - namespace: {{ .Release.Namespace }} - path: /oidc-mutate-v1-deployment - port: {{ .Values.service.port | int }} - caBundle: - rules: - - operations: ["CREATE", "UPDATE"] - apiGroups: ["apps"] - apiVersions: ["v1"] - resources: ["deployments"] - {{- with .Values.webhook.objectSelector }} - objectSelector: - {{- toYaml . | nindent 6 }} - {{- end }} - {{- with .Values.webhook.namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 6 }} - {{- end }} - sideEffects: NoneOnDryRun - admissionReviewVersions: - - v1 - reinvocationPolicy: IfNeeded - - name: {{ include "oidc-apps-extension.fullname" . }}-statefulsets.gardener.cloud - clientConfig: - service: - name: {{ include "oidc-apps-extension.fullname" . }} - namespace: {{ .Release.Namespace }} - path: /oidc-mutate-v1-statefulset - port: {{ .Values.service.port | int }} - caBundle: - rules: - - operations: [ "CREATE", "UPDATE" ] - apiGroups: [ "apps" ] - apiVersions: [ "v1" ] - resources: [ "statefulsets" ] - {{- with .Values.webhook.objectSelector }} - objectSelector: - {{- toYaml . | nindent 6 }} - {{- end }} - {{- with .Values.webhook.namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 6 }} - {{- end }} - sideEffects: NoneOnDryRun - admissionReviewVersions: - - v1 - reinvocationPolicy: IfNeeded - name: {{ include "oidc-apps-extension.fullname" . }}-pods.gardener.cloud clientConfig: service: @@ -73,9 +23,10 @@ webhooks: apiGroups: [ "" ] apiVersions: [ "v1" ] resources: [ "pods" ] + {{- with .Values.webhook.objectSelector }} objectSelector: - matchLabels: - oidc-application-controller/component: pod + {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values.webhook.namespaceSelector }} namespaceSelector: {{- toYaml . | nindent 6 }} diff --git a/charts/oidc-apps-controller/values.yaml b/charts/oidc-apps-controller/values.yaml index 4a385fe..cb03ece 100644 --- a/charts/oidc-apps-controller/values.yaml +++ b/charts/oidc-apps-controller/values.yaml @@ -156,12 +156,11 @@ targets: # In case of Gardener there can be a mapping between seed names and oidc client ids clients: # Seed name shall match the seed identifier - - name: app1 - # clientId shall be the oidc client id configured at the oidc provider - clientId: 10365f8c-d9ba-44f9-8e85-741e8cc01f9c - - name: app2 - clientId : 98a36b69-6592-463a-b605-d4c563f8b016 - + # - name: app1 + # clientId shall be the oidc client id configured at the oidc provider + # clientId: 10365f8c-d9ba-44f9-8e85-741e8cc01f9c + # - name: app2 + # clientId : 98a36b69-6592-463a-b605-d4c563f8b016 gardener: # seed: diff --git a/go.mod b/go.mod index 03c81e0..6e2b2aa 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.2 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect diff --git a/pkg/controllers/deployment-reconciler.go b/pkg/controllers/deployment-reconciler.go index 3d17142..6615a98 100644 --- a/pkg/controllers/deployment-reconciler.go +++ b/pkg/controllers/deployment-reconciler.go @@ -18,7 +18,6 @@ import ( "context" "github.com/gardener/oidc-apps-controller/pkg/configuration" - oidc_apps_controller "github.com/gardener/oidc-apps-controller/pkg/constants" appsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,22 +33,18 @@ type DeploymentReconciler struct { // Reconcile creates the auth & zutz secrets mounted to the target deployment func (d *DeploymentReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - _log := log.FromContext(ctx) - reconciledDeployment := &appsv1.Deployment{} if err := d.Client.Get(ctx, request.NamespacedName, reconciledDeployment); client.IgnoreNotFound(err) != nil { return reconcile.Result{}, err } + _log := log.FromContext(ctx).WithValues("resourceVersion", reconciledDeployment.GetResourceVersion()) // Skip resource without an identity if reconciledDeployment.GetName() == "" && reconciledDeployment.GetNamespace() == "" { _log.V(9).Info("reconciled deployment is empty, returning ...") return reconcile.Result{}, nil } - _log = _log.WithValues( - "resourceVersion", reconciledDeployment.GetResourceVersion(), - "generation", reconciledDeployment.GetGeneration(), - ) + _log.V(9).Info("handling deployment reconcile request") if reconciledDeployment.GetLabels() != nil { @@ -59,52 +54,12 @@ func (d *DeploymentReconciler) Reconcile(ctx context.Context, request reconcile. } } - // In case the deployment is an OIDC target but has not been modified by the oidc admission controller - // then we trigger an update of the resource - annotations := reconciledDeployment.GetAnnotations() - if len(annotations) == 0 { - _log.Info("Reconciled deployment is not annotated with the oidc-application-controller annotations, " + - "re-triggering the admission controller...") - return reconcile.Result{}, nil - } - if _, found := annotations[oidc_apps_controller.AnnotationTargetKey]; !found { - _log.Info("Reconciled deployment is not annotated with the oidc-application-controller annotations, " + - "re-triggering the admission controller...") - return reconcile.Result{}, nil - } - - // add a finalizer - /* - if !controllerutil.ContainsFinalizer(reconciledDeployment, oidc_apps_controller.Finalizer) && !reconciledDeployment. - GetDeletionTimestamp().IsZero() { - controllerutil.AddFinalizer(reconciledDeployment, oidc_apps_controller.Finalizer) - if err := d.Client.Update(ctx, reconciledDeployment); err != nil { - return reconcile.Result{}, err - } - }*/ - // Check for deletion & handle cleanup of the dependencies if !reconciledDeployment.GetDeletionTimestamp().IsZero() { _log.V(9).Info("Remove owned resources") if err := deleteOwnedResources(ctx, d.Client, reconciledDeployment); err != nil { return reconcile.Result{}, err } - - /* - _log.V(9).Info("Remove finalizer") - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := d.Client.Get(ctx, request.NamespacedName, reconciledDeployment); client.IgnoreNotFound( - err) != nil { - return err - } - controllerutil.RemoveFinalizer(reconciledDeployment, oidc_apps_controller.Finalizer) - return d.Client.Update(ctx, reconciledDeployment) - }); err != nil { - _log.Error(err, "Error removing finalizer") - return reconcile.Result{}, nil - } - */ - return reconcile.Result{}, nil } @@ -112,10 +67,6 @@ func (d *DeploymentReconciler) Reconcile(ctx context.Context, request reconcile. return reconcile.Result{}, err } - if _log.GetV() == 9 { - logOwnedResources(ctx, d.Client, reconciledDeployment) - } - return reconcile.Result{}, nil } diff --git a/pkg/controllers/modifiers.go b/pkg/controllers/modifiers.go index 813a5fd..aa248d3 100644 --- a/pkg/controllers/modifiers.go +++ b/pkg/controllers/modifiers.go @@ -178,7 +178,7 @@ func reconcileDeploymentDependencies(ctx context.Context, c client.Client, objec } // Create or update the oauth2 service setting the owner reference - if oauth2Service, err = createOauth2Service("", object); err != nil { + if oauth2Service, err = createOauth2Service(object); err != nil { return fmt.Errorf("failed to create oauth2 service: %w", err) } if err := controllerutil.SetOwnerReference(object, &oauth2Service, c.Scheme()); err != nil { @@ -227,12 +227,10 @@ func reconcileDeploymentDependencies(ctx context.Context, c client.Client, objec } // Create or update the oauth2 ingress setting the owner reference - if oauth2Ingress, err = createIngress(object.GetAnnotations()[oidc_apps_controller.AnnotationHostKey], "", - object); err != nil { + if oauth2Ingress, err = createIngressForDeployment(object); err != nil { return fmt.Errorf("failed to create oauth2 ingress: %w", err) } - if err = controllerutil.SetOwnerReference(object, &oauth2Ingress, - c.Scheme()); err != nil { + if err = controllerutil.SetOwnerReference(object, &oauth2Ingress, c.Scheme()); err != nil { return fmt.Errorf("failed to set owner reference to oauth2 ingress: %w", err) } if _, err = controllerutil.CreateOrUpdate(ctx, c, &oauth2Ingress, mutateFn); err != nil { @@ -282,28 +280,16 @@ func reconcileStatefulSetDependencies(ctx context.Context, c client.Client, obje if err := c.List(ctx, podList, labelSelector, client.InNamespace(object.GetNamespace())); err != nil { return fmt.Errorf("failed to list pods: %w", err) } - hostPrefix := object.GetAnnotations()[oidc_apps_controller.AnnotationHostKey] - suffix := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] + for _, pod := range podList.Items { - if len(pod.Annotations) == 0 { - pod.Annotations = make(map[string]string, 1) - } - pod.Annotations[oidc_apps_controller.AnnotationSuffixKey] = suffix - - var podIndex string - host, domain, found := strings.Cut(hostPrefix, ".") - if found { - // In some environments, the pod index is added as a label: apps.kubernetes.io/pod-index - if idx, present := pod.GetObjectMeta().GetLabels()["statefulset.kubernetes.io/pod-name"]; present { - l := strings.Split(idx, "-") - host = fmt.Sprintf("%s-%s.%s", host, l[len(l)-1], domain) - podIndex = l[len(l)-1] - } else { - host = fmt.Sprintf("%s.%s", host, domain) - } + log.FromContext(ctx).V(9).Info("Reconciling pod", "pod", pod.GetName(), "annotations", pod.GetAnnotations()) + _, found := pod.GetAnnotations()[oidc_apps_controller.AnnotationHostKey] + if !found { + continue } + // Create or update the oauth2 service setting the owner reference - if oauth2Service, err = createOauth2Service(podIndex, &pod); err != nil { + if oauth2Service, err = createOauth2Service(&pod); err != nil { return fmt.Errorf("failed to create oauth2 service: %w", err) } if err := controllerutil.SetOwnerReference(&pod, &oauth2Service, c.Scheme()); err != nil { @@ -314,7 +300,7 @@ func reconcileStatefulSetDependencies(ctx context.Context, c client.Client, obje } // Create or update the oauth2 ingress setting the owner reference - if oauth2Ingress, err = createIngress(host, podIndex, object); err != nil { + if oauth2Ingress, err = createIngressForStatefulSetPod(&pod, object); err != nil { return fmt.Errorf("failed to create oauth2 ingress: %w", err) } if err = controllerutil.SetOwnerReference(&pod, &oauth2Ingress, c.Scheme()); err != nil { diff --git a/pkg/controllers/oidc-apps-ingresses.go b/pkg/controllers/oidc-apps-ingresses.go index 1f2c286..946032c 100644 --- a/pkg/controllers/oidc-apps-ingresses.go +++ b/pkg/controllers/oidc-apps-ingresses.go @@ -16,26 +16,28 @@ package controllers import ( "fmt" + "strings" + "github.com/gardener/oidc-apps-controller/pkg/configuration" oidc_apps_controller "github.com/gardener/oidc-apps-controller/pkg/constants" + "github.com/gardener/oidc-apps-controller/pkg/rand" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) -func createIngress(host string, index string, object client.Object) (networkingv1.Ingress, error) { - suffix, ok := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] - if !ok { - return networkingv1.Ingress{}, fmt.Errorf("missing suffix annotation") - } +func createIngressForDeployment(object client.Object) (networkingv1.Ingress, error) { + suffix := rand.GenerateSha256(object.GetName() + "-" + object.GetNamespace()) ingressClassName := configuration.GetOIDCAppsControllerConfig().GetIngressClassName(object) ingressTLSSecretName := configuration.GetOIDCAppsControllerConfig().GetIngressTLSSecretName(object) + host := configuration.GetOIDCAppsControllerConfig().GetHost(object) return networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: "ingress-" + addOptionalIndex(index+"-") + suffix, + Name: "ingress-" + suffix, Namespace: object.GetNamespace(), Labels: map[string]string{oidc_apps_controller.LabelKey: "oauth2"}, }, @@ -50,6 +52,58 @@ func createIngress(host string, index string, object client.Object) (networkingv Rules: []networkingv1.IngressRule{ { Host: host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: ptr.To(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "oauth2-service-" + suffix, + Port: networkingv1.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, nil +} + +func createIngressForStatefulSetPod(pod *corev1.Pod, object client.Object) (networkingv1.Ingress, error) { + suffix := rand.GenerateSha256(pod.GetName() + "-" + pod.GetNamespace()) + ingressClassName := configuration.GetOIDCAppsControllerConfig().GetIngressClassName(object) + ingressTLSSecretName := configuration.GetOIDCAppsControllerConfig().GetIngressTLSSecretName(object) + hostPrefix, ok := pod.GetAnnotations()[oidc_apps_controller.AnnotationHostKey] + if !ok { + return networkingv1.Ingress{}, fmt.Errorf("host annotation not found in pod %s/%s", pod.GetNamespace(), pod.GetName()) + } + host, domain, _ := strings.Cut(hostPrefix, ".") + index := fetchStrIndexIfPresent(pod) + + return networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-" + addOptionalIndex(index+"-") + suffix, + Namespace: object.GetNamespace(), + Labels: map[string]string{oidc_apps_controller.LabelKey: "oauth2"}, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To(ingressClassName), + TLS: []networkingv1.IngressTLS{ + { + Hosts: []string{fmt.Sprintf("%s-%s.%s", host, fetchStrIndexIfPresent(pod), domain)}, + SecretName: ingressTLSSecretName, + }, + }, + Rules: []networkingv1.IngressRule{ + { + Host: fmt.Sprintf("%s-%s.%s", host, fetchStrIndexIfPresent(pod), domain), IngressRuleValue: networkingv1.IngressRuleValue{ HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ diff --git a/pkg/controllers/oidc-apps-secrets.go b/pkg/controllers/oidc-apps-secrets.go index e06c986..dffec27 100644 --- a/pkg/controllers/oidc-apps-secrets.go +++ b/pkg/controllers/oidc-apps-secrets.go @@ -35,10 +35,8 @@ import ( var errSecretDoesNotExist = errors.New("secret does not exist") func createOauth2Secret(object client.Object) (corev1.Secret, error) { - suffix, ok := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] - if !ok { - return corev1.Secret{}, fmt.Errorf("missing suffix annotation") - } + suffix := rand.GenerateSha256(object.GetName() + "-" + object.GetNamespace()) + extConfig := configuration.GetOIDCAppsControllerConfig() var cfg string switch extConfig.GetClientSecret(object) { @@ -75,10 +73,7 @@ func createOauth2Secret(object client.Object) (corev1.Secret, error) { } func createResourceAttributesSecret(object client.Object, targetNamespace string) (corev1.Secret, error) { - suffix, ok := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] - if !ok { - return corev1.Secret{}, fmt.Errorf("missing suffix annotation") - } + suffix := rand.GenerateSha256(object.GetName() + "-" + object.GetNamespace()) // TODO: add configurable resource, subresource cfg := configuration.NewResourceAttributes( @@ -96,10 +91,7 @@ func createResourceAttributesSecret(object client.Object, targetNamespace string } func createKubeconfigSecret(object client.Object) (corev1.Secret, error) { - suffix, ok := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] - if !ok { - return corev1.Secret{}, fmt.Errorf("missing suffix annotation") - } + suffix := rand.GenerateSha256(object.GetName() + "-" + object.GetNamespace()) kubeConfigStr := configuration.GetOIDCAppsControllerConfig().GetKubeConfigStr(object) if len(kubeConfigStr) > 0 { @@ -186,10 +178,7 @@ func createKubeconfigSecret(object client.Object) (corev1.Secret, error) { } func createOidcCaBundleSecret(object client.Object) (corev1.Secret, error) { - suffix, ok := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] - if !ok { - return corev1.Secret{}, fmt.Errorf("missing suffix annotation") - } + suffix := rand.GenerateSha256(object.GetName() + "-" + object.GetNamespace()) oidcCABundle := configuration.GetOIDCAppsControllerConfig().GetOidcCABundle(object) if len(oidcCABundle) > 0 { // TODO: verify the oidcCABundle str, it shall be CA certificates in PEM format diff --git a/pkg/controllers/oidc-apps-services.go b/pkg/controllers/oidc-apps-services.go index 31229cb..14acc4b 100644 --- a/pkg/controllers/oidc-apps-services.go +++ b/pkg/controllers/oidc-apps-services.go @@ -15,9 +15,10 @@ package controllers import ( - "fmt" + "strings" oidc_apps_controller "github.com/gardener/oidc-apps-controller/pkg/constants" + "github.com/gardener/oidc-apps-controller/pkg/rand" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,11 +26,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func createOauth2Service(index string, object client.Object) (corev1.Service, error) { - suffix, ok := object.GetAnnotations()[oidc_apps_controller.AnnotationSuffixKey] - if !ok { - return corev1.Service{}, fmt.Errorf("missing suffix annotation") - } +func createOauth2Service(object client.Object) (corev1.Service, error) { + suffix := rand.GenerateSha256(object.GetName() + "-" + object.GetNamespace()) + index := fetchStrIndexIfPresent(object) return corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -50,3 +49,12 @@ func createOauth2Service(index string, object client.Object) (corev1.Service, er }, }, nil } + +func fetchStrIndexIfPresent(object client.Object) string { + idx, present := object.GetLabels()["statefulset.kubernetes.io/pod-name"] + if present { + l := strings.Split(idx, "-") + return l[len(l)-1] + } + return "" +} diff --git a/pkg/controllers/statefulset-reconciler.go b/pkg/controllers/statefulset-reconciler.go index 7da331e..0219d46 100644 --- a/pkg/controllers/statefulset-reconciler.go +++ b/pkg/controllers/statefulset-reconciler.go @@ -18,7 +18,6 @@ import ( "context" "github.com/gardener/oidc-apps-controller/pkg/configuration" - oidc_apps_controller "github.com/gardener/oidc-apps-controller/pkg/constants" appsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,42 +33,22 @@ type StatefulSetReconciler struct { // Reconcile creates the auth & zutz secrets mounted to the target statefulset func (s *StatefulSetReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - _log := log.FromContext(ctx) - reconciledStatefulSet := &appsv1.StatefulSet{} + if err := s.Client.Get(ctx, request.NamespacedName, reconciledStatefulSet); client.IgnoreNotFound(err) != nil { return reconcile.Result{}, err } + _log := log.FromContext(ctx).WithValues("resourceVersion", reconciledStatefulSet.GetResourceVersion()) - // Skip resource without an identity if reconciledStatefulSet.GetName() == "" && reconciledStatefulSet.GetNamespace() == "" { _log.V(9).Info("Reconciled statefulset is empty, returning ...") return reconcile.Result{}, nil } - _log = _log.WithValues( - "resourceVersion", reconciledStatefulSet.GetResourceVersion(), - "generation", reconciledStatefulSet.GetGeneration(), - ) - _log.V(9).Info("handling statefulset reconcile request") - if reconciledStatefulSet.GetLabels() != nil { - if !configuration.GetOIDCAppsControllerConfig().Match(reconciledStatefulSet) { - _log.V(9).Info("Reconciled statefulset is not an oidc-application-controller target, returning ...") - return reconcile.Result{}, nil - } - } + _log.V(9).Info("handling statefulset reconcile request") - // In case the deployment is an OIDC target but has not been modified by the oidc admission controller - // then we trigger an update of the resource - annotations := reconciledStatefulSet.GetAnnotations() - if len(annotations) == 0 { - _log.Info("Reconciled statefulset is not annotated with the oidc-application-controller annotations, " + - "re-triggering the admission controller...") - return reconcile.Result{}, nil - } - if _, found := annotations[oidc_apps_controller.AnnotationTargetKey]; !found { - _log.Info("Reconciled statefulset is not annotated with the oidc-application-controller annotations, " + - "re-triggering the admission controller...") + if !configuration.GetOIDCAppsControllerConfig().Match(reconciledStatefulSet) { + _log.V(9).Info("Reconciled statefulset is not an oidc-application-controller target, returning ...") return reconcile.Result{}, nil } @@ -86,9 +65,6 @@ func (s *StatefulSetReconciler) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - if _log.GetV() == 9 { - logOwnedResources(ctx, s.Client, reconciledStatefulSet) - } return reconcile.Result{}, nil } diff --git a/pkg/oidc-apps-controller/cmd.go b/pkg/oidc-apps-controller/cmd.go index 8ffee8a..81e7499 100644 --- a/pkg/oidc-apps-controller/cmd.go +++ b/pkg/oidc-apps-controller/cmd.go @@ -492,31 +492,14 @@ func addWebhooks(mgr manager.Manager, o *OidcAppsControllerOptions) error { }) webhookServer.Register( - constants.DeploymentWebHookPath, - &webhook.Admission{Handler: &oidcappswebhook.DeploymentMutator{ - Client: mgr.GetClient(), - Decoder: admission.NewDecoder(scheme.Scheme), - ImagePullSecret: o.registrySecret, - }}, - ) - - webhookServer.Register( - constants.StatefulsetWebHookPath, - &webhook.Admission{Handler: &oidcappswebhook.StatefulSetMutator{ + constants.PodWebHookPath, + &webhook.Admission{Handler: &oidcappswebhook.PodMutator{ Client: mgr.GetClient(), Decoder: admission.NewDecoder(scheme.Scheme), ImagePullSecret: o.registrySecret, }}, ) - webhookServer.Register( - constants.PodWebHookPath, - &webhook.Admission{Handler: &oidcappswebhook.PodMutator{ - Client: mgr.GetClient(), - Decoder: admission.NewDecoder(scheme.Scheme), - }}, - ) - // Add the server to the manager return mgr.Add(webhookServer) diff --git a/pkg/webhook/deployment-webhook.go b/pkg/webhook/deployment-webhook.go deleted file mode 100644 index eb2fb94..0000000 --- a/pkg/webhook/deployment-webhook.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package webhook - -import ( - "context" - "fmt" - "net/http" - - "github.com/gardener/oidc-apps-controller/pkg/configuration" - "github.com/gardener/oidc-apps-controller/pkg/constants" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/util/json" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// Register the webhook with the server -var _ admission.Handler = &DeploymentMutator{} - -// DeploymentMutator is a handler modifying the resource definitions of the deployment targets -type DeploymentMutator struct { - Client client.Client - Decoder *webhook.AdmissionDecoder - ImagePullSecret string -} - -// Handle provides interface implementation for the DeploymentMutator -func (a *DeploymentMutator) Handle(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse { - - _log := log.FromContext(ctx) - - if a.Decoder == nil { - return webhook.Errored(http.StatusInternalServerError, - fmt.Errorf("decoder in the admission handler cannot be nil")) - } - - deployment := &appsv1.Deployment{} - if err := a.Decoder.Decode(req, deployment); err != nil { - return webhook.Errored(http.StatusBadRequest, err) - } - - _log.V(9).Info("handling admission request", "resourceVersion", deployment.GetResourceVersion(), - "generation", deployment.GetGeneration()) - - // Simply return it the deployment is not part of the described targets - if !configuration.GetOIDCAppsControllerConfig().Match(deployment) { - return webhook.Allowed("not a target") - } - - // Simply return if it is a delete operation - if !deployment.GetDeletionTimestamp().IsZero() { - return webhook.Allowed("delete") - } - - patch := deployment.DeepCopy() - clientId := configuration.GetOIDCAppsControllerConfig().GetClientID(patch) - issuerUrl := configuration.GetOIDCAppsControllerConfig().GetOidcIssuerUrl(patch) - target := configuration.GetOIDCAppsControllerConfig().GetUpstreamTarget(patch) - upstreamUrl := fetchUpstreamUrl(target, patch.Spec.Template.Spec) - suffix := fetchTargetSuffix(patch) - - // Add the OIDC annotation to the deployment template - addAnnotations(patch) - - // Add required labels to pod spec template - addPodLabels(&patch.Spec.Template, nil) - - // Add the oauth2-proxy volume - addSecretSourceVolume( - constants.Oauth2VolumeName, - "oauth2-proxy-"+suffix, - &patch.Spec.Template.Spec, - ) - - // Add the resource-attribute secret volume for the kube-rbac-proxy - addProjectedSecretSourceVolume( - constants.KubeRbacProxyVolumeName, - "resource-attributes-"+suffix, - &patch.Spec.Template.Spec, - ) - - // Add an optional kubeconfig secret for the kube-rbac-proxy - if shallAddKubeConfigSecretName(patch) { - addProjectedSecretSourceVolume( - constants.KubeRbacProxyVolumeName, - fetchKubconfigSecretName(suffix, patch), - &patch.Spec.Template.Spec, - ) - } - - // Add an optional oidc ca secret for the kube-rbac-proxy - if shallAddOidcCaSecretName(patch) { - addProjectedSecretSourceVolume( - constants.KubeRbacProxyVolumeName, - fetchOidcCASecretName(suffix, patch), - &patch.Spec.Template.Spec, - ) - } - - // Add OIDC Apps init container to deployment. - addInitContainer("oidc-init", &patch.Spec.Template.Spec, getInitContainer(issuerUrl)) - - // Add the OAUTH2 proxy sidecar to the pod template - addProxyContainer("oauth2-proxy", &patch.Spec.Template.Spec, getOIDCProxyContainer()) - - // Add the kube-rbac-proxy sidecar to the pod template - addProxyContainer("kube-rbac-proxy", &patch.Spec.Template.Spec, getKubeRbacProxyContainer(clientId, issuerUrl, upstreamUrl, patch)) - - // Add image pull secret if the proxy container images are served from private registry - if len(a.ImagePullSecret) > 0 { - addImagePullSecret(a.ImagePullSecret, &patch.Spec.Template.Spec) - } - - original, err := json.Marshal(deployment) - if err != nil { - _log.Info("Unable to marshal pod") - } - - patched, err := json.Marshal(patch) - if err != nil { - _log.Info("Unable to marshal pod") - } - return admission.PatchResponseFromRaw(original, patched) -} diff --git a/pkg/webhook/modifiers.go b/pkg/webhook/modifiers.go index 3c81ded..1f23a88 100644 --- a/pkg/webhook/modifiers.go +++ b/pkg/webhook/modifiers.go @@ -84,33 +84,33 @@ func get2ProxySecretChecksum(object client.Object) string { // Add gardener specific labels to the target pods. // Those are needed to construct the correct k8s network policies. // TODO: add configurable labels in the helm chart -func addPodLabels(object *corev1.PodTemplateSpec, lbls map[string]string) { - labels := object.GetLabels() +func addPodLabels(pod *corev1.Pod, lbls map[string]string) { + labels := pod.GetLabels() if len(labels) == 0 { labels = make(map[string]string, 2) } labels[constants.GardenerPublicLabelsKey] = "allowed" labels[constants.GardenerPrivateLabelsKey] = "allowed" if len(lbls) == 0 { - object.SetLabels(labels) + pod.SetLabels(labels) return } maps.Copy(labels, lbls) - object.SetLabels(labels) + pod.SetLabels(labels) } -func addPodAnnotations(object *corev1.PodTemplateSpec, ann map[string]string) { - annotations := object.GetAnnotations() +func addPodAnnotations(pod *corev1.Pod, ann map[string]string) { + annotations := pod.GetAnnotations() if len(annotations) == 0 { annotations = make(map[string]string, 1) } if len(ann) == 0 { - object.SetAnnotations(annotations) + pod.SetAnnotations(annotations) return } maps.Copy(annotations, ann) - object.SetAnnotations(annotations) + pod.SetAnnotations(annotations) } func addImagePullSecret(secretName string, podSpec *corev1.PodSpec) { @@ -357,10 +357,35 @@ func getInitContainer(oidcIssuerUrl string) corev1.Container { } } -func getKubeRbacProxyContainer(clientID, issuerUrl, upstream string, target client.Object) corev1.Container { +func getKubeRbacProxyContainer(clientID, issuerUrl, upstream string, pod *corev1.Pod) corev1.Container { image, _ := imagevector.ImageVector().FindImage("kube-rbac-proxy-watcher") + volumeMounts := []corev1.VolumeMount{ + { + Name: constants.KubeRbacProxyVolumeName, + ReadOnly: true, + MountPath: "/etc/kube-rbac-proxy", + }, + } + + // Add the service account token volume mount + for _, v := range pod.Spec.Volumes { + if v.Projected != nil && v.Projected.Sources != nil { + for _, s := range v.Projected.Sources { + if s.ServiceAccountToken != nil { + serviceAccountVolumeMount := corev1.VolumeMount{ + Name: v.Name, + ReadOnly: true, + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + } + volumeMounts = append(volumeMounts, serviceAccountVolumeMount) + break + } + } + } + } + container := corev1.Container{ Name: "kube-rbac-proxy", Image: image.String(), @@ -384,22 +409,16 @@ func getKubeRbacProxyContainer(clientID, issuerUrl, upstream string, target clie "memory": resource.MustParse("50Mi"), }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: constants.KubeRbacProxyVolumeName, - ReadOnly: true, - MountPath: "/etc/kube-rbac-proxy", - }, - }, + VolumeMounts: volumeMounts, } - if shallAddKubeConfigSecretName(target) { + if shallAddKubeConfigSecretName(pod) { // Add volume mount and start parameter if the secret name is provided container.Args = append(container.Args, "--kubeconfig=/etc/kube-rbac-proxy/kubeconfig") } // TODO: There is a bug https://github.com/brancz/kube-rbac-proxy/issues/259 - if shallAddOidcCaSecretName(target) { + if shallAddOidcCaSecretName(pod) { // Add volume mount and start parameter if the secret name is provided container.Args = append(container.Args, "--oidc-ca-file=/etc/kube-rbac-proxy/ca.crt") } diff --git a/pkg/webhook/pod-webhook.go b/pkg/webhook/pod-webhook.go index 1c11461..fe6743b 100644 --- a/pkg/webhook/pod-webhook.go +++ b/pkg/webhook/pod-webhook.go @@ -21,8 +21,10 @@ import ( "slices" "strings" - oidc_apps_controller "github.com/gardener/oidc-apps-controller/pkg/constants" + "github.com/gardener/oidc-apps-controller/pkg/configuration" + "github.com/gardener/oidc-apps-controller/pkg/constants" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/json" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,8 +38,9 @@ var _ admission.Handler = &PodMutator{} // PodMutator is a handler modifying the resource definitions of the pod targets type PodMutator struct { - Client client.Client - Decoder *webhook.AdmissionDecoder + Client client.Client + Decoder *webhook.AdmissionDecoder + ImagePullSecret string } // Handle provides interface implementation for the PodMutator @@ -62,41 +65,110 @@ func (p *PodMutator) Handle(ctx context.Context, req webhook.AdmissionRequest) w return webhook.Allowed("delete") } + // Simply return it the pod is not part of the described targets + + target, owner := isTarget(ctx, p.Client, pod) + if !target { + return webhook.Allowed("not a target") + } + patch := pod.DeepCopy() + clientId := configuration.GetOIDCAppsControllerConfig().GetClientID(owner) + issuerUrl := configuration.GetOIDCAppsControllerConfig().GetOidcIssuerUrl(owner) + upstream := configuration.GetOIDCAppsControllerConfig().GetUpstreamTarget(owner) + upstreamUrl := fetchUpstreamUrl(upstream, patch.Spec) + suffix := fetchTargetSuffix(owner) + + // Add the OIDC annotation to the deployment template + addAnnotations(patch) + + // Add required annotations to pod spec template + addPodAnnotations(patch, + map[string]string{ + constants.AnnotationHostKey: configuration.GetOIDCAppsControllerConfig().GetHost(owner), + }, + ) + + // Add required labels to pod spec template + addPodLabels(patch, + map[string]string{ + constants.LabelKey: "pod", + }, + ) + + // Add the oauth2-proxy volume + // TODO: handle scaling statefulset pods + addSecretSourceVolume( + constants.Oauth2VolumeName, + "oauth2-proxy-"+suffix, + &patch.Spec, + ) + + // Add the resource-attribute secret volume for the kube-rbac-proxy + addProjectedSecretSourceVolume( + constants.KubeRbacProxyVolumeName, + "resource-attributes-"+suffix, + &patch.Spec, + ) + + // Add an optional kubeconfig secret for the kube-rbac-proxy + if shallAddKubeConfigSecretName(patch) { + addProjectedSecretSourceVolume( + constants.KubeRbacProxyVolumeName, + fetchKubconfigSecretName(suffix, patch), + &patch.Spec, + ) + } - hostPrefix, ok := patch.GetAnnotations()[oidc_apps_controller.AnnotationHostKey] - if !ok { - return webhook.Errored(http.StatusBadRequest, fmt.Errorf("cannot find host annotation")) + // Add an optional oidc ca secret for the kube-rbac-proxy + if shallAddOidcCaSecretName(patch) { + addProjectedSecretSourceVolume( + constants.KubeRbacProxyVolumeName, + fetchOidcCASecretName(suffix, patch), + &patch.Spec, + ) } - host, domain, found := strings.Cut(hostPrefix, ".") - if found { - // In some envorinments, the pod index is added as a label: apps.kubernetes.io/pod-index - podIndex, present := patch.GetObjectMeta().GetLabels()["statefulset.kubernetes.io/pod-name"] - if present { - l := strings.Split(podIndex, "-") - host = fmt.Sprintf("%s-%s.%s", host, l[len(l)-1], domain) - } else { - host = fmt.Sprintf("%s.%s", host, domain) - } + // Add OIDC Apps init container to deployment. + addInitContainer("oidc-init", &patch.Spec, getInitContainer(issuerUrl)) + + // Add the OAUTH2 proxy sidecar to the pod template + addProxyContainer("oauth2-proxy", &patch.Spec, getOIDCProxyContainer()) + + // Add the kube-rbac-proxy sidecar to the pod template + addProxyContainer("kube-rbac-proxy", &patch.Spec, getKubeRbacProxyContainer(clientId, issuerUrl, upstreamUrl, patch)) + + // Add image pull secret if the proxy container images are served from private registry + if len(p.ImagePullSecret) > 0 { + addImagePullSecret(p.ImagePullSecret, &patch.Spec) } - _log.Info(fmt.Sprintf("host: %s", host)) - for idx, container := range patch.Spec.Containers { - if container.Name != "oauth2-proxy" { - continue + podIndex, present := patch.GetObjectMeta().GetLabels()["statefulset.kubernetes.io/pod-name"] + if present { + hostPrefix := configuration.GetOIDCAppsControllerConfig().GetHost(owner) + host, domain, found := strings.Cut(hostPrefix, ".") + if found { + l := strings.Split(podIndex, "-") + host = fmt.Sprintf("%s-%s.%s", host, l[len(l)-1], domain) } - // Remove the argument if present - for i, arg := range container.Args { - if strings.HasPrefix(arg, "--redirect-url") { - slices.Delete(patch.Spec.Containers[idx].Args, i, i+1) + _log.Info(fmt.Sprintf("host: %s", host)) + + for idx, container := range patch.Spec.Containers { + if container.Name != "oauth2-proxy" { + continue } + // Remove the argument if present + for i, arg := range container.Args { + if strings.HasPrefix(arg, "--redirect-url") { + slices.Delete(patch.Spec.Containers[idx].Args, i, i+1) + } + } + // Add the correct argument + patch.Spec.Containers[idx].Args = append(patch.Spec.Containers[idx].Args, + fmt.Sprintf("--redirect-url=https://%s/oauth2/callback", host), + ) + break } - // Add the correct argument - patch.Spec.Containers[idx].Args = append(patch.Spec.Containers[idx].Args, - fmt.Sprintf("--redirect-url=https://%s/oauth2/callback", host), - ) - break } original, err := json.Marshal(pod) @@ -110,3 +182,41 @@ func (p *PodMutator) Handle(ctx context.Context, req webhook.AdmissionRequest) w } return admission.PatchResponseFromRaw(original, patched) } + +func isTarget(ctx context.Context, c client.Client, pod *corev1.Pod) (bool, client.Object) { + //1. Identify the workload + owners := pod.GetOwnerReferences() + if len(owners) == 0 { + return false, nil + } + + for _, o := range owners { + if o.Kind == "StatefulSet" { + statefulset := &appsv1.StatefulSet{} + if err := c.Get(ctx, client.ObjectKey{Name: o.Name, Namespace: pod.GetNamespace()}, + statefulset); err != nil { + log.FromContext(ctx).Error(err, "unable to get statefulset for object", "object", pod) + return false, nil + } + return configuration.GetOIDCAppsControllerConfig().Match(statefulset), statefulset + } + + if o.Kind == "ReplicaSet" { + replicaset := &appsv1.ReplicaSet{} + if err := c.Get(ctx, client.ObjectKey{Name: o.Name, Namespace: pod.GetNamespace()}, replicaset); err != nil { + log.FromContext(ctx).Error(err, "unable to get replicaset for object", "object", pod) + return false, nil + } + deployment := &appsv1.Deployment{} + if err := c.Get(ctx, client.ObjectKey{Name: replicaset.GetOwnerReferences()[0].Name, + Namespace: pod.GetNamespace()}, + deployment); err != nil { + log.FromContext(ctx).Error(err, "unable to get deployment for object", "object", pod) + return false, nil + } + return configuration.GetOIDCAppsControllerConfig().Match(deployment), deployment + + } + } + return false, nil +} diff --git a/pkg/webhook/statefulset-webhook.go b/pkg/webhook/statefulset-webhook.go deleted file mode 100644 index e17e991..0000000 --- a/pkg/webhook/statefulset-webhook.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package webhook - -import ( - "context" - "fmt" - "net/http" - - "github.com/gardener/oidc-apps-controller/pkg/configuration" - "github.com/gardener/oidc-apps-controller/pkg/constants" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/util/json" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// Register the webhook with the server -var _ admission.Handler = &StatefulSetMutator{} - -// StatefulSetMutator is a handler modifying the resource definitions of the statefulset targets -type StatefulSetMutator struct { - Client client.Client - Decoder *webhook.AdmissionDecoder - ImagePullSecret string -} - -// Handle provides interface implementation for the StatefulSetMutator -func (s *StatefulSetMutator) Handle(ctx context.Context, req webhook.AdmissionRequest) webhook.AdmissionResponse { - _log := log.FromContext(ctx) - - if s.Decoder == nil { - return webhook.Errored(http.StatusInternalServerError, - fmt.Errorf("decoder in the admission handler cannot be nil")) - } - - statefulset := &appsv1.StatefulSet{} - if err := s.Decoder.Decode(req, statefulset); err != nil { - return webhook.Errored(http.StatusBadRequest, err) - } - - _log.V(9).Info("handling admission request", "resourceVersion", statefulset.GetResourceVersion(), - "generation", statefulset.GetGeneration()) - - // Simply return it the deployment is not part of the described targets - if !configuration.GetOIDCAppsControllerConfig().Match(statefulset) { - return webhook.Allowed("not a target") - } - - // Simply return if it is a delete operation - if !statefulset.GetDeletionTimestamp().IsZero() { - return webhook.Allowed("delete") - } - - patch := statefulset.DeepCopy() - clientId := configuration.GetOIDCAppsControllerConfig().GetClientID(patch) - issuerUrl := configuration.GetOIDCAppsControllerConfig().GetOidcIssuerUrl(patch) - target := configuration.GetOIDCAppsControllerConfig().GetUpstreamTarget(patch) - upstreamUrl := fetchUpstreamUrl(target, patch.Spec.Template.Spec) - suffix := fetchTargetSuffix(patch) - - // Add the OIDC annotation to the deployment template - addAnnotations(patch) - - // Add required annotations to pod spec template - addPodAnnotations(&patch.Spec.Template, - map[string]string{ - constants.AnnotationHostKey: configuration.GetOIDCAppsControllerConfig().GetHost(patch), - }, - ) - - // Add required labels to pod spec template - addPodLabels(&patch.Spec.Template, - map[string]string{ - constants.LabelKey: "pod", - }, - ) - - // Add the oauth2-proxy volume - // TODO: handle scaling statefulset pods - addSecretSourceVolume( - constants.Oauth2VolumeName, - "oauth2-proxy-"+suffix, - &patch.Spec.Template.Spec, - ) - - // Add the resource-attribute secret volume for the kube-rbac-proxy - addProjectedSecretSourceVolume( - constants.KubeRbacProxyVolumeName, - "resource-attributes-"+suffix, - &patch.Spec.Template.Spec, - ) - - // Add an optional kubeconfig secret for the kube-rbac-proxy - if shallAddKubeConfigSecretName(patch) { - addProjectedSecretSourceVolume( - constants.KubeRbacProxyVolumeName, - fetchKubconfigSecretName(suffix, patch), - &patch.Spec.Template.Spec, - ) - } - - // Add an optional oidc ca secret for the kube-rbac-proxy - if shallAddOidcCaSecretName(patch) { - addProjectedSecretSourceVolume( - constants.KubeRbacProxyVolumeName, - fetchOidcCASecretName(suffix, patch), - &patch.Spec.Template.Spec, - ) - } - - // Add OIDC Apps init container to deployment. - addInitContainer("oidc-init", &patch.Spec.Template.Spec, getInitContainer(issuerUrl)) - - // Add the OAUTH2 proxy sidecar to the pod template - addProxyContainer("oauth2-proxy", &patch.Spec.Template.Spec, getOIDCProxyContainer()) - - // Add the kube-rbac-proxy sidecar to the pod template - addProxyContainer("kube-rbac-proxy", &patch.Spec.Template.Spec, getKubeRbacProxyContainer(clientId, issuerUrl, upstreamUrl, patch)) - - // Add image pull secret if the proxy container images are served from private registry - if len(s.ImagePullSecret) > 0 { - addImagePullSecret(s.ImagePullSecret, &patch.Spec.Template.Spec) - } - - original, err := json.Marshal(statefulset) - if err != nil { - _log.Info("Unable to marshal pod") - } - - patched, err := json.Marshal(patch) - if err != nil { - _log.Info("Unable to marshal pod") - } - - return admission.PatchResponseFromRaw(original, patched) -} diff --git a/pkg/webhook/test/deployment_webhook_test.go b/pkg/webhook/test/deployment_webhook_test.go deleted file mode 100644 index d758399..0000000 --- a/pkg/webhook/test/deployment_webhook_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "context" - _ "embed" - "os" - "path/filepath" - "strings" - "time" - - "github.com/gardener/oidc-apps-controller/pkg/configuration" - "github.com/gardener/oidc-apps-controller/pkg/webhook" - - jsonpatch "github.com/evanphx/json-patch" - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - adminssionv1 "k8s.io/api/admission/v1" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/json" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -var tmpDir string - -//go:embed config.yaml -var configFile string - -var _log = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) - -var _ = BeforeSuite(func() { - tmpDir = GinkgoT().TempDir() - err := os.WriteFile(filepath.Join(tmpDir, "config.yaml"), []byte(configFile), 0444) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, tmpDir) -}) - -var _ = Describe("Oidc Apps Deployment MutatingWebhook Framework Test", func() { - var ( - ctx context.Context - cancel context.CancelFunc - fakeClient client.Client - decoder *admission.Decoder - ) - - // Initialize the test environment - BeforeEach(func() { - - ctx, cancel = context.WithTimeout( - logr.NewContext(context.Background(), _log), - 30*time.Second, - ) - - s := runtime.NewScheme() - fakeClient = fake.NewClientBuilder().WithScheme(s).Build() - decoder = admission.NewDecoder(runtime.NewScheme()) - configuration.CreateControllerConfigOrDie( - filepath.Join(tmpDir, "config.yaml"), - configuration.WithClient(fakeClient), - configuration.WithLog(_log), - ) - }) - - AfterEach(func() { - cancel() - }) - Context("when the deployment is not a target", func() { - - deployment := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, - ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: "default"}, - Spec: appsv1.DeploymentSpec{}, - } - rawDeployment, err := json.Marshal(deployment) - Expect(err).NotTo(HaveOccurred()) - - It("should allow the admission request with nil patch", func() { - // Create a Deployment object - - mutator := webhook.DeploymentMutator{ - Client: fakeClient, - Decoder: decoder, - } - req := admission.Request{ - AdmissionRequest: adminssionv1.AdmissionRequest{ - UID: "uid-request", - Kind: metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, - Resource: metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, - Namespace: "default", - Operation: adminssionv1.Create, - Object: runtime.RawExtension{ - Raw: rawDeployment, - }, - }, - } - resp := mutator.Handle(ctx, req) - Expect(resp.Allowed).To(BeTrue()) - Expect(resp.Patch).To(BeNil()) - Expect(resp.PatchType).To(BeNil()) - }) - }) - Context("when the deployment is a target", func() { - deployment := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, - ObjectMeta: metav1.ObjectMeta{Name: "nginx", Namespace: "nginx", Labels: map[string]string{"app": "nginx"}}, - Spec: appsv1.DeploymentSpec{}, - } - rawDeployment, err := json.Marshal(deployment) - Expect(err).NotTo(HaveOccurred()) - - It("should allow the admission request with patch not being nil", func() { - // Create a Deployment object - - mutator := webhook.DeploymentMutator{ - Client: fakeClient, - Decoder: decoder, - } - req := admission.Request{ - AdmissionRequest: adminssionv1.AdmissionRequest{ - UID: "uid-request", - Kind: metav1.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, - Resource: metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, - Namespace: "default", - Operation: adminssionv1.Create, - Object: runtime.RawExtension{ - Raw: rawDeployment, - }, - }, - } - resp := mutator.Handle(ctx, req) - _log.Info("resp", "resp", resp) - Expect(resp.Allowed).To(BeTrue()) - Expect(resp.Patches).ToNot(BeNil()) - Expect(*resp.PatchType).To(BeEquivalentTo(adminssionv1.PatchTypeJSONPatch)) - - By("verifying patched deployment", func() { - patchBytes, err := json.Marshal(resp.Patches) - Expect(err).NotTo(HaveOccurred()) - decodedPatch, err := jsonpatch.DecodePatch(patchBytes) - Expect(err).NotTo(HaveOccurred()) - - // Apply the patch - patchDeploymentBytes, err := decodedPatch.Apply(rawDeployment) - Expect(err).NotTo(HaveOccurred()) - - // Deserialize the patched Deployment back into a struct - var modifiedDeployment appsv1.Deployment - err = json.Unmarshal(patchDeploymentBytes, &modifiedDeployment) - Expect(err).NotTo(HaveOccurred()) - _log.Info("modifiedDeployment", "modifiedDeployment", modifiedDeployment) - By("verifying containers", func() { - expectedContainerImages := []string{"kube-rbac-proxy-watcher", "oauth2-proxy"} - - Expect(len(modifiedDeployment.Spec.Template.Spec.Containers)).To(Equal(2)) - - for _, c := range modifiedDeployment.Spec.Template.Spec.Containers { - image, _, ok := strings.Cut(c.Image, ":") - Expect(ok).To(BeTrue()) - n := strings.SplitAfter(image, "/") - Expect(n[len(n)-1]).To(BeElementOf(expectedContainerImages)) - } - }) - By("verifying init container", func() { - Expect(len(modifiedDeployment.Spec.Template.Spec.InitContainers)).To(Equal(1)) - Expect(modifiedDeployment.Spec.Template.Spec.InitContainers[0].Image).To(ContainSubstring("curl")) - }) - }) - }) - }) -}) diff --git a/pkg/webhook/test/mutating-webhook_test.go b/pkg/webhook/test/mutating-webhook_test.go new file mode 100644 index 0000000..ea4eab2 --- /dev/null +++ b/pkg/webhook/test/mutating-webhook_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + _ "embed" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Oidc Apps MutatingAdmission Framework Test", func() { + Context("when a pod belongs to a target", func() { + It("there shall be a auth & authz proxies in the pod templates spec", func() {}) + When("the pod is created with a service account with token", func() { + It("there shall be a volume mount in the pod template spec", func() {}) + }) + When("there is a kubeconfig secret", func() { + It("there shall be a kubeconfig volume in the pod templates spec", func() {}) + }) + }) + Context("when a pod does not belong to a target", func() { + It("there shall be no ", func() {}) + }) +}) diff --git a/pkg/webhook/test/webhook_suite_test.go b/pkg/webhook/test/oisc-apps_suite_test.go similarity index 68% rename from pkg/webhook/test/webhook_suite_test.go rename to pkg/webhook/test/oisc-apps_suite_test.go index e757ebd..c636499 100644 --- a/pkg/webhook/test/webhook_suite_test.go +++ b/pkg/webhook/test/oisc-apps_suite_test.go @@ -15,12 +15,30 @@ package test import ( + _ "embed" + "os" + "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/log/zap" ) +var tmpDir string + +//go:embed config.yaml +var configFile string + +var _log = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) + +var _ = BeforeSuite(func() { + tmpDir = GinkgoT().TempDir() + err := os.WriteFile(filepath.Join(tmpDir, "config.yaml"), []byte(configFile), 0444) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, tmpDir) +}) + func TestOidcApps(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "OIDC Apps Webhook Suite") diff --git a/test/e2e/deployment_test.go b/test/e2e/deployment_test.go new file mode 100644 index 0000000..b89032e --- /dev/null +++ b/test/e2e/deployment_test.go @@ -0,0 +1,56 @@ +// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + _ "embed" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Oidc Apps Deployment Framework Test", func() { + var ( + _ context.Context + cancel context.CancelFunc + ) + + // Initialize the test environment + BeforeEach(func() { + _, cancel = context.WithTimeout( + logr.NewContext(context.Background(), _log), + 30*time.Second, + ) + }) + + AfterEach(func() { + cancel() + }) + Context("when a deployment is a target", func() { + It("there shall be auth & autz proxies present in the deployment", func() {}) + It("there shall be ingress present in the deployment namespace", func() {}) + It("there shall be service present in the deployment namespace", func() {}) + It("there shall be outh2 and kube-rbac secrets present in the deployment namespace", func() {}) + When("the deployment is deleted", func() { + It("there shall be no auth & autz proxies present in the deployment", func() {}) + }) + }) + Context("when a deployment is not a target", func() { + It("there shall be no auth & autz proxies present in the deployment", func() {}) + }) + +}) diff --git a/test/e2e/deployment_webhook.go b/test/e2e/deployment_webhook.go deleted file mode 100644 index 5f1e8a2..0000000 --- a/test/e2e/deployment_webhook.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "context" - "crypto/tls" - _ "embed" - "github.com/gardener/oidc-apps-controller/pkg/configuration" - "github.com/gardener/oidc-apps-controller/pkg/constants" - "github.com/gardener/oidc-apps-controller/pkg/controllers" - "github.com/gardener/oidc-apps-controller/pkg/webhook" - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "os" - "path/filepath" - controllerruntime "sigs.k8s.io/controller-runtime" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - . "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/handler" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager" - ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "strings" - "time" -) - -//go:embed config.yaml -var configFile string - -var ( - env *Environment - mgr controllerruntime.Manager - client ctrlclient.Client - _log logr.Logger - tmpDir string - deployment *appsv1.Deployment - cancel context.CancelFunc -) - -var _ = BeforeSuite(func() { - var err error - // Initialize logging - _log = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) - logf.SetLogger(_log) - - //Initialize test environment - env = &Environment{} - installWebHooks(env) - - Expect(env.Start()).NotTo(BeNil()) - - //Initialize controller-runtime manager - mgr, err = manager.New(env.Config, manager.Options{ - WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{ - Port: env.WebhookInstallOptions.LocalServingPort, - Host: env.WebhookInstallOptions.LocalServingHost, - CertDir: env.WebhookInstallOptions.LocalServingCertDir, - TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, - }), - }) - Expect(err).NotTo(HaveOccurred()) - server := mgr.GetWebhookServer() - client, err = ctrlclient.New(env.Config, ctrlclient.Options{}) - Expect(err).NotTo(HaveOccurred()) - server.Register( - constants.DeploymentWebHookPath, - &ctrlwebhook.Admission{Handler: &webhook.DeploymentMutator{ - Client: client, - Decoder: admission.NewDecoder(runtime.NewScheme()), - ImagePullSecret: "", - }}, - ) - - // Initialize oidc-apps deployment reconciler - err = controllerruntime.NewControllerManagedBy(mgr). - Named("oidc-apps-deployments"). - For(&appsv1.Deployment{}). - Watches( - &corev1.Secret{}, - handler.EnqueueRequestForOwner( - mgr.GetScheme(), - mgr.GetRESTMapper(), - &appsv1.Deployment{}, - ), - ). - Watches( - &corev1.Service{}, - handler.EnqueueRequestForOwner( - mgr.GetScheme(), - mgr.GetRESTMapper(), - &appsv1.Deployment{}, - ), - ). - Watches( - &networkingv1.Ingress{}, - handler.EnqueueRequestForOwner( - mgr.GetScheme(), - mgr.GetRESTMapper(), - &appsv1.Deployment{}, - ), - ). - Complete(&controllers.DeploymentReconciler{Client: mgr.GetClient()}) - Expect(err).NotTo(HaveOccurred()) - - // Start manager - var ctx context.Context - ctx, cancel = context.WithCancel(context.Background()) - - go func() { - Expect(mgr.Start(ctx)).NotTo(HaveOccurred()) - }() - -}) - -var _ = AfterSuite(func() { - cancel() - Expect(env.Stop()).NotTo(HaveOccurred()) - Expect(os.RemoveAll(tmpDir)).NotTo(HaveOccurred()) -}) - -var _ = Describe("Deployment target", func() { - - BeforeEach(func() { - Expect(env.Config).NotTo(BeNil()) - }) - - It("A deployment matching target labelSelectors shall be enhanced with the auth & autz proxies", func() { - // Create oidc-apps controller config - tmpDir = GinkgoT().TempDir() - Expect( - os.WriteFile(filepath.Join(tmpDir, "config.yaml"), []byte(configFile), 0444), - ).NotTo(HaveOccurred()) - - configuration.CreateControllerConfigOrDie( - filepath.Join(tmpDir, "config.yaml"), - configuration.WithClient(client), - configuration.WithLog(_log), - ) - - deployment = createDeployment() - - Eventually(func() error { - return client.Create(context.TODO(), deployment) - }, 1*time.Second).ShouldNot(HaveOccurred()) - - Expect( - client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "nginx"}, deployment), - ).NotTo(HaveOccurred()) - - By("verifying containers", func() { - expectedContainerImages := []string{"kube-rbac-proxy-watcher", "oauth2-proxy", "nginx"} - - Expect(len(deployment.Spec.Template.Spec.Containers)).To(Equal(3)) - - for _, c := range deployment.Spec.Template.Spec.Containers { - image, _, ok := strings.Cut(c.Image, ":") - Expect(ok).To(BeTrue()) - n := strings.SplitAfter(image, "/") - Expect(n[len(n)-1]).To(BeElementOf(expectedContainerImages)) - } - }) - By("verifying init container", func() { - Expect(len(deployment.Spec.Template.Spec.InitContainers)).To(Equal(1)) - Expect(deployment.Spec.Template.Spec.InitContainers[0].Image).To(ContainSubstring("curl")) - }) - - By("verifying secrets", func() { - secretList := &corev1.SecretList{} - Eventually(func() bool { - err := client.List(context.TODO(), secretList, &ctrlclient.ListOptions{Namespace: "default"}) - return err == nil && len(secretList.Items) > 0 - }, 10*time.Second, 1*time.Second).Should(BeTrue()) - - Expect(len(secretList.Items)).To(Equal(2)) - expectedSecretNames := []string{"oauth2-proxy", "resource-attributes"} - for _, s := range secretList.Items { - suffix, found := deployment.ObjectMeta.Annotations[constants.AnnotationSuffixKey] - Expect(found).To(BeTrue()) - Expect(strings.TrimSuffix(s.Name, "-"+suffix)).To(BeElementOf(expectedSecretNames)) - } - }) - }) -}) diff --git a/test/e2e/statefulset_test.go b/test/e2e/statefulset_test.go new file mode 100644 index 0000000..d1b15b8 --- /dev/null +++ b/test/e2e/statefulset_test.go @@ -0,0 +1,63 @@ +// Copyright 2024 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + _ "embed" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Oidc Apps StatefulSets Framework Test", func() { + + var ( + _ context.Context + cancel context.CancelFunc + ) + + // Initialize the test environment + BeforeEach(func() { + _, cancel = context.WithTimeout( + logr.NewContext(context.Background(), _log), + 30*time.Second, + ) + + }) + + AfterEach(func() { + cancel() + }) + Context("when a statefulset is a target", func() { + It("there shall be auth & autz proxies present in the statefulset", func() {}) + + When("the statefulset is created with two pods", func() { + It("there shall be an ingress and service for each pod", func() {}) + }) + It("there shall be oauth2 and kube-rbac secrets present in the statefulset namespace", func() {}) + When("the statefulset is scaled to 0", func() { + It("there shall be no ingress & services for pods present in the statefulset", func() {}) + }) + When("the statefulset is deleted", func() { + It("there shall be no auth & autz proxies present in the statefulset", func() {}) + }) + }) + Context("when a statefulset is not a target", func() { + It("there shall be no auth & autz proxies present in the statefulset", func() {}) + }) + +}) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index eaf8d32..3389a28 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -15,6 +15,10 @@ package e2e import ( + _ "embed" + "os" + "path/filepath" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "testing" . "github.com/onsi/ginkgo/v2" @@ -25,3 +29,15 @@ func TestSute(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "E2E Suite") } + +//go:embed config.yaml +var configFile string + +var _log = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) + +var _ = BeforeSuite(func() { + tmpDir := GinkgoT().TempDir() + err := os.WriteFile(filepath.Join(tmpDir, "config.yaml"), []byte(configFile), 0444) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, tmpDir) +})