Remove usage of kube-rbac-proxy and switch to the built-in `WithAut…

…henticationAndAuthorization` filter instead (#338) * Add test highlighting the expectations * Remove all usage of `kube-rbac-proxy` * Protect the metrics endpoint using the built-in `WithAuthenticationAndAuthorization` filter This is enabled only if `--metrics-over-https` is enabled, which is done to avoid passing service account tokens over HTTP. Metrics serving over HTTPS is disabled by default (when running the operator locally), but enabled when deployed either with or without OLM. Ref: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/metrics/filters#WithAuthenticationAndAuthorization * Fix E2E test
redhat-developer · Oct 17, 2024 · 823ae0d · 823ae0d
1 parent 7c947c9
commit 823ae0d
Show file tree

Hide file tree

Showing 25 changed files with 421 additions and 382 deletions.
diff --git a/.rhdh/bundle/manifests/rhdh-operator.clusterserviceversion.yaml b/.rhdh/bundle/manifests/rhdh-operator.clusterserviceversion.yaml
@@ -222,32 +222,10 @@ spec:
                         - linux
               automountServiceAccountToken: true
               containers:
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: registry.redhat.io/openshift4/ose-kube-rbac-proxy-rhel9:v4.16
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               - args:
                 - --health-probe-bind-address=:8081
-                - --metrics-bind-address=127.0.0.1:8080
+                - --metrics-bind-address=:8443
+                - --metrics-secure=true
                 - --leader-elect
                 command:
                 - /manager
@@ -263,17 +241,22 @@ spec:
                 - name: RELATED_IMAGE_backstage
                   value: registry.redhat.io/rhdh/rhdh-hub-rhel9:1.4
                 image: registry.redhat.io/rhdh/rhdh-rhel9-operator:1.4
+                ports:
+                  - name: health
+                    containerPort: 8081
+                  - name: metrics
+                    containerPort: 8443
                 livenessProbe:
                   httpGet:
                     path: /healthz
-                    port: 8081
+                    port: health
                   initialDelaySeconds: 15
                   periodSeconds: 20
                 name: manager
                 readinessProbe:
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: health
                   initialDelaySeconds: 5
                   periodSeconds: 10
                 resources:

diff --git a/bundle/backstage.io/manifests/backstage-controller-manager-metrics-service_v1_service.yaml b/bundle/backstage.io/manifests/backstage-controller-manager-metrics-service_v1_service.yaml
@@ -3,7 +3,7 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics
     app.kubernetes.io/created-by: backstage-operator
     app.kubernetes.io/instance: controller-manager-metrics-service
     app.kubernetes.io/managed-by: kustomize
@@ -13,10 +13,10 @@ metadata:
   name: backstage-controller-manager-metrics-service
 spec:
   ports:
-  - name: https
+  - name: metrics
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: metrics
   selector:
     control-plane: controller-manager
 status:

diff --git a/...stage.io/manifests/backstage-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml b/...stage.io/manifests/backstage-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml
@@ -3,7 +3,7 @@ kind: ClusterRole
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics-rbac
     app.kubernetes.io/created-by: backstage-operator
     app.kubernetes.io/instance: metrics-reader
     app.kubernetes.io/managed-by: kustomize

diff --git a/bundle/backstage.io/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/backstage.io/manifests/backstage-operator.clusterserviceversion.yaml
@@ -25,7 +25,7 @@ metadata:
           }
         }
       ]
-    createdAt: "2024-10-14T11:21:00Z"
+    createdAt: "2024-10-16T20:54:13Z"
     description: Backstage Operator
     operators.operatorframework.io/builder: operator-sdk-v1.36.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -172,56 +172,36 @@ spec:
                         operator: In
                         values:
                         - amd64
-                        - arm64
-                        - ppc64le
-                        - s390x
                       - key: kubernetes.io/os
                         operator: In
                         values:
                         - linux
               automountServiceAccountToken: true
               containers:
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               - args:
                 - --health-probe-bind-address=:8081
-                - --metrics-bind-address=127.0.0.1:8080
+                - --metrics-bind-address=:8443
+                - --metrics-secure=true
                 - --leader-elect
                 command:
                 - /manager
                 image: quay.io/rhdh-community/operator:0.4.0
                 livenessProbe:
                   httpGet:
                     path: /healthz
-                    port: 8081
+                    port: health
                   initialDelaySeconds: 15
                   periodSeconds: 20
                 name: manager
+                ports:
+                - containerPort: 8081
+                  name: health
+                - containerPort: 8443
+                  name: metrics
                 readinessProbe:
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: health
                   initialDelaySeconds: 5
                   periodSeconds: 10
                 resources:

diff --git a/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/rhdh/manifests/backstage-operator.clusterserviceversion.yaml
@@ -29,7 +29,7 @@ metadata:
     categories: Developer Tools
     certified: "true"
     containerImage: registry-proxy.engineering.redhat.com/rh-osbs/rhdh-rhdh-rhel9-operator:1.3
-    createdAt: "2024-10-16T16:51:27Z"
+    createdAt: "2024-10-16T20:54:18Z"
     description: Red Hat Developer Hub is a Red Hat supported version of Backstage.
       It comes with pre-built plug-ins and configuration settings, supports use of
       an external database, and can help streamline the process of setting up a self-managed
@@ -211,56 +211,36 @@ spec:
                         operator: In
                         values:
                         - amd64
-                        - arm64
-                        - ppc64le
-                        - s390x
                       - key: kubernetes.io/os
                         operator: In
                         values:
                         - linux
               automountServiceAccountToken: true
               containers:
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               - args:
                 - --health-probe-bind-address=:8081
-                - --metrics-bind-address=127.0.0.1:8080
+                - --metrics-bind-address=:8443
+                - --metrics-secure=true
                 - --leader-elect
                 command:
                 - /manager
                 image: quay.io/rhdh-community/operator:0.4.0
                 livenessProbe:
                   httpGet:
                     path: /healthz
-                    port: 8081
+                    port: health
                   initialDelaySeconds: 15
                   periodSeconds: 20
                 name: manager
+                ports:
+                - containerPort: 8081
+                  name: health
+                - containerPort: 8443
+                  name: metrics
                 readinessProbe:
                   httpGet:
                     path: /readyz
-                    port: 8081
+                    port: health
                   initialDelaySeconds: 5
                   periodSeconds: 10
                 resources:

diff --git a/bundle/rhdh/manifests/rhdh-controller-manager-metrics-service_v1_service.yaml b/bundle/rhdh/manifests/rhdh-controller-manager-metrics-service_v1_service.yaml
@@ -3,7 +3,7 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics
     app.kubernetes.io/created-by: backstage-operator
     app.kubernetes.io/instance: controller-manager-metrics-service
     app.kubernetes.io/managed-by: kustomize
@@ -13,10 +13,10 @@ metadata:
   name: rhdh-controller-manager-metrics-service
 spec:
   ports:
-  - name: https
+  - name: metrics
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: metrics
   selector:
     control-plane: controller-manager
 status:

diff --git a/bundle/rhdh/manifests/rhdh-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml b/bundle/rhdh/manifests/rhdh-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml
@@ -3,7 +3,7 @@ kind: ClusterRole
 metadata:
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: kube-rbac-proxy
+    app.kubernetes.io/component: metrics-rbac
     app.kubernetes.io/created-by: backstage-operator
     app.kubernetes.io/instance: metrics-reader
     app.kubernetes.io/managed-by: kustomize

diff --git a/config/profile/backstage.io/kustomization.yaml b/config/profile/backstage.io/kustomization.yaml
@@ -16,16 +16,6 @@ resources:
 - ../../rbac
 - manager.yaml
 
-patches:
-- path: manager_auth_proxy_patch.yaml
-  target:
-    group: apps
-    kind: Deployment
-    name: controller-manager
-    version: v1
-#      name: controller-manager
-#      namespace: system
-
 images:
 - name: controller
   newName: quay.io/rhdh-community/operator

diff --git a/config/profile/backstage.io/manager.yaml b/config/profile/backstage.io/manager.yaml
@@ -38,26 +38,26 @@ spec:
     spec:
       # Required because the operator does not work without a Service Account Token
       automountServiceAccountToken: true # NOSONAR
-      # (user): Uncomment the following code to configure the nodeAffinity expression
+      # Configure the nodeAffinity expression
       # according to the platforms which are supported by your solution. 
       # It is considered best practice to support multiple architectures. You can
       # build your manager image using the makefile target docker-buildx.
-      # affinity:
-      #   nodeAffinity:
-      #     requiredDuringSchedulingIgnoredDuringExecution:
-      #       nodeSelectorTerms:
-      #         - matchExpressions:
-      #           - key: kubernetes.io/arch
-      #             operator: In
-      #             values:
-      #               - amd64
-      #               - arm64
-      #               - ppc64le
-      #               - s390x
-      #           - key: kubernetes.io/os
-      #             operator: In
-      #             values:
-      #               - linux
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: kubernetes.io/arch
+                  operator: In
+                  values:
+                    - amd64
+#                    - arm64
+#                    - ppc64le
+#                    - s390x
+                - key: kubernetes.io/os
+                  operator: In
+                  values:
+                    - linux
       securityContext:
         runAsNonRoot: true
         # (user): For common cases that do not require escalating privileges
@@ -71,24 +71,32 @@ spec:
       - command:
         - /manager
         args:
-        - --leader-elect
+          - --health-probe-bind-address=:8081
+          - --metrics-bind-address=:8443
+          - --metrics-secure=true
+          - --leader-elect
         image: controller:latest
         name: manager
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
               - "ALL"
+        ports:
+          - name: health
+            containerPort: 8081
+          - name: metrics
+            containerPort: 8443
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: health
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: health
           initialDelaySeconds: 5
           periodSeconds: 10
         resources: