diff --git a/.github/workflows/test-sample-go.yml b/.github/workflows/test-sample-go.yml index 08e1c9a1984..debc8f909b3 100644 --- a/.github/workflows/test-sample-go.yml +++ b/.github/workflows/test-sample-go.yml @@ -26,7 +26,7 @@ jobs: sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '42s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '46,143s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '46,142s/^#//' $KUSTOMIZATION_FILE_PATH - name: Test run: | diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml index e0e588792cf..a90721e62b2 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 74c49152afb..00000000000 --- a/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml index 893610e2014..91d41742932 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index ac8e7be7bc9..00000000000 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 17e0a11d32b..00000000000 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index e1f50c3178a..00000000000 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml index 9f6506d4c5b..20b2e1d12aa 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml @@ -9,16 +9,11 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines # if you do not want those helpers be installed with your Project. - projectconfig_editor_role.yaml - projectconfig_viewer_role.yaml + diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_service.yaml similarity index 85% rename from docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml rename to docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_service.yaml index aff147e644a..1cb008b3b59 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index e445fec445d..9cd07c6181b 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: - ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 1064aa49c80..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/manager/manager.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/manager/manager.yaml index 839f4b67565..c51cb2471d6 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/manager/manager.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager securityContext: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml index 893610e2014..91d41742932 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index ac8e7be7bc9..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 17e0a11d32b..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index e1f50c3178a..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml index 8db606e9e72..09d2ee4d606 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml @@ -9,16 +9,11 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines # if you do not want those helpers be installed with your Project. - cronjob_editor_role.yaml - cronjob_viewer_role.yaml + diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_service.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml similarity index 85% rename from docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_service.yaml rename to docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml index aff147e644a..1cb008b3b59 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_service.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index d851be9cae7..3e7e6da4538 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/docs/book/src/getting-started/testdata/project/config/manager/manager.yaml b/docs/book/src/getting-started/testdata/project/config/manager/manager.yaml index 60fe260a2d6..602974cc5fc 100644 --- a/docs/book/src/getting-started/testdata/project/config/manager/manager.yaml +++ b/docs/book/src/getting-started/testdata/project/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager env: diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml index 893610e2014..91d41742932 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index ac8e7be7bc9..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 17e0a11d32b..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index e1f50c3178a..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml index 3dc289427b8..a8f1075285b 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml @@ -9,16 +9,11 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines # if you do not want those helpers be installed with your Project. - memcached_editor_role.yaml - memcached_viewer_role.yaml + diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_service.yaml similarity index 85% rename from docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml rename to docs/book/src/getting-started/testdata/project/config/rbac/metrics_service.yaml index aff147e644a..1cb008b3b59 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index 33e7e3b0a13..d80d7829af2 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -3,45 +3,204 @@ By default, controller-runtime builds a global prometheus registry and publishes [a collection of performance metrics](/reference/metrics-reference.md) for each controller. + + +## Enabling the Metrics + +First, you will need enable the Metrics by uncommenting the following line +in the file `config/default/kustomization.yaml`, see: + +```sh +# [Metrics] The following patch will enable the metrics endpoint. +# Ensure that you also protect this endpoint. +#- path: manager_metrics_patch.yaml +``` + ## Protecting the Metrics -These metrics are protected by [kube-rbac-proxy](https://github.com/brancz/kube-rbac-proxy) -by default if using kubebuilder. Kubebuilder v2.2.0+ scaffold a clusterrole which -can be found at `config/rbac/auth_proxy_client_clusterrole.yaml`. +Unprotected metrics endpoints can expose valuable data to unauthorized users, +such as system performance, application behavior, and potentially confidential +operational metrics. This exposure can lead to security vulnerabilities +where an attacker could gain insights into the system's operation +and exploit weaknesses. + +### By using Network Policy -You will need to grant permissions to your Prometheus server so that it can -scrape the protected metrics. To achieve that, you can create a -`clusterRoleBinding` to bind the `clusterRole` to the service account that your -Prometheus server uses. If you are using [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus), -this cluster binding already exists. +NetworkPolicy acts as a basic firewall for pods within a Kubernetes cluster, controlling traffic +flow at the IP address or port level. However, it doesn't handle authentication (authn), authorization (authz), +or encryption directly like [kube-rbac-proxy](https://github.com/brancz/kube-rbac-proxy) solution. -You can either run the following command, or apply the example yaml file provided below to create `clusterRoleBinding`. +### By exposing the metrics endpoint using HTTPS and CertManager -If using kubebuilder -`` is the `namePrefix` field in `config/default/kustomization.yaml`. +Integrating `cert-manager` with your metrics service can secure the endpoint via TLS encryption. -```bash -kubectl create clusterrolebinding metrics --clusterrole=-metrics-reader --serviceaccount=: +To modify your project setup to expose metrics using HTTPS with +the help of cert-manager, you'll need to change the configuration of both +the `Service` and the `ServiceMonitor` to use a secure HTTPS port +and ensure the necessary certificate is applied. + +**Example** + +- Modify the `Service` under `config/rbac/metrics_service.yaml` to expose HTTPS: + +```ymalapiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: my-project + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager ``` -You can also apply the following `ClusterRoleBinding`: +- You will need to configure `cert-manager` to issue a certificate for the metrics +service which would like: ```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: - name: prometheus-k8s-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus-k8s-role -subjects: - - kind: ServiceAccount - name: - namespace: + name: controller-manager-metrics-certificate + namespace: system +spec: + secretName: controller-manager-metrics-cert + issuerRef: + name: letsencrypt-prod + kind: Issuer + dnsNames: + - "controller-manager-metrics-service.system.svc.cluster.local" ``` -The `prometheus-k8s-role` referenced here should provide the necessary permissions to allow prometheus scrape metrics from operator pods. +**Note**: Make sure that issuerRef refers to a valid Issuer or ClusterIssuer +that you have configured in your cluster. + +- Modify the `ServiceMonitor` under `config/prometheus/monitor.yaml` +to use HTTPS, such as: + +``` +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: my-project + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + tlsConfig: + caFile: /etc/prometheus/secrets/controller-manager-metrics-cert/ca.crt + certFile: /etc/prometheus/secrets/controller-manager-metrics-cert/tls.crt + keyFile: /etc/prometheus/secrets/controller-manager-metrics-cert/tls.key + serverName: "controller-manager-metrics-service.system.svc.cluster.local" + insecureSkipVerify: false + selector: + matchLabels: + control-plane: controller-manager +``` + +- Update your Manager config under `config/manager/manager.yaml` +to mount the TLS certificates: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + ... +spec: + ... + spec: + containers: + - name: manager + args: + - "--leader-elect" + - "--cert-dir=/etc/certs" + image: controller:latest + ports: + - containerPort: 8443 + name: https + volumeMounts: + - name: cert-volume + mountPath: /etc/certs + readOnly: true + volumes: + - name: cert-volume + secret: + secretName: controller-manager-metrics-cert + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 +``` + +- By last change the metrics port at `config/default/,amager_metrics_patch.yaml` +the patch to use the port `8443`: + +```yaml +... + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8443" +``` + +### By using Controller-Runtime new feature + +Also, you might want to check the new feature added in Controller-Runtime via +the [pr](https://github.com/kubernetes-sigs/controller-runtime/pull/2407) which can handle authentication (authn), +authorization (authz) such as kube-rbac-proxy. + +Note that, after the [issue](https://github.com/kubernetes-sigs/controller-runtime/issues/2781) opened +for controller-runtime enhance this new feature to address the concerns raised we plan add an option +to use it in the default scaffold combined with cert-manager. For further information, please check the +[proposal](../../../../designs/discontinue_usage_of_kube_rbac_proxy.md). ## Exporting Metrics for Prometheus @@ -91,6 +250,14 @@ for the metrics exported from the namespace where the project is running Screenshot 2019-10-02 at 13 07 13 +## Consuming the Metrics from other Pods. + +Then, see an example to create a Pod using Curl to reach out the metrics: + +```sh +kubectl run curl --restart=Never -n --image=curlimages/curl:7.78.0 -- /bin/sh -c "curl -v http://-controller-manager-metrics-service..svc.cluster.local:8080/metrics" +``` + ## Publishing Additional Metrics If you wish to publish additional metrics from your controllers, this @@ -141,4 +308,4 @@ In order to publish metrics and view them on the Prometheus UI, the Prometheus i Those metrics will be available for prometheus or other openmetrics systems to scrape. -![Screen Shot 2021-06-14 at 10 15 59 AM](https://user-images.githubusercontent.com/37827279/121932262-8843cd80-ccf9-11eb-9c8e-98d0eda80169.png) +![Screen Shot 2021-06-14 at 10 15 59 AM](https://user-images.githubusercontent.com/37827279/121932262-8843cd80-ccf9-11eb-9c8e-98d0eda80169.png) \ No newline at end of file diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go index 97258ee47aa..3de1859c668 100644 --- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go +++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go @@ -583,13 +583,6 @@ func updateExample(sp *Sample) { filepath.Join(sp.ctx.Dir, "config/samples/batch_v1_cronjob.yaml"), `# TODO(user): Add fields here`, "") CheckError("fixing samples/batch_v1_cronjob.yaml", err) - - // update default/manager_auth_proxy_patch.yaml - err = pluginutil.InsertCode( - filepath.Join(sp.ctx.Dir, "config/default/manager_auth_proxy_patch.yaml"), - ` template: - spec:`, ManagerAuthProxySample) - CheckError("fixing default/manager_auth_proxy_patch.yaml", err) } func addControllerTest(sp *Sample) { diff --git a/hack/docs/internal/cronjob-tutorial/sample.go b/hack/docs/internal/cronjob-tutorial/sample.go index baebb4cfa4b..7e413af1065 100644 --- a/hack/docs/internal/cronjob-tutorial/sample.go +++ b/hack/docs/internal/cronjob-tutorial/sample.go @@ -130,21 +130,3 @@ const DefaultKustomization = `#replacements: # delimiter: '.' # index: 1 # create: true` - -const ManagerAuthProxySample = ` - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux` diff --git a/pkg/plugin/util/util.go b/pkg/plugin/util/util.go index ebf5418adda..bae4e76f47f 100644 --- a/pkg/plugin/util/util.go +++ b/pkg/plugin/util/util.go @@ -97,6 +97,36 @@ func InsertCodeIfNotExist(filename, target, code string) error { return InsertCode(filename, target, code) } +// AppendCodeIfNotExist checks if the code does not already exist in the file, and if not, appends it to the end. +func AppendCodeIfNotExist(filename, code string) error { + contents, err := os.ReadFile(filename) + if err != nil { + return err + } + + if strings.Contains(string(contents), code) { + return nil // Code already exists, no need to append. + } + + return AppendCodeAtTheEnd(filename, code) +} + +// AppendCodeAtTheEnd appends the given code at the end of the file. +func AppendCodeAtTheEnd(filename, code string) error { + f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + return + } + }() + + _, err = f.WriteString(code) + return err +} + // UncommentCode searches for target in the file and remove the comment prefix // of the target content. The target content may span multiple lines. func UncommentCode(filename, target, prefix string) error { diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/api.go b/pkg/plugins/common/kustomize/v2/scaffolds/api.go index dc875d01235..03e3ec2b8e6 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/api.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/api.go @@ -102,28 +102,37 @@ func (s *apiScaffolder) Scaffold() error { // Add scaffolded CRD Editor and Viewer roles in config/rbac/kustomization.yaml rbacKustomizeFilePath := "config/rbac/kustomization.yaml" - comment := ` -# For each CRD, "Editor" and "Viewer" roles are scaffolded by -# default, aiding admins in cluster management. Those roles are -# not used by the Project itself. You can comment the following lines -# if you do not want those helpers be installed with your Project.` - err = pluginutil.InsertCodeIfNotExist(rbacKustomizeFilePath, - "- auth_proxy_client_clusterrole.yaml", comment) + err = pluginutil.AppendCodeIfNotExist(rbacKustomizeFilePath, + editViewRulesCommentFragment) if err != nil { - log.Errorf("Unable to add a comment in the file "+ + log.Errorf("Unable to append the edit/view roles editViewRulesCommentFragment in the file "+ "%s.", rbacKustomizeFilePath) } crdName := strings.ToLower(s.resource.Kind) if s.config.IsMultiGroup() && s.resource.Group != "" { crdName = strings.ToLower(s.resource.Group) + "_" + crdName } - err = pluginutil.InsertCodeIfNotExist(rbacKustomizeFilePath, comment, + err = pluginutil.InsertCodeIfNotExist(rbacKustomizeFilePath, editViewRulesCommentFragment, fmt.Sprintf("\n- %[1]s_editor_role.yaml\n- %[1]s_viewer_role.yaml", crdName)) if err != nil { log.Errorf("Unable to add Editor and Viewer roles in the file "+ "%s.", rbacKustomizeFilePath) } + // Add an empty line at the end of the file + err = pluginutil.AppendCodeIfNotExist(rbacKustomizeFilePath, + ` + +`) + if err != nil { + log.Errorf("Unable to append empty line at the end of the file"+ + "%s.", rbacKustomizeFilePath) + } } return nil } + +const editViewRulesCommentFragment = `# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the Project itself. You can comment the following lines +# if you do not want those helpers be installed with your Project.` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/init.go b/pkg/plugins/common/kustomize/v2/scaffolds/init.go index baea4bb55c6..cc034727e11 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/init.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/init.go @@ -64,10 +64,7 @@ func (s *initScaffolder) Scaffold() error { templates := []machinery.Builder{ &rbac.Kustomization{}, - &rbac.AuthProxyRole{}, - &rbac.AuthProxyRoleBinding{}, - &rbac.AuthProxyService{}, - &rbac.AuthProxyClientRole{}, + &rbac.MetricsService{}, &rbac.RoleBinding{}, // We need to create a Role because if the project // has not CRD define the controller-gen will not generate this file @@ -76,9 +73,9 @@ func (s *initScaffolder) Scaffold() error { &rbac.LeaderElectionRoleBinding{}, &rbac.ServiceAccount{}, &manager.Kustomization{}, + &kdefault.ManagerMetricsPatch{}, &manager.Config{Image: imageName}, &kdefault.Kustomization{}, - &kdefault.ManagerAuthProxyPatch{}, &kdefault.ManagerConfigPatch{}, &prometheus.Kustomization{}, &prometheus.Monitor{}, diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role_binding.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/enable_matrics_patch.go similarity index 52% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role_binding.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/enable_matrics_patch.go index 4ee86ed5c0f..d403c557765 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role_binding.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/enable_matrics_patch.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rbac +package kdefault import ( "path/filepath" @@ -22,38 +22,38 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &AuthProxyRoleBinding{} +var _ machinery.Template = &ManagerMetricsPatch{} -// AuthProxyRoleBinding scaffolds a file that defines the role binding for the auth proxy -type AuthProxyRoleBinding struct { +// ManagerMetricsPatch scaffolds a file that defines the patch that enables prometheus metrics for the manager +type ManagerMetricsPatch struct { machinery.TemplateMixin - machinery.ProjectNameMixin + machinery.ComponentConfigMixin } // SetTemplateDefaults implements file.Template -func (f *AuthProxyRoleBinding) SetTemplateDefaults() error { +func (f *ManagerMetricsPatch) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_role_binding.yaml") + f.Path = filepath.Join("config", "default", "manager_metrics_patch.yaml") } - f.TemplateBody = proxyRoleBindinggTemplate + f.TemplateBody = kustomizeMetricsPatchTemplate + + f.IfExistsAction = machinery.Error return nil } -const proxyRoleBindinggTemplate = `apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +const kustomizeMetricsPatchTemplate = `# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment metadata: - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount name: controller-manager namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index 319bcf1a6b3..c676a357010 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -73,10 +73,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml {{ if .ComponentConfig -}} # Mount the controller config file for loading manager configurations diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go deleted file mode 100644 index d8d57261952..00000000000 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kdefault - -import ( - "path/filepath" - - "sigs.k8s.io/kubebuilder/v3/pkg/machinery" -) - -var _ machinery.Template = &ManagerAuthProxyPatch{} - -// ManagerAuthProxyPatch scaffolds a file that defines the patch that enables prometheus metrics for the manager -type ManagerAuthProxyPatch struct { - machinery.TemplateMixin - machinery.ComponentConfigMixin -} - -// SetTemplateDefaults implements file.Template -func (f *ManagerAuthProxyPatch) SetTemplateDefaults() error { - if f.Path == "" { - f.Path = filepath.Join("config", "default", "manager_auth_proxy_patch.yaml") - } - - f.TemplateBody = kustomizeAuthProxyPatchTemplate - - f.IfExistsAction = machinery.Error - - return nil -} - -const kustomizeAuthProxyPatchTemplate = `# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi -{{- if not .ComponentConfig }} - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" -{{- end }} -` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager/config.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager/config.go index 271ccd633f2..cf8b6036794 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager/config.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager/config.go @@ -109,7 +109,9 @@ spec: - /manager {{- if not .ComponentConfig }} args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 {{- end }} image: {{ .Image }} name: manager diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go index 339ca03f072..dbae8fdfb8a 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go @@ -54,11 +54,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_client_role.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_client_role.go deleted file mode 100644 index a348524ab4a..00000000000 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_client_role.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rbac - -import ( - "path/filepath" - - "sigs.k8s.io/kubebuilder/v3/pkg/machinery" -) - -var _ machinery.Template = &AuthProxyClientRole{} - -// AuthProxyClientRole scaffolds a file that defines the role for the metrics reader -type AuthProxyClientRole struct { - machinery.TemplateMixin - machinery.ProjectNameMixin -} - -// SetTemplateDefaults implements file.Template -func (f *AuthProxyClientRole) SetTemplateDefaults() error { - if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_client_clusterrole.yaml") - } - - f.TemplateBody = clientClusterRoleTemplate - - return nil -} - -const clientClusterRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get -` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role.go deleted file mode 100644 index d1b639c0ee3..00000000000 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rbac - -import ( - "path/filepath" - - "sigs.k8s.io/kubebuilder/v3/pkg/machinery" -) - -var _ machinery.Template = &AuthProxyRole{} - -// AuthProxyRole scaffolds a file that defines the role for the auth proxy -type AuthProxyRole struct { - machinery.TemplateMixin - machinery.ProjectNameMixin -} - -// SetTemplateDefaults implements file.Template -func (f *AuthProxyRole) SetTemplateDefaults() error { - if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_role.yaml") - } - - f.TemplateBody = proxyRoleTemplate - - return nil -} - -const proxyRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go index d3ea9b22fd9..b2ce8b8ebc3 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go @@ -53,11 +53,5 @@ const kustomizeRBACTemplate = `resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_service.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go similarity index 72% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_service.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go index 219efc86ed7..8ab5b4d6c3b 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_service.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go @@ -22,26 +22,26 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &AuthProxyService{} +var _ machinery.Template = &MetricsService{} -// AuthProxyService scaffolds a file that defines the service for the auth proxy -type AuthProxyService struct { +// MetricsService scaffolds a file that defines the service for the auth proxy +type MetricsService struct { machinery.TemplateMixin machinery.ProjectNameMixin } // SetTemplateDefaults implements file.Template -func (f *AuthProxyService) SetTemplateDefaults() error { +func (f *MetricsService) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_service.yaml") + f.Path = filepath.Join("config", "rbac", "metrics_service.yaml") } - f.TemplateBody = authProxyServiceTemplate + f.TemplateBody = metricsServiceTemplate return nil } -const authProxyServiceTemplate = `apiVersion: v1 +const metricsServiceTemplate = `apiVersion: v1 kind: Service metadata: labels: @@ -52,10 +52,10 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager ` diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index 02f72b01910..582d60d3ef7 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -54,6 +54,49 @@ func GenerateV4(kbc *utils.TestContext) { fmt.Sprintf("%s_webhook.go", strings.ToLower(kbc.Kind)))) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- ../certmanager", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- ../prometheus", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- path: webhookcainjection_patch.yaml", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "#- path: manager_metrics_patch.yaml", "#")).To(Succeed()) + + ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + certManagerTarget, "#")).To(Succeed()) + + if kbc.IsRestricted { + By("uncomment kustomize files to ensure that pods are restricted") + uncommentPodStandards(kbc) + } +} + +// GenerateV4WithoutMetrics implements a go/v4 plugin project defined by a TestContext. +func GenerateV4WithoutMetrics(kbc *utils.TestContext) { + initingTheProject(kbc) + creatingAPI(kbc) + + By("scaffolding mutating and validating webhooks") + err := kbc.CreateWebhook( + "--group", kbc.Group, + "--version", kbc.Version, + "--kind", kbc.Kind, + "--defaulting", + "--programmatic-validation", + ) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("implementing the mutating and validating webhooks") + err = pluginutil.ImplementWebhooks(filepath.Join( + kbc.Dir, "api", kbc.Version, + fmt.Sprintf("%s_webhook.go", strings.ToLower(kbc.Kind)))) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../certmanager", "#")).To(Succeed()) diff --git a/test/e2e/v4/plugin_cluster_test.go b/test/e2e/v4/plugin_cluster_test.go index 034266e6741..1e312a13bbf 100644 --- a/test/e2e/v4/plugin_cluster_test.go +++ b/test/e2e/v4/plugin_cluster_test.go @@ -17,7 +17,6 @@ limitations under the License. package v4 import ( - "encoding/json" "fmt" "os" "os/exec" @@ -35,18 +34,6 @@ import ( "sigs.k8s.io/kubebuilder/v3/test/e2e/utils" ) -const ( - tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}` -) - -// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type -// that we want to use for extracting the token. -type tokenRequest struct { - Status struct { - Token string `json:"token"` - } `json:"status"` -} - var _ = Describe("kubebuilder", func() { Context("plugin go/v4", func() { var kbc *utils.TestContext @@ -68,24 +55,29 @@ var _ = Describe("kubebuilder", func() { It("should generate a runnable project", func() { kbc.IsRestricted = false GenerateV4(kbc) - Run(kbc, true, false) + Run(kbc, true, false, true) }) It("should generate a runnable project with the Installer", func() { kbc.IsRestricted = false GenerateV4(kbc) - Run(kbc, false, true) + Run(kbc, false, true, true) + }) + It("should generate a runnable project without metrics exposed", func() { + kbc.IsRestricted = false + GenerateV4WithoutMetrics(kbc) + Run(kbc, true, false, false) }) It("should generate a runnable project with the manager running "+ "as restricted and without webhooks", func() { kbc.IsRestricted = true GenerateV4WithoutWebhooks(kbc) - Run(kbc, false, false) + Run(kbc, false, false, true) }) }) }) // Run runs a set of e2e tests for a scaffolded project defined by a TestContext. -func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { +func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) { var controllerPodName string var err error @@ -119,13 +111,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { var output []byte if !isToUseInstaller { - // NOTE: If you want to run the test against a GKE cluster, you will need to grant yourself permission. - // Otherwise, you may see "... is forbidden: attempt to grant extra privileges" - // $ kubectl create clusterrolebinding myname-cluster-admin-binding \ - // --clusterrole=cluster-admin --user=myname@mycompany.com - // https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control By("deploying the controller-manager") - cmd := exec.Command("make", "deploy", "IMG="+kbc.ImageName) output, err = kbc.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -134,13 +120,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { err = kbc.Make("build-installer", "IMG="+kbc.ImageName) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - // NOTE: If you want to run the test against a GKE cluster, you will need to grant yourself permission. - // Otherwise, you may see "... is forbidden: attempt to grant extra privileges" - // $ kubectl create clusterrolebinding myname-cluster-admin-binding \ - // --clusterrole=cluster-admin --user=myname@mycompany.com - // https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control By("deploying the controller-manager with the installer") - _, err = kbc.Kubectl.Apply(true, "-f", "dist/install.yaml") ExpectWithOffset(1, err).NotTo(HaveOccurred()) } @@ -183,14 +163,8 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { }() EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) - By("granting permissions to access the metrics") - _, err = kbc.Kubectl.Command( - "create", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix), - fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", kbc.TestSuffix), - fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - _ = curlMetrics(kbc) + By("validating the metrics endpoint") + _ = curlMetrics(kbc, hasMetrics) if hasWebhook { By("validating that cert-manager has provisioned the certificate Secret") @@ -267,12 +241,14 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { return err }, time.Minute, time.Second).Should(Succeed()) - By("validating that the created resource object gets reconciled in the controller") - metricsOutput := curlMetrics(kbc) - ExpectWithOffset(1, metricsOutput).To(ContainSubstring(fmt.Sprintf( - `controller_runtime_reconcile_total{controller="%s",result="success"} 1`, - strings.ToLower(kbc.Kind), - ))) + if hasMetrics { + By("checking the metrics values to validate that the created resource object gets reconciled") + metricsOutput := curlMetrics(kbc, hasMetrics) + ExpectWithOffset(1, metricsOutput).To(ContainSubstring(fmt.Sprintf( + `controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + strings.ToLower(kbc.Kind), + ))) + } if hasWebhook { By("validating that mutating and validating webhooks are working fine") @@ -285,92 +261,117 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) ExpectWithOffset(1, count).To(BeNumerically("==", 5)) } + } // curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned. -func curlMetrics(kbc *utils.TestContext) string { - By("reading the metrics token") - // Filter token query by service account in case more than one exists in a namespace. - token, err := ServiceAccountToken(kbc) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0)) +func curlMetrics(kbc *utils.TestContext, hasMetrics bool) string { + By("validating that the controller-manager service is available") + _, err := kbc.Kubectl.Get( + true, + "service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + ) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist") - By("creating a curl pod") - cmdOpts := []string{ - "run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--", - "curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, strings.TrimSpace(token)), - fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics", - kbc.TestSuffix, kbc.Kubectl.Namespace), + By("validating that the controller-manager deployment is ready") + verifyDeploymentReady := func() error { + output, err := kbc.Kubectl.Get( + true, + "deployment", fmt.Sprintf("e2e-%s-controller-manager", kbc.TestSuffix), + "-o", "jsonpath={.status.readyReplicas}", + ) + if err != nil { + return err + } + readyReplicas, _ := strconv.Atoi(output) + if readyReplicas < 1 { + return fmt.Errorf("expected at least 1 ready replica, got %d", readyReplicas) + } + return nil } - _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) + EventuallyWithOffset(2, verifyDeploymentReady, 240*time.Second, time.Second).Should(Succeed(), + "Deployment is not ready") - By("validating that the curl pod is running as expected") - verifyCurlUp := func() error { - // Validate pod status - status, err := kbc.Kubectl.Get( + By("ensuring the service endpoint is ready") + eventuallyCheckServiceEndpoint := func() error { + output, err := kbc.Kubectl.Get( true, - "pods", "curl", "-o", "jsonpath={.status.phase}") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - if status != "Completed" && status != "Succeeded" { - return fmt.Errorf("curl pod in %s status", status) + "endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + "-o", "jsonpath={.subsets[*].addresses[*].ip}", + ) + if err != nil { + return err + } + if output == "" { + return fmt.Errorf("no endpoints found") } return nil } - EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(), + "Service endpoint should be ready") - By("validating that the metrics endpoint is serving as expected") - var metricsOutput string - getCurlLogs := func() string { - metricsOutput, err = kbc.Kubectl.Logs("curl") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - return metricsOutput + By("creating a curl pod to access the metrics endpoint") + // nolint:lll + cmdOpts := []string{ + "run", "curl", + "--restart=Never", + "--namespace", kbc.Kubectl.Namespace, + "--image=curlimages/curl:7.78.0", + "--", + "/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics", + kbc.TestSuffix, kbc.Kubectl.Namespace), } - EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/2 200")) + _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + + var metricsOutput string + if hasMetrics { + By("validating that the curl pod is running as expected") + verifyCurlUp := func() error { + status, err := kbc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + + By("validating that the metrics endpoint is serving as expected") + getCurlLogs := func() string { + metricsOutput, err = kbc.Kubectl.Logs("curl") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + return metricsOutput + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/1.1 200 OK")) + } else { + By("validating that the curl pod fail as expected") + verifyCurlUp := func() error { + status, err := kbc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status == "Succeeded" { + return fmt.Errorf( + "curl pod in %s status when should fail with an error", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + By("validating that the metrics endpoint is not working as expected") + getCurlLogs := func() string { + metricsOutput, err = kbc.Kubectl.Logs("curl") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + return metricsOutput + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("Connection refused")) + } By("cleaning up the curl pod") _, err = kbc.Kubectl.Delete(true, "pods/curl") ExpectWithOffset(3, err).NotTo(HaveOccurred()) return metricsOutput } - -// ServiceAccountToken provides a helper function that can provide you with a service account -// token that you can use to interact with the service. This function leverages the k8s' -// TokenRequest API in raw format in order to make it generic for all version of the k8s that -// is currently being supported in kubebuilder test infra. -// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. -func ServiceAccountToken(kbc *utils.TestContext) (out string, err error) { - By("Creating the ServiceAccount token") - secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount) - tokenRequestFile := filepath.Join(kbc.Dir, secretName) - err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) - if err != nil { - return out, err - } - var rawJson string - Eventually(func() error { - // Output of this is already a valid JWT token. No need to covert this from base64 to string format - rawJson, err = kbc.Kubectl.Command( - "create", - "--raw", fmt.Sprintf( - "/api/v1/namespaces/%s/serviceaccounts/%s/token", - kbc.Kubectl.Namespace, - kbc.Kubectl.ServiceAccount, - ), - "-f", tokenRequestFile, - ) - if err != nil { - return err - } - var token tokenRequest - err = json.Unmarshal([]byte(rawJson), &token) - if err != nil { - return err - } - out = token.Status.Token - return nil - }, time.Minute, time.Second).Should(Succeed()) - - return out, err -} diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml index 2f78dfb54aa..e81d73d4bc3 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/manager/manager.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/manager/manager.yaml index 22430e2a678..4e217f48c6d 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/manager/manager.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager securityContext: diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml index c7e880652bc..afc86491c6f 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 8f3b7014f0b..00000000000 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup-with-deploy-image - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 13038ff7689..00000000000 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup-with-deploy-image - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index aae73208a49..00000000000 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup-with-deploy-image - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml index 08b359e46b5..51bf3b2bea4 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml @@ -9,13 +9,7 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -40,3 +34,4 @@ resources: - ship_frigate_viewer_role.yaml - crew_captain_editor_role.yaml - crew_captain_viewer_role.yaml + diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_service.yaml similarity index 86% rename from testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_service.yaml index 32b2aac0952..a01a6872e4a 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml b/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml index 26592d65d9a..1f880c1fcdc 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml @@ -1191,40 +1191,6 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-multigroup-with-deploy-image - name: project-v4-multigroup-with-deploy-image-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-multigroup-with-deploy-image - name: project-v4-multigroup-with-deploy-image-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -1506,22 +1472,6 @@ subjects: name: project-v4-multigroup-with-deploy-image-controller-manager namespace: project-v4-multigroup-with-deploy-image-system --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-multigroup-with-deploy-image - name: project-v4-multigroup-with-deploy-image-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: project-v4-multigroup-with-deploy-image-proxy-role -subjects: -- kind: ServiceAccount - name: project-v4-multigroup-with-deploy-image-controller-manager - namespace: project-v4-multigroup-with-deploy-image-system ---- apiVersion: v1 kind: Service metadata: @@ -1533,10 +1483,10 @@ metadata: namespace: project-v4-multigroup-with-deploy-image-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -1579,9 +1529,9 @@ spec: spec: containers: - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 command: - /manager image: controller:latest @@ -1618,29 +1568,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-multigroup-with-deploy-image-controller-manager diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 9fe6e3630df..d00ee2596e1 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/testdata/project-v4-multigroup/config/manager/manager.yaml b/testdata/project-v4-multigroup/config/manager/manager.yaml index d80c8f33132..a65ca4a1bc2 100644 --- a/testdata/project-v4-multigroup/config/manager/manager.yaml +++ b/testdata/project-v4-multigroup/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager securityContext: diff --git a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml index bb60c0d334a..7b9bc2077a7 100644 --- a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-multigroup/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 53613e3eeef..00000000000 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-multigroup/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 56c97ddca82..00000000000 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-multigroup/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index 10f89301cdb..00000000000 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/testdata/project-v4-multigroup/config/rbac/kustomization.yaml b/testdata/project-v4-multigroup/config/rbac/kustomization.yaml index 08b359e46b5..51bf3b2bea4 100644 --- a/testdata/project-v4-multigroup/config/rbac/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/rbac/kustomization.yaml @@ -9,13 +9,7 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -40,3 +34,4 @@ resources: - ship_frigate_viewer_role.yaml - crew_captain_editor_role.yaml - crew_captain_viewer_role.yaml + diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_service.yaml similarity index 86% rename from testdata/project-v4-multigroup/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-multigroup/config/rbac/metrics_service.yaml index b168382998d..efdd5394534 100644 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-multigroup/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup/dist/install.yaml b/testdata/project-v4-multigroup/dist/install.yaml index e2584bdeac2..8b03103111f 100644 --- a/testdata/project-v4-multigroup/dist/install.yaml +++ b/testdata/project-v4-multigroup/dist/install.yaml @@ -1191,40 +1191,6 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-multigroup - name: project-v4-multigroup-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-multigroup - name: project-v4-multigroup-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -1506,22 +1472,6 @@ subjects: name: project-v4-multigroup-controller-manager namespace: project-v4-multigroup-system --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-multigroup - name: project-v4-multigroup-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: project-v4-multigroup-proxy-role -subjects: -- kind: ServiceAccount - name: project-v4-multigroup-controller-manager - namespace: project-v4-multigroup-system ---- apiVersion: v1 kind: Service metadata: @@ -1533,10 +1483,10 @@ metadata: namespace: project-v4-multigroup-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -1579,9 +1529,9 @@ spec: spec: containers: - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 command: - /manager image: controller:latest @@ -1618,29 +1568,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-multigroup-controller-manager diff --git a/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml index 62e78ccdbbe..bf965dc95f0 100644 --- a/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml +++ b/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/testdata/project-v4-with-deploy-image/config/manager/manager.yaml b/testdata/project-v4-with-deploy-image/config/manager/manager.yaml index f232684fa6b..1fd54a41dc2 100644 --- a/testdata/project-v4-with-deploy-image/config/manager/manager.yaml +++ b/testdata/project-v4-with-deploy-image/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager env: diff --git a/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml b/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml index 0f805f2c2e7..2d08ceebac3 100644 --- a/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index b439cad7f2c..00000000000 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-with-deploy-image - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 438d9bd0702..00000000000 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-with-deploy-image - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index 3be0002395d..00000000000 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project-v4-with-deploy-image - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml index 67076dab990..234947b0ce5 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml @@ -9,13 +9,7 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -24,3 +18,4 @@ resources: - busybox_viewer_role.yaml - memcached_editor_role.yaml - memcached_viewer_role.yaml + diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_service.yaml similarity index 86% rename from testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-with-deploy-image/config/rbac/metrics_service.yaml index 80efb4fa08c..905ebc18c94 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-with-deploy-image/dist/install.yaml b/testdata/project-v4-with-deploy-image/dist/install.yaml index 70babe98cbe..b2abdf48910 100644 --- a/testdata/project-v4-with-deploy-image/dist/install.yaml +++ b/testdata/project-v4-with-deploy-image/dist/install.yaml @@ -516,40 +516,6 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-with-deploy-image - name: project-v4-with-deploy-image-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-with-deploy-image - name: project-v4-with-deploy-image-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -582,22 +548,6 @@ subjects: name: project-v4-with-deploy-image-controller-manager namespace: project-v4-with-deploy-image-system --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-with-deploy-image - name: project-v4-with-deploy-image-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: project-v4-with-deploy-image-proxy-role -subjects: -- kind: ServiceAccount - name: project-v4-with-deploy-image-controller-manager - namespace: project-v4-with-deploy-image-system ---- apiVersion: v1 kind: Service metadata: @@ -609,10 +559,10 @@ metadata: namespace: project-v4-with-deploy-image-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -655,9 +605,9 @@ spec: spec: containers: - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 command: - /manager env: @@ -699,29 +649,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-with-deploy-image-controller-manager diff --git a/testdata/project-v4-with-grafana/config/default/kustomization.yaml b/testdata/project-v4-with-grafana/config/default/kustomization.yaml index 7fca0820b0c..df5296e37fd 100644 --- a/testdata/project-v4-with-grafana/config/default/kustomization.yaml +++ b/testdata/project-v4-with-grafana/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/testdata/project-v4-with-grafana/config/manager/manager.yaml b/testdata/project-v4-with-grafana/config/manager/manager.yaml index 3d44ae43d62..64ab9a2daa7 100644 --- a/testdata/project-v4-with-grafana/config/manager/manager.yaml +++ b/testdata/project-v4-with-grafana/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager securityContext: diff --git a/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml b/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml index 8505bfa5bfc..5f3e9887eae 100644 --- a/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-with-grafana/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 5a5a1d0259d..00000000000 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-with-grafana - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 979bc272f7a..00000000000 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4-with-grafana - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index b5302ea3805..00000000000 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project-v4-with-grafana - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml b/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml index 731832a6ac3..cb51d20d1cf 100644 --- a/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml @@ -9,10 +9,4 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_service.yaml similarity index 86% rename from testdata/project-v4-with-grafana/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-with-grafana/config/rbac/metrics_service.yaml index b52e2ef0f4d..4c40c3579cb 100644 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-with-grafana/dist/install.yaml b/testdata/project-v4-with-grafana/dist/install.yaml index 5877a647805..97764fed65b 100644 --- a/testdata/project-v4-with-grafana/dist/install.yaml +++ b/testdata/project-v4-with-grafana/dist/install.yaml @@ -75,40 +75,6 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-with-grafana - name: project-v4-with-grafana-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-with-grafana - name: project-v4-with-grafana-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -141,22 +107,6 @@ subjects: name: project-v4-with-grafana-controller-manager namespace: project-v4-with-grafana-system --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4-with-grafana - name: project-v4-with-grafana-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: project-v4-with-grafana-proxy-role -subjects: -- kind: ServiceAccount - name: project-v4-with-grafana-controller-manager - namespace: project-v4-with-grafana-system ---- apiVersion: v1 kind: Service metadata: @@ -168,10 +118,10 @@ metadata: namespace: project-v4-with-grafana-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -198,32 +148,9 @@ spec: spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 command: - /manager image: controller:latest diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index ae7fc170730..131ea0843e0 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -27,10 +27,10 @@ resources: #- ../prometheus patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# More info: https://book.kubebuilder.io/reference/metrics +# If you want to expose the metric endpoint of your controller-manager uncomment the following line. +#- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4/config/default/manager_metrics_patch.yaml b/testdata/project-v4/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..ee197d3f718 --- /dev/null +++ b/testdata/project-v4/config/default/manager_metrics_patch.yaml @@ -0,0 +1,13 @@ +# This patch adds the args to allow exposing the metrics endpoint securely +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--metrics-bind-address=0.0.0.0:8080" diff --git a/testdata/project-v4/config/manager/manager.yaml b/testdata/project-v4/config/manager/manager.yaml index 7c552d545f1..29ab53f42f9 100644 --- a/testdata/project-v4/config/manager/manager.yaml +++ b/testdata/project-v4/config/manager/manager.yaml @@ -61,7 +61,9 @@ spec: - command: - /manager args: - - --leader-elect + - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 image: controller:latest name: manager securityContext: diff --git a/testdata/project-v4/config/prometheus/monitor.yaml b/testdata/project-v4/config/prometheus/monitor.yaml index 767555588d4..d4a09ebf9fe 100644 --- a/testdata/project-v4/config/prometheus/monitor.yaml +++ b/testdata/project-v4/config/prometheus/monitor.yaml @@ -11,11 +11,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index c534a7a5387..00000000000 --- a/testdata/project-v4/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/testdata/project-v4/config/rbac/auth_proxy_role.yaml b/testdata/project-v4/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 43aa96480ad..00000000000 --- a/testdata/project-v4/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/testdata/project-v4/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index e5bbe0214cd..00000000000 --- a/testdata/project-v4/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/testdata/project-v4/config/rbac/kustomization.yaml b/testdata/project-v4/config/rbac/kustomization.yaml index 8518bf9e24d..6dd56c7db27 100644 --- a/testdata/project-v4/config/rbac/kustomization.yaml +++ b/testdata/project-v4/config/rbac/kustomization.yaml @@ -9,13 +9,7 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -26,3 +20,4 @@ resources: - firstmate_viewer_role.yaml - captain_editor_role.yaml - captain_viewer_role.yaml + diff --git a/testdata/project-v4/config/rbac/auth_proxy_service.yaml b/testdata/project-v4/config/rbac/metrics_service.yaml similarity index 85% rename from testdata/project-v4/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4/config/rbac/metrics_service.yaml index 2b723ff3239..3cd90980fa3 100644 --- a/testdata/project-v4/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4/config/rbac/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4/dist/install.yaml b/testdata/project-v4/dist/install.yaml index 4466171f902..47b70abe185 100644 --- a/testdata/project-v4/dist/install.yaml +++ b/testdata/project-v4/dist/install.yaml @@ -510,40 +510,6 @@ rules: - update --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4 - name: project-v4-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4 - name: project-v4-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -576,22 +542,6 @@ subjects: name: project-v4-controller-manager namespace: project-v4-system --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: project-v4 - name: project-v4-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: project-v4-proxy-role -subjects: -- kind: ServiceAccount - name: project-v4-controller-manager - namespace: project-v4-system ---- apiVersion: v1 kind: Service metadata: @@ -603,10 +553,10 @@ metadata: namespace: project-v4-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -649,9 +599,9 @@ spec: spec: containers: - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + - --health-probe-bind-address=:8081 + - --metrics-bind-address=0 command: - /manager image: controller:latest @@ -688,29 +638,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-controller-manager