feat/fix: enhance cert-manager integration for metrics endpoints (fol…

…low-up to PR kubernetes-sigs#4243) This commit is a follow-up to PR kubernetes-sigs#4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to kubernetes-sigs#3871 and kubernetes-sigs#4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready
camilamacedo86 · Dec 1, 2024 · 183701b · 183701b
1 parent 781e93f
commit 183701b
Show file tree

Hide file tree

Showing 44 changed files with 731 additions and 292 deletions.
diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml
@@ -41,7 +41,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4/
           go mod tidy
 
@@ -82,8 +82,8 @@ jobs:
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment only ValidatingWebhookConfiguration
           # from cert-manager replaces
-          sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '153,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,123s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '155,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-with-plugins/
           go mod tidy
 
@@ -122,7 +122,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-multigroup
           go mod tidy
 

diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"path/filepath"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -30,6 +31,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`.
 */
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = ctrl.Log.WithName("setup")
+	scheme      = runtime.NewScheme()
+	setupLog    = ctrl.Log.WithName("setup")
+	certWatcher *certwatcher.CertWatcher
 )
 
 func init() {
@@ -74,6 +77,9 @@ func main() {
 	/*
 	 */
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -87,6 +93,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.")
+	flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -133,16 +143,27 @@ func main() {
 		// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
 		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
 
-		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
-		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
-
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
-
+		// If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates
+		// for the metrics server, suitable for development but not recommended for production. To use certificates
+		// created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml".
+		if len(certDir) > 0 {
+			setupLog.Info("using certificates for the metrics server",
+				"cert-dir", certDir, "cert-name", certName, "cert-key", certKey)
+
+			var err error
+			certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey))
+			if err != nil {
+				setupLog.Error(err, "to initialize certificate watcher", "error", err)
+				os.Exit(1)
+			}
+
+			metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+				config.GetCertificate = certWatcher.GetCertificate
+			})
+		} else {
+			setupLog.Info("WARNING: Using self-signed certificates for the metrics server." +
+				"Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.")
+		}
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -196,6 +217,14 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	if secureMetrics && certWatcher != nil {
+		setupLog.Info("Adding certificate watcher to manager")
+		if err := mgr.Add(certWatcher); err != nil {
+			setupLog.Error(err, "unable to add certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

diff --git a/...book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/...book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,35 @@
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/serving-certs
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/tmp/k8s-metrics-server/serving-certs
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false
+      items:
+        - key: ca.crt
+          path: ca.crt
+        - key: tls.crt
+          path: tls.crt
+        - key: tls.key
+          path: tls.key
diff --git a/...c/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/...c/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
@@ -42,9 +42,11 @@ patches:
     kind: Deployment
 
 # Uncomment the patches line if you enable Metrics and CertManager
-# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
-# This patch will protect the metrics with certmanager self-signed certs.
-- path: certmanager_metrics_manager_patch.yaml
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+- path: cert_metrics_manager_patch.yaml
+  target:
+    kind: Deployment
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml
@@ -2,3 +2,6 @@
 - op: add
   path: /spec/template/spec/containers/0/args/0
   value: --metrics-bind-address=:8443
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: --metrics-secure=true
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
@@ -4115,9 +4115,11 @@ spec:
     spec:
       containers:
       - args:
+        - --metrics-secure=true
         - --metrics-bind-address=:8443
         - --leader-elect
         - --health-probe-bind-address=:8081
+        - --cert-dir=/tmp/k8s-metrics-server/serving-certs
         command:
         - /manager
         image: controller:latest
@@ -4154,6 +4156,9 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
+        - mountPath: /tmp/k8s-metrics-server/serving-certs
+          name: metrics-certs
+          readOnly: true
       securityContext:
         runAsNonRoot: true
         seccompProfile:
@@ -4165,10 +4170,104 @@ spec:
         secret:
           defaultMode: 420
           secretName: webhook-server-cert
+      - name: metrics-certs
+        secret:
+          items:
+          - key: ca.crt
+            path: ca.crt
+          - key: tls.crt
+            path: tls.crt
+          - key: tls.key
+            path: tls.key
+          optional: false
+          secretName: metrics-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/component: certificate
+    app.kubernetes.io/created-by: project
+    app.kubernetes.io/instance: metrics-certs
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: certificate
+    app.kubernetes.io/part-of: project
+  name: project-metrics-certs
+  namespace: project-system
+spec:
+  dnsNames:
+  - project-webhook-service.project-system.svc
+  - project-webhook-service.project-system.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: project-selfsigned-issuer
+  secretName: metrics-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/component: certificate
+    app.kubernetes.io/created-by: project
+    app.kubernetes.io/instance: serving-cert
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: certificate
+    app.kubernetes.io/part-of: project
+  name: project-serving-cert
+  namespace: project-system
+spec:
+  dnsNames:
+  - project-webhook-service.project-system.svc
+  - project-webhook-service.project-system.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: project-selfsigned-issuer
+  secretName: webhook-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: project
+  name: project-selfsigned-issuer
+  namespace: project-system
+spec:
+  selfSigned: {}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/name: project
+    control-plane: controller-manager
+  name: project-controller-manager-metrics-monitor
+  namespace: project-system
+spec:
+  endpoints:
+  - tlsConfig:
+      ca:
+        secret:
+          key: ca.crt
+          name: metrics-server-cert
+      cert:
+        secret:
+          key: tls.crt
+          name: metrics-server-cert
+      insecureSkipVerify: false
+      keySecret:
+        key: tls.key
+        name: metrics-server-cert
+  selector:
+    matchLabels:
+      control-plane: controller-manager
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: project-system/project-serving-cert
   name: project-mutating-webhook-configuration
 webhooks:
 - admissionReviewVersions:
@@ -4195,6 +4294,8 @@ webhooks:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: project-system/project-serving-cert
   name: project-validating-webhook-configuration
 webhooks:
 - admissionReviewVersions:

diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go
@@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the
 First, it will contain the necessary imports.
 */
 
-
 package controller
 
 import (
-Original file line number
+Diff line change
@@ Expand Up @@
     First, it will contain the necessary imports.
     */
     package controller
     import (
@@ Expand Down @@