From 183701b5135e96505b939e142cae8bb7ad12d05b Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:10:28 +0000 Subject: [PATCH] feat/fix: enhance cert-manager integration for metrics endpoints (follow-up to PR #4243) This commit is a follow-up to PR #4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Related to https://github.com/kubernetes-sigs/kubebuilder/issues/3871 and https://github.com/kubernetes-sigs/kubebuilder/pull/4003 Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - Use Certwatcher to allow certificate rotation This configuration provides an option for users to be production-ready --- .github/workflows/test-e2e-samples.yml | 8 +- .../testdata/project/cmd/main.go | 53 ++++++--- .../default/cert_metrics_manager_patch.yaml | 35 ++++++ .../certmanager_metrics_manager_patch.yaml | 21 ---- .../project/config/default/kustomization.yaml | 8 +- .../config/default/manager_metrics_patch.yaml | 3 + .../testdata/project/dist/install.yaml | 101 ++++++++++++++++++ .../project/internal/controller/suite_test.go | 1 - .../testdata/project/cmd/main.go | 53 ++++++--- .../default/cert_metrics_manager_patch.yaml | 35 ++++++ .../certmanager_metrics_manager_patch.yaml | 21 ---- .../project/config/default/kustomization.yaml | 8 +- .../config/default/manager_metrics_patch.yaml | 3 + .../testdata/project/dist/install.yaml | 1 + .../testdata/project/cmd/main.go | 53 ++++++--- .../default/cert_metrics_manager_patch.yaml | 35 ++++++ .../certmanager_metrics_manager_patch.yaml | 21 ---- .../project/config/default/kustomization.yaml | 8 +- .../config/default/manager_metrics_patch.yaml | 3 + .../testdata/project/dist/install.yaml | 12 ++- .../cronjob-tutorial/generate_cronjob.go | 49 ++++----- ...patch.go => cert_metrics_manager_patch.go} | 59 ++++++---- .../config/kdefault/kustomization.go | 8 +- .../config/kdefault/manager_metrics_patch.go | 3 + .../scaffolds/internal/templates/cmd/main.go | 48 +++++++-- test/e2e/v4/generate_test.go | 18 ++-- testdata/project-v4-multigroup/cmd/main.go | 49 +++++++-- .../default/cert_metrics_manager_patch.yaml | 35 ++++++ .../certmanager_metrics_manager_patch.yaml | 21 ---- .../config/default/kustomization.yaml | 8 +- .../config/default/manager_metrics_patch.yaml | 3 + .../project-v4-multigroup/dist/install.yaml | 1 + testdata/project-v4-with-plugins/cmd/main.go | 53 ++++++--- .../default/cert_metrics_manager_patch.yaml | 35 ++++++ .../certmanager_metrics_manager_patch.yaml | 21 ---- .../config/default/kustomization.yaml | 8 +- .../config/default/manager_metrics_patch.yaml | 3 + .../project-v4-with-plugins/dist/install.yaml | 1 + testdata/project-v4/cmd/main.go | 49 +++++++-- .../default/cert_metrics_manager_patch.yaml | 35 ++++++ .../certmanager_metrics_manager_patch.yaml | 21 ---- .../config/default/kustomization.yaml | 8 +- .../config/default/manager_metrics_patch.yaml | 3 + testdata/project-v4/dist/install.yaml | 1 + 44 files changed, 731 insertions(+), 292 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml delete mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/{certmanager_metrics_manager_patch.go => cert_metrics_manager_patch.go} (52%) create mode 100644 testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4/config/default/cert_metrics_manager_patch.yaml delete mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index 1a73e9f6890..a169d1f961f 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -41,7 +41,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,184s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ go mod tidy @@ -82,8 +82,8 @@ jobs: sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces - sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '153,182s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,123s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '155,184s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-with-plugins/ go mod tidy @@ -122,7 +122,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '57,184s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4-multigroup go mod tidy diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index 458a9f8c3e2..fffbf77932f 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -30,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -54,8 +56,9 @@ Builtin types such as Job have their scheme added by `clientgoscheme`. */ var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -74,6 +77,9 @@ func main() { /* */ var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -87,6 +93,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -133,16 +143,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -196,6 +217,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2be15875d4 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index ab2925bc69e..4b1da297e94 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml index 2aaef6536f4..e504b415313 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -2,3 +2,6 @@ - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml index c20c83edbc5..91c4d86323d 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml @@ -4115,9 +4115,11 @@ spec: spec: containers: - args: + - --metrics-secure=true - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --cert-dir=/tmp/k8s-metrics-server/serving-certs command: - /manager image: controller:latest @@ -4154,6 +4156,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + - mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -4165,10 +4170,104 @@ spec: secret: defaultMode: 420 secretName: webhook-server-cert + - name: metrics-certs + secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project + app.kubernetes.io/instance: metrics-certs + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: project + name: project-metrics-certs + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: project + name: project-serving-cert + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + name: project-selfsigned-issuer + namespace: project-system +spec: + selfSigned: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project + control-plane: controller-manager + name: project-controller-manager-metrics-monitor + namespace: project-system +spec: + endpoints: + - tlsConfig: + ca: + secret: + key: ca.crt + name: metrics-server-cert + cert: + secret: + key: tls.crt + name: metrics-server-cert + insecureSkipVerify: false + keySecret: + key: tls.key + name: metrics-server-cert + selector: + matchLabels: + control-plane: controller-manager --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-mutating-webhook-configuration webhooks: - admissionReviewVersions: @@ -4195,6 +4294,8 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: + annotations: + cert-manager.io/inject-ca-from: project-system/project-serving-cert name: project-validating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go index 6c4e45c3d2e..528ad881da2 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/internal/controller/suite_test.go @@ -22,7 +22,6 @@ Kubebuilder scaffolded a `internal/controller/suite_test.go` file that does the First, it will contain the necessary imports. */ - package controller import ( diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index 7347d215463..318ec30e106 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -41,8 +43,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -54,6 +57,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -67,6 +73,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -113,16 +123,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -158,6 +179,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2be15875d4 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 20d178afc14..09cf05ed1b5 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml index 2aaef6536f4..e504b415313 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml @@ -2,3 +2,6 @@ - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true diff --git a/docs/book/src/getting-started/testdata/project/dist/install.yaml b/docs/book/src/getting-started/testdata/project/dist/install.yaml index 6af37f625be..67101b7ee61 100644 --- a/docs/book/src/getting-started/testdata/project/dist/install.yaml +++ b/docs/book/src/getting-started/testdata/project/dist/install.yaml @@ -418,6 +418,7 @@ spec: spec: containers: - args: + - --metrics-secure=true - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go index a199eec734b..13abd2fd3e6 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -31,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -51,8 +53,9 @@ import ( */ var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -73,6 +76,9 @@ func main() { /* */ var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -86,6 +92,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -132,16 +142,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -199,6 +220,14 @@ func main() { /* */ + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2be15875d4 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 9cee4b4f580..00000000000 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml index 9cba521a7c6..29834bce8bb 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +- path: cert_metrics_manager_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_metrics_patch.yaml index 2aaef6536f4..e504b415313 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_metrics_patch.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -2,3 +2,6 @@ - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml index 8a9b9c9f3ab..bfc17089808 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml @@ -7926,9 +7926,11 @@ spec: spec: containers: - args: + - --metrics-secure=true - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 + - --cert-dir=/tmp/k8s-metrics-server/serving-certs command: - /manager image: controller:latest @@ -7965,7 +7967,7 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - mountPath: /tmp/k8s-metrics-server/metrics-certs + - mountPath: /tmp/k8s-metrics-server/serving-certs name: metrics-certs readOnly: true securityContext: @@ -7981,6 +7983,14 @@ spec: secretName: webhook-server-cert - name: metrics-certs secret: + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + optional: false secretName: metrics-server-cert --- apiVersion: cert-manager.io/v1 diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go index a2b17b3b32a..c5ceb56a4a2 100644 --- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go +++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go @@ -90,7 +90,18 @@ func (sp *Sample) UpdateTutorial() { // 4. update makefile sp.updateMakefile() // 5. generate extra files - sp.codeGen() + cmd := exec.Command("go", "mod", "tidy") + _, err := sp.ctx.Run(cmd) + hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) + + cmd = exec.Command("go", "get", "github.com/robfig/cron") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("Failed to get package robfig/cron", err) + + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) + // 6. compensate other intro in API sp.updateAPIStuff() // 7. update reconciliation and main.go @@ -99,7 +110,9 @@ func (sp *Sample) UpdateTutorial() { // 7.2 update main.go sp.updateMain() // 8. generate extra files - sp.codeGen() + cmd = exec.Command("make", "generate", "manifests") + _, err = sp.ctx.Run(cmd) + hackutils.CheckError("run make generate and manifests", err) // 9. update suite_test explanation sp.updateSuiteTest() // 10. uncomment kustomization @@ -113,25 +126,14 @@ func (sp *Sample) UpdateTutorial() { // CodeGen is a noop for this sample, just to make generation of all samples // more efficient. We may want to refactor `UpdateTutorial` some day to take // advantage of a separate call, but it is not necessary. -func (sp *Sample) CodeGen() {} - -func (sp *Sample) codeGen() { - cmd := exec.Command("go", "mod", "tidy") +func (sp *Sample) CodeGen() { + cmd := exec.Command("make", "all") _, err := sp.ctx.Run(cmd) - hackutils.CheckError("Failed to run go mod tidy for cronjob tutorial", err) - - cmd = exec.Command("go", "get", "github.com/robfig/cron") - _, err = sp.ctx.Run(cmd) - hackutils.CheckError("Failed to get package robfig/cron", err) - - cmd = exec.Command("make", "all") - _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make all for cronjob tutorial", err) cmd = exec.Command("make", "build-installer") _, err = sp.ctx.Run(cmd) hackutils.CheckError("Failed to run make build-installer for cronjob tutorial", err) - } // insert code to fix docs @@ -355,15 +357,6 @@ CronJob controller's`+" `"+`SetupWithManager`+"`"+` method. }`, ` // +kubebuilder:docs-gen:collapse=old stuff`) hackutils.CheckError("fixing main.go", err) - - // Enabling metrics with certs - err = pluginutil.UncommentCode( - filepath.Join(sp.ctx.Dir, "cmd/main.go"), - `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"`, ` - // `) - hackutils.CheckError("enabling metrics service options into main.go", err) } func (sp *Sample) updateMakefile() { @@ -602,8 +595,10 @@ func (sp *Sample) updateKustomization() { err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment`, `#`) + hackutils.CheckError("enabling cert_metrics_manager_patch.yaml", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/prometheus/kustomization.yaml"), @@ -611,7 +606,7 @@ func (sp *Sample) updateKustomization() { # - path: monitor_tls_patch.yaml # target: # kind: ServiceMonitor`, `#`) - hackutils.CheckError("enabling certmanager_metrics_manager_patch.yaml", err) + hackutils.CheckError("enabling monitor tls patch", err) err = pluginutil.UncommentCode( filepath.Join(sp.ctx.Dir, "config/default/kustomization.yaml"), diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go similarity index 52% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go index 31859c3fd89..8a59fe26f27 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/cert_metrics_manager_patch.go @@ -35,7 +35,7 @@ type CertManagerMetricsPatch struct { // SetTemplateDefaults implements machinery.Template func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml") + f.Path = filepath.Join("config", "default", "cert_metrics_manager_patch.yaml") } f.TemplateBody = metricsManagerPatchTemplate @@ -50,25 +50,40 @@ func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { return nil } -const metricsManagerPatchTemplate = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: {{ .ProjectName }} - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert +// nolint:lll +const metricsManagerPatchTemplate = `# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index f6122cb8854..8c5e0b4cd18 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -87,9 +87,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go index ca822c79315..218bcc24ec7 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go @@ -46,4 +46,7 @@ const kustomizeMetricsPatchTemplate = `# This patch adds the args to allow expos - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true ` diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go index 136be031810..b0346411b43 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go @@ -235,6 +235,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -246,6 +247,7 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -256,6 +258,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -269,6 +274,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -315,16 +324,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -357,6 +377,14 @@ func main() { %s + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index 19a751ef511..f66e3002b5a 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -71,10 +71,9 @@ func GenerateV4(kbc *utils.TestContext) { monitorTlsPatch, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) + `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment`, "#")).To(Succeed()) } // GenerateV4WithoutMetrics implements a go/v4 plugin project defined by a TestContext. @@ -172,13 +171,12 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) { metricsTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - `#- path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + `#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment`, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), monitorTlsPatch, "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "cmd", "main.go"), - tlsConfigManager, "// ")).To(Succeed()) By("uncomment kustomization.yaml to enable network policy") ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), @@ -434,7 +432,3 @@ const monitorTlsPatch = `#patches: # - path: monitor_tls_patch.yaml # target: # kind: ServiceMonitor` - -const tlsConfigManager = `// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key"` diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go index 275bd9065c7..29e41d7b2c3 100644 --- a/testdata/project-v4-multigroup/cmd/main.go +++ b/testdata/project-v4-multigroup/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -70,8 +72,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -95,6 +98,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -108,6 +114,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -154,16 +164,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -348,6 +369,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2be15875d4 --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index a6b8d9bc0de..00000000000 --- a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-multigroup - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 69cc961d470..d3fb03b7ef7 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml index 2aaef6536f4..e504b415313 100644 --- a/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml @@ -2,3 +2,6 @@ - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true diff --git a/testdata/project-v4-multigroup/dist/install.yaml b/testdata/project-v4-multigroup/dist/install.yaml index 9e6adf59b9d..3c02224937b 100644 --- a/testdata/project-v4-multigroup/dist/install.yaml +++ b/testdata/project-v4-multigroup/dist/install.yaml @@ -2112,6 +2112,7 @@ spec: spec: containers: - args: + - --metrics-secure=true - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go index f01870085dc..8c105725da7 100644 --- a/testdata/project-v4-with-plugins/cmd/main.go +++ b/testdata/project-v4-with-plugins/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -45,8 +47,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -60,6 +63,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -73,6 +79,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -119,16 +129,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. - - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" - + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) + + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -194,6 +215,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2be15875d4 --- /dev/null +++ b/testdata/project-v4-with-plugins/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index ed2f033ddbb..00000000000 --- a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4-with-plugins - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml index cbbd06934bb..35bc92f493a 100644 --- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-plugins/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-plugins/config/default/manager_metrics_patch.yaml index 2aaef6536f4..e504b415313 100644 --- a/testdata/project-v4-with-plugins/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4-with-plugins/config/default/manager_metrics_patch.yaml @@ -2,3 +2,6 @@ - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true diff --git a/testdata/project-v4-with-plugins/dist/install.yaml b/testdata/project-v4-with-plugins/dist/install.yaml index 79528f60b21..d018ea04896 100644 --- a/testdata/project-v4-with-plugins/dist/install.yaml +++ b/testdata/project-v4-with-plugins/dist/install.yaml @@ -805,6 +805,7 @@ spec: spec: containers: - args: + - --metrics-secure=true - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go index 1a72e4c73c1..427cb1024ae 100644 --- a/testdata/project-v4/cmd/main.go +++ b/testdata/project-v4/cmd/main.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "flag" "os" + "path/filepath" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,6 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -48,8 +50,9 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + certWatcher *certwatcher.CertWatcher ) func init() { @@ -63,6 +66,9 @@ func init() { func main() { var metricsAddr string + var certDir string + var certName string + var certKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool @@ -76,6 +82,10 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&certDir, "cert-dir", "", + "The directory that contains the server key and certificate. If set, the metrics server will serve using the provided key and certificate.") + flag.StringVar(&certName, "cert-name", "tls.crt", "CertName is the server certificate name. Defaults to tls.crt") + flag.StringVar(&certKey, "cert-key", "tls.key", "KeyName is the server key name. Defaults to tls.key") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -122,16 +132,27 @@ func main() { // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization - // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically - // generate self-signed certificates for the metrics server. While convenient for development and testing, - // this setup is not recommended for production. + // If cert-name, cert-key, and cert-name are not set, controller-runtime generates self-signed certificates + // for the metrics server, suitable for development but not recommended for production. To use certificates + // created by CertManager, enable [METRICS-WITH-CERTS] at config/default/kustomization.yaml". + if len(certDir) > 0 { + setupLog.Info("using certificates for the metrics server", + "cert-dir", certDir, "cert-name", certName, "cert-key", certKey) - // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, - // you can uncomment the following lines to use the certificate managed by cert-manager. - // metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs" - // metricsServerOptions.CertName = "tls.crt" - // metricsServerOptions.KeyName = "tls.key" + var err error + certWatcher, err = certwatcher.New(filepath.Join(certDir, certName), filepath.Join(certDir, certKey)) + if err != nil { + setupLog.Error(err, "to initialize certificate watcher", "error", err) + os.Exit(1) + } + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = certWatcher.GetCertificate + }) + } else { + setupLog.Info("WARNING: Using self-signed certificates for the metrics server." + + "Consider informing your own certificates by setting the cert-dir, and cert-name and cert-key.") + } } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -230,6 +251,14 @@ func main() { } // +kubebuilder:scaffold:builder + if secureMetrics && certWatcher != nil { + setupLog.Info("Adding certificate watcher to manager") + if err := mgr.Add(certWatcher); err != nil { + setupLog.Error(err, "unable to add certificate watcher to manager") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2be15875d4 --- /dev/null +++ b/testdata/project-v4/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,35 @@ +# This patch adds the args and volumes to allow the manager to use the metrics-server certs +# Ensure the volumeMounts field exists by creating it if missing +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: [] +# Add the volume mount for the serving certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/serving-certs + name: metrics-certs + readOnly: true +# Add the cert-dir argument +- op: add + path: /spec/template/spec/containers/0/args/- + value: --cert-dir=/tmp/k8s-metrics-server/serving-certs +# Ensure the volumes field exists by creating it if missing +- op: add + path: /spec/template/spec/volumes + value: [] +# Add the volume for the serving certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml deleted file mode 100644 index 862a49388f2..00000000000 --- a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - app.kubernetes.io/name: project-v4 - app.kubernetes.io/managed-by: kustomize -spec: - template: - spec: - containers: - - name: manager - volumeMounts: - - mountPath: /tmp/k8s-metrics-server/metrics-certs - name: metrics-certs - readOnly: true - volumes: - - name: metrics-certs - secret: - secretName: metrics-server-cert diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index 3ad5f1c67e8..22a0611fdf6 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -42,9 +42,11 @@ patches: kind: Deployment # Uncomment the patches line if you enable Metrics and CertManager -# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. -# This patch will protect the metrics with certmanager self-signed certs. -#- path: certmanager_metrics_manager_patch.yaml +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4/config/default/manager_metrics_patch.yaml b/testdata/project-v4/config/default/manager_metrics_patch.yaml index 2aaef6536f4..e504b415313 100644 --- a/testdata/project-v4/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4/config/default/manager_metrics_patch.yaml @@ -2,3 +2,6 @@ - op: add path: /spec/template/spec/containers/0/args/0 value: --metrics-bind-address=:8443 +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure=true diff --git a/testdata/project-v4/dist/install.yaml b/testdata/project-v4/dist/install.yaml index 33f1336fa30..199cce55a10 100644 --- a/testdata/project-v4/dist/install.yaml +++ b/testdata/project-v4/dist/install.yaml @@ -675,6 +675,7 @@ spec: spec: containers: - args: + - --metrics-secure=true - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081