chore(base-cluster/kyverno)!: upgrade 🤮

charts/base-cluster/README.md.gotmpl

@@ -291,10 +291,22 @@ The new [t8s-cluster](../t8s-cluster) is going to provide these, the enduser can
 ignore this change.
 
 ### 4.x.x -> 5.0.0
+
 The condition if velero gets deployed changed. Velero will not be deployed if you
 have not configured its backupstoragelocation. This change is necessary, because
 in the current version of velero this value is mandatory. Please move
-your exiting backupstoragelocation configuration to the base-cluster chart if you
+your existing backupstoragelocation configuration to the base-cluster chart if you
 haven't already.
 
+### 5.x.x -> 6.0.0
+
+The kyverno 2.x.x -> 3.x.x upgrade cannot be done without manual intervention, see
+https://artifacthub.io/packages/helm/kyverno/kyverno#option-1---uninstallation-and-reinstallation
+
+So you have to backup your resources and delete the kyverno HelmReleases before the
+upgrade, they will be recreated in version 6.
+
+This also makes kyverno HA, so be aware that kyverno will need more resources in
+you cluster.
+
 {{ .Files.Get "values.md"  }}

charts/base-cluster/templates/kyverno/kyverno.yaml

@@ -1,5 +1,4 @@
 {{- if .Values.kyverno.enabled }}
-# https://github.com/kyverno/kyverno/tree/main/charts/kyverno
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
@@ -17,44 +16,43 @@ spec:
       namespace: monitoring
   {{- end }}
   values:
-    testImage:
-      registry: {{ $.Values.global.imageRegistry | default (include "base-cluster.defaultRegistry" (dict)) }}
     {{- if .Values.global.imageRegistry }}
-    image:
-      repository: {{ printf "%s/kyverno/kyverno" $.Values.global.imageRegistry }}
-    initImage:
-      repository: {{ printf "%s/kyverno/kyvernopre" $.Values.global.imageRegistry }}
+    global:
+      registry: {{ .Values.global.imageRegistry }}
     {{- end }}
     installCRDs: true
     networkPolicy:
       enabled: false
     webhooksCleanup:
       enable: true
-      image: {{ include "base-cluster.kubectl.image" . }}
     serviceMonitor:
       enabled: {{ .Values.monitoring.prometheus.enabled }}
       additionalLabels: {{- .Values.monitoring.labels | toYaml | nindent 8 }}
     priorityClassName: system-cluster-critical
     # this only works in version 3
-    {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled }}
     admissionController:
+      replicas: 3
+      {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled }}
       tracing: &tracingConfig
         enabled: true
         address: open-telemetry-collector-opentelemetry-collector.monitoring
         port: 14250 # jaeger-grpc
+      {{- end }}
     backgroundController:
+      replicas: 2
+      {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled }}
       tracing: *tracingConfig
+      {{- end }}
     reportsController:
+      replicas: 2
+      {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled }}
       tracing: *tracingConfig
-    {{- end }}
+      {{- end }}
     cleanupController:
+      replicas: 2
       {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled }}
       tracing: *tracingConfig
       {{- end }}
-      {{- if .Values.global.imageRegistry }}
-      image:
-        repository: {{ printf "%s/kyverno/cleanup-controller" $.Values.global.imageRegistry }}
-      {{- end }}
     podDisruptionBudget:
       enabled: true
       minAvailable: 1

charts/base-cluster/templates/kyverno/validation.tpl

@@ -0,0 +1,8 @@
+{{- $existingKyverno := lookup "helm.toolkit.fluxcd.io/v2beta1" "HelmRelease" "kyverno" "kyverno" -}}
+{{- $lastAttemptedRevision := dig "status" "lastAttemptedRevision" "" $existingKyverno }}
+{{- $lastAppliedRevision := dig "status" "lastAppliedRevision" "" $existingKyverno }}
+{{- if or $lastAppliedRevision $lastAttemptedRevision -}}
+  {{- if semverCompare "<3.x.x" ($lastAppliedRevision | default $lastAttemptedRevision) -}}
+    {{- fail "Cannot upgrade kyverno in-place, please backup your resources and delete it beforehand, see https://artifacthub.io/packages/helm/kyverno/kyverno#option-1---uninstallation-and-reinstallation" -}}
+  {{- end -}}
+{{- end -}}

charts/base-cluster/values.yaml

@@ -137,9 +137,8 @@ global:
     kyverno:
       url: https://kyverno.github.io/kyverno
       charts:
-        # TODO: migrate to v3 https://artifacthub.io/packages/helm/kyverno/kyverno#migrating-from-v2-to-v3
-        kyverno: 2.x.x
-        kyverno-policies: 2.x.x
+        kyverno: 3.x.x
+        kyverno-policies: 3.x.x
       condition: "{{ .Values.kyverno.enabled }}"
     cetic:
       url: https://cetic.github.io/helm-charts