From b6b52f29b3cc3fc05e5aa65c5af129e08f1ea9aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Tue, 6 Aug 2024 12:18:02 +0200 Subject: [PATCH 1/2] feature: add controlPlane.distro.k8s.version chart value to ease CAPI intergration --- chart/templates/_init-containers.tpl | 37 +++++- chart/tests/statefulset_test.yaml | 175 +++++++++++++++++++++++++++ chart/values.schema.json | 4 + chart/values.yaml | 11 ++ config/config.go | 12 ++ config/values.yaml | 1 + 6 files changed, 237 insertions(+), 3 deletions(-) diff --git a/chart/templates/_init-containers.tpl b/chart/templates/_init-containers.tpl index 9c6409bc67..3c74522a41 100644 --- a/chart/templates/_init-containers.tpl +++ b/chart/templates/_init-containers.tpl @@ -8,6 +8,37 @@ {{- end -}} {{- end -}} +{{/* Bump $defaultTag value whenever k8s version is bumped */}} +{{- define "vcluster.k8s.controllerManager.image.tag" -}} +{{- $defaultTag := "v1.30.2" -}} +{{- if and (not (empty .Values.controlPlane.distro.k8s.version)) (eq .Values.controlPlane.distro.k8s.controllerManager.image.tag $defaultTag) -}} +{{ .Values.controlPlane.distro.k8s.version}} +{{- else -}} +{{ .Values.controlPlane.distro.k8s.controllerManager.image.tag }} +{{- end -}} +{{- end -}} + +{{/* Bump $defaultTag value whenever k8s version is bumped */}} +{{- define "vcluster.k8s.apiServer.image.tag" -}} +{{- $defaultTag := "v1.30.2" -}} +{{- if and (not (empty .Values.controlPlane.distro.k8s.version)) (eq .Values.controlPlane.distro.k8s.apiServer.image.tag $defaultTag) -}} +{{ .Values.controlPlane.distro.k8s.version}} +{{- else -}} +{{ .Values.controlPlane.distro.k8s.apiServer.image.tag }} +{{- end -}} +{{- end -}} + + +{{/* Bump $defaultTag value whenever k8s version is bumped */}} +{{- define "vcluster.k8s.scheduler.image.tag" -}} +{{- $defaultTag := "v1.30.2" -}} +{{- if and (not (empty .Values.controlPlane.distro.k8s.version)) (eq .Values.controlPlane.distro.k8s.scheduler.image.tag $defaultTag) -}} +{{ .Values.controlPlane.distro.k8s.version}} +{{- else -}} +{{ .Values.controlPlane.distro.k8s.scheduler.image.tag }} +{{- end -}} +{{- end -}} + {{- define "vcluster.k8s.initContainers" -}} {{- include "vcluster.oldPlugins.initContainers" . }} {{- include "vcluster.plugins.initContainers" . }} @@ -32,7 +63,7 @@ {{ toYaml .Values.controlPlane.distro.k8s.resources | indent 4 }} {{- if .Values.controlPlane.distro.k8s.controllerManager.enabled }} - name: kube-controller-manager - image: "{{ include "vcluster.image" (dict "defaultImageRegistry" .Values.controlPlane.advanced.defaultImageRegistry "registry" .Values.controlPlane.distro.k8s.controllerManager.image.registry "repository" .Values.controlPlane.distro.k8s.controllerManager.image.repository "tag" .Values.controlPlane.distro.k8s.controllerManager.image.tag) }}" + image: "{{ include "vcluster.image" (dict "defaultImageRegistry" .Values.controlPlane.advanced.defaultImageRegistry "registry" .Values.controlPlane.distro.k8s.controllerManager.image.registry "repository" .Values.controlPlane.distro.k8s.controllerManager.image.repository "tag" (include "vcluster.k8s.controllerManager.image.tag" .)) }}" volumeMounts: - mountPath: /binaries name: binaries @@ -52,7 +83,7 @@ {{- end }} {{- if .Values.controlPlane.advanced.virtualScheduler.enabled }} - name: kube-scheduler-manager - image: "{{ include "vcluster.image" (dict "defaultImageRegistry" .Values.controlPlane.advanced.defaultImageRegistry "registry" .Values.controlPlane.distro.k8s.scheduler.image.registry "repository" .Values.controlPlane.distro.k8s.scheduler.image.repository "tag" .Values.controlPlane.distro.k8s.scheduler.image.tag) }}" + image: "{{ include "vcluster.image" (dict "defaultImageRegistry" .Values.controlPlane.advanced.defaultImageRegistry "registry" .Values.controlPlane.distro.k8s.scheduler.image.registry "repository" .Values.controlPlane.distro.k8s.scheduler.image.repository "tag" (include "vcluster.k8s.scheduler.image.tag" .)) }}" volumeMounts: - mountPath: /binaries name: binaries @@ -72,7 +103,7 @@ {{- end }} {{- if .Values.controlPlane.distro.k8s.apiServer.enabled }} - name: kube-apiserver - image: "{{ include "vcluster.image" (dict "defaultImageRegistry" .Values.controlPlane.advanced.defaultImageRegistry "registry" .Values.controlPlane.distro.k8s.apiServer.image.registry "repository" .Values.controlPlane.distro.k8s.apiServer.image.repository "tag" .Values.controlPlane.distro.k8s.apiServer.image.tag) }}" + image: "{{ include "vcluster.image" (dict "defaultImageRegistry" .Values.controlPlane.advanced.defaultImageRegistry "registry" .Values.controlPlane.distro.k8s.apiServer.image.registry "repository" .Values.controlPlane.distro.k8s.apiServer.image.repository "tag" (include "vcluster.k8s.apiServer.image.tag" .)) }}" volumeMounts: - mountPath: /binaries name: binaries diff --git a/chart/tests/statefulset_test.yaml b/chart/tests/statefulset_test.yaml index 385bcb699f..77bd4512f2 100644 --- a/chart/tests/statefulset_test.yaml +++ b/chart/tests/statefulset_test.yaml @@ -576,3 +576,178 @@ tests: name: data persistentVolumeClaim: claimName: my-custom-pvc + + - it: k8s version not set, default tag images used for apiServer and controllerManager + set: + controlPlane: + distro: + k8s: + enabled: true + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.30.2 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-apiserver:v1.30.2 + + - it: k8s version sets image tag for apiServer and controllerManager + set: + controlPlane: + distro: + k8s: + enabled: true + version: v1.35.999 + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.35.999 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-apiserver:v1.35.999 + + - it: k8s version set but overridden by image tag for apiServer and controllerManager + set: + controlPlane: + distro: + k8s: + enabled: true + version: v1.30.999 + apiServer: + image: + tag: v99912 + controllerManager: + image: + tag: v23123 + + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v23123 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-apiserver:v99912 + + - it: k8s not version set but image tags for apiServer and controllerManager set + set: + controlPlane: + distro: + k8s: + enabled: true + apiServer: + image: + tag: v99914 + controllerManager: + image: + tag: v23127 + + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v23127 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-apiserver:v99914 + + - it: k8s version not set, default tag images used for apiServer and controllerManager (virtual scheduler enabled) + set: + controlPlane: + distro: + k8s: + enabled: true + advanced: + virtualScheduler: + enabled: true + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.30.2 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-scheduler:v1.30.2 + - equal: + path: spec.template.spec.initContainers[3].image + value: registry.k8s.io/kube-apiserver:v1.30.2 + + - it: k8s version sets image tag for apiServer and controllerManager (virtual scheduler enabled) + set: + controlPlane: + distro: + k8s: + enabled: true + version: v1.35.999 + advanced: + virtualScheduler: + enabled: true + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.35.999 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-scheduler:v1.35.999 + - equal: + path: spec.template.spec.initContainers[3].image + value: registry.k8s.io/kube-apiserver:v1.35.999 + + - it: k8s version set but overridden by image tag for apiServer and controllerManager (virtual scheduler enabled) + set: + controlPlane: + distro: + k8s: + enabled: true + version: v1.30.999 + apiServer: + image: + tag: v99912 + controllerManager: + image: + tag: v23123 + scheduler: + image: + tag: v123654 + advanced: + virtualScheduler: + enabled: true + + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v23123 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-scheduler:v123654 + - equal: + path: spec.template.spec.initContainers[3].image + value: registry.k8s.io/kube-apiserver:v99912 + + - it: k8s not version set but image tags for apiServer and controllerManager set (virtual scheduler enabled) + set: + controlPlane: + distro: + k8s: + enabled: true + apiServer: + image: + tag: v99914 + controllerManager: + image: + tag: v23127 + scheduler: + image: + tag: v123656 + + advanced: + virtualScheduler: + enabled: true + + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v23127 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-scheduler:v123656 + - equal: + path: spec.template.spec.initContainers[3].image + value: registry.k8s.io/kube-apiserver:v99914 diff --git a/chart/values.schema.json b/chart/values.schema.json index 870e377855..e638514f4b 100755 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -921,6 +921,10 @@ "type": "boolean", "description": "Enabled specifies if the K8s distro should be enabled. Only one distro can be enabled at the same time." }, + "version": { + "type": "string", + "description": "Version specifies k8s components (scheduler, kube-controller-manager \u0026 apiserver) version.\nIt is a shortcut for controlPlane.distro.k8s.apiServer.image.tag,\ncontrolPlane.distro.k8s.controllerManager.image.tag and\ncontrolPlane.distro.k8s.scheduler.image.tag\nIf e.g. controlPlane.distro.k8s.version is set to v1.30.1 and\ncontrolPlane.distro.k8s.scheduler.image.tag\n(or controlPlane.distro.k8s.controllerManager.image.tag or controlPlane.distro.k8s.apiServer.image.tag)\nis set to v1.31.0,\nvalue from controlPlane.distro.k8s.\u003ccontrolPlane-component\u003e.image.tag will be used\n(where \u003ccontrolPlane-component is apiServer, controllerManager and scheduler)." + }, "apiServer": { "$ref": "#/$defs/DistroContainerEnabled", "description": "APIServer holds configuration specific to starting the api server." diff --git a/chart/values.yaml b/chart/values.yaml index 9f6d25994f..b78769c0c4 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -124,6 +124,17 @@ controlPlane: k8s: # Enabled specifies if the K8s distro should be enabled. Only one distro can be enabled at the same time. enabled: false + # Version specifies k8s components (scheduler, kube-controller-manager & apiserver) version. + # It is a shortcut for controlPlane.distro.k8s.apiServer.image.tag, + # controlPlane.distro.k8s.controllerManager.image.tag and + # controlPlane.distro.k8s.scheduler.image.tag + # If e.g. controlPlane.distro.k8s.version is set to v1.30.1 and + # controlPlane.distro.k8s.scheduler.image.tag + # (or controlPlane.distro.k8s.controllerManager.image.tag or controlPlane.distro.k8s.apiServer.image.tag) + # is set to v1.31.0, + # value from controlPlane.distro.k8s..image.tag will be used + # (where .image.tag will be used + // (where Date: Tue, 6 Aug 2024 17:36:25 +0200 Subject: [PATCH 2/2] add unit test to check if default image tags for k8s control plane components are in sync --- config/config_test.go | 23 +++++++++++++++++++++++ hack/test.sh | 3 +-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index aae2257748..34bbee7588 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2,6 +2,8 @@ package config import ( _ "embed" + "fmt" + "os" "strings" "testing" @@ -378,3 +380,24 @@ func TestConfig_IsProFeatureEnabled(t *testing.T) { }) } } + +func TestIfDefaultImagesVersionsAreInSync(t *testing.T) { + defaultConfig, err := NewDefaultConfig() + assert.NilError(t, err) + // this will fail when this test is moved or _init-containers.tpl is moved + initContainersTplFilePath := "../chart/templates/_init-containers.tpl" + tplBytes, err := os.ReadFile(initContainersTplFilePath) + + assert.NilError(t, err) + assert.Equal(t, defaultConfig.ControlPlane.Distro.K8S.ControllerManager.Image.Tag, defaultConfig.ControlPlane.Distro.K8S.APIServer.Image.Tag) + assert.Equal(t, defaultConfig.ControlPlane.Distro.K8S.ControllerManager.Image.Tag, defaultConfig.ControlPlane.Distro.K8S.Scheduler.Image.Tag) + assert.Equal(t, defaultConfig.ControlPlane.Distro.K8S.APIServer.Image.Tag, defaultConfig.ControlPlane.Distro.K8S.Scheduler.Image.Tag) + expectedDefaultTag := fmt.Sprintf("{{- $defaultTag := %q -}}", defaultConfig.ControlPlane.Distro.K8S.ControllerManager.Image.Tag) + got := strings.Count(string(tplBytes), expectedDefaultTag) + assert.Equal( + t, got, 3, + fmt.Sprintf("please update $defaultTag in %s so it's equal to the "+ + ".Values.controlPlane.distro.k8s.controllerManager.image.tag", + initContainersTplFilePath), + ) +} diff --git a/hack/test.sh b/hack/test.sh index 8a43c88e5b..b50781ecc7 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -7,10 +7,9 @@ export GOFLAGS=-mod=vendor # Test if we can build the program echo "Building virtual cluster..." go generate ./... && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build cmd/vcluster/main.go || exit 1 - + # List packages PKGS=$(go list ./... | grep -v -e /vendor/ -e /test) - echo "Start testing..." fail=false for pkg in $PKGS; do