From 919aa398a3d717987fbb8c926d446fc47ed700ab Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Tue, 5 Mar 2024 13:45:50 +0100 Subject: [PATCH 1/5] refactor: add new merged vcluster chart --- chart/.helmignore | 24 + chart/Chart.yaml | 26 + chart/README.md | 64 ++ chart/templates/_backingstore.tpl | 22 + chart/templates/_coredns.tpl | 50 + chart/templates/_distro.tpl | 39 + chart/templates/_helper.tpl | 7 + chart/templates/_init-containers.tpl | 325 +++++++ chart/templates/_persistence.tpl | 41 + chart/templates/_plugin.tpl | 122 +++ chart/templates/_rbac.tpl | 138 +++ chart/templates/clusterrole.yaml | 96 ++ chart/templates/clusterrolebinding.yaml | 27 + chart/templates/config-secret.yaml | 17 + chart/templates/coredns-configmap.yaml | 279 ++++++ chart/templates/etcd-headless-service.yaml | 34 + chart/templates/etcd-service.yaml | 33 + chart/templates/etcd-statefulset.yaml | 210 +++++ chart/templates/headless-service.yaml | 41 + chart/templates/ingress.yaml | 37 + chart/templates/limitrange.yaml | 35 + chart/templates/networkpolicy.yaml | 100 ++ chart/templates/resourcequota.yaml | 43 + chart/templates/role.yaml | 84 ++ chart/templates/rolebinding.yaml | 41 + chart/templates/service-monitor.yaml | 44 + chart/templates/service.yaml | 44 + chart/templates/serviceaccount.yaml | 28 + chart/templates/statefulset.yaml | 215 +++++ chart/templates/workload-serviceaccount.yaml | 29 + chart/tests/README.md | 9 + chart/tests/clusterrole_test.yaml | 176 ++++ chart/tests/clusterrolebinding_test.yaml | 141 +++ chart/tests/coredns-configmap_test.yaml | 328 +++++++ chart/tests/etcd-headless-service_test.yaml | 119 +++ chart/tests/etcd-service_test.yaml | 36 + chart/tests/etcd-statefulset_test.yaml | 121 +++ chart/tests/headless-service_test.yaml | 50 + chart/tests/ingress_test.yaml | 56 ++ chart/tests/limitrange_test.yaml | 30 + chart/tests/networkpolicy_test.yaml | 45 + chart/tests/resourcequota_test.yaml | 30 + chart/tests/role_test.yaml | 154 +++ chart/tests/rolebinding_test.yaml | 59 ++ chart/tests/service-monitor_test.yaml | 34 + chart/tests/service_test.yaml | 60 ++ chart/tests/serviceaccount_test.yaml | 61 ++ chart/tests/statefulset_test.yaml | 239 +++++ chart/tests/workload-serviceaccount_test.yaml | 64 ++ chart/values.yaml | 462 +++++++++ cmd/vcluster/cmd/start.go | 52 +- config/README.md | 1 + config/config.go | 890 ++++++++++++++++++ pkg/config/config.go | 245 ++--- pkg/{options => config}/controller_context.go | 8 +- pkg/config/defaults.go | 5 + pkg/{options => config}/flags.go | 22 +- pkg/{options => config}/options.go | 34 +- pkg/config/parse.go | 182 +--- pkg/{options => config}/pro_options.go | 6 +- pkg/config/validation.go | 273 ++++++ pkg/{pro => config}/validation_test.go | 54 +- pkg/controllers/generic/export_syncer.go | 17 +- pkg/controllers/generic/import_syncer.go | 13 +- pkg/controllers/generic/patcher.go | 6 +- .../k8sdefaultendpoint/k8sdefaultendpoint.go | 6 +- .../k8sdefaultendpoint/register.go | 4 +- pkg/controllers/register.go | 56 +- .../resources/namespaces/syncer.go | 8 +- pkg/controllers/resources/nodes/translate.go | 4 +- .../pods/translate/image_translator.go | 19 +- pkg/controllers/syncer/context/context.go | 6 +- pkg/genericsyncconfig/config.go | 3 + .../config_test.go | 6 +- pkg/genericsyncconfig/parse.go | 2 + pkg/{config => }/helmvalues/values_test.go | 0 pkg/leaderelection/leaderelection.go | 10 +- pkg/patches/conditions.go | 6 +- pkg/patches/patch.go | 26 +- pkg/patches/patch_test.go | 90 +- pkg/patches/patch_types.go | 26 +- pkg/plugin/plugin.go | 8 +- pkg/plugin/types/types.go | 4 +- pkg/plugin/v1/plugin.go | 6 +- pkg/plugin/v2/config.go | 7 +- pkg/plugin/v2/plugin.go | 19 +- pkg/plugin/v2/types.go | 4 +- pkg/pro/flags.go | 26 - pkg/pro/generic.go | 8 +- pkg/pro/integrated_coredns.go | 6 +- pkg/pro/license.go | 2 +- pkg/pro/noop.go | 4 +- pkg/pro/remote.go | 11 +- pkg/pro/validation.go | 97 -- pkg/setup/controller_context.go | 3 +- pkg/setup/proxy.go | 6 +- pkg/telemetry/collect.go | 10 +- pkg/telemetry/helpers.go | 6 +- pkg/telemetry/noop.go | 4 +- pkg/util/context/converter.go | 7 +- 100 files changed, 6140 insertions(+), 747 deletions(-) create mode 100644 chart/.helmignore create mode 100644 chart/Chart.yaml create mode 100644 chart/README.md create mode 100644 chart/templates/_backingstore.tpl create mode 100644 chart/templates/_coredns.tpl create mode 100644 chart/templates/_distro.tpl create mode 100644 chart/templates/_helper.tpl create mode 100644 chart/templates/_init-containers.tpl create mode 100644 chart/templates/_persistence.tpl create mode 100644 chart/templates/_plugin.tpl create mode 100644 chart/templates/_rbac.tpl create mode 100644 chart/templates/clusterrole.yaml create mode 100644 chart/templates/clusterrolebinding.yaml create mode 100644 chart/templates/config-secret.yaml create mode 100644 chart/templates/coredns-configmap.yaml create mode 100644 chart/templates/etcd-headless-service.yaml create mode 100644 chart/templates/etcd-service.yaml create mode 100644 chart/templates/etcd-statefulset.yaml create mode 100644 chart/templates/headless-service.yaml create mode 100644 chart/templates/ingress.yaml create mode 100644 chart/templates/limitrange.yaml create mode 100644 chart/templates/networkpolicy.yaml create mode 100644 chart/templates/resourcequota.yaml create mode 100644 chart/templates/role.yaml create mode 100644 chart/templates/rolebinding.yaml create mode 100644 chart/templates/service-monitor.yaml create mode 100644 chart/templates/service.yaml create mode 100644 chart/templates/serviceaccount.yaml create mode 100644 chart/templates/statefulset.yaml create mode 100644 chart/templates/workload-serviceaccount.yaml create mode 100644 chart/tests/README.md create mode 100644 chart/tests/clusterrole_test.yaml create mode 100644 chart/tests/clusterrolebinding_test.yaml create mode 100644 chart/tests/coredns-configmap_test.yaml create mode 100644 chart/tests/etcd-headless-service_test.yaml create mode 100644 chart/tests/etcd-service_test.yaml create mode 100644 chart/tests/etcd-statefulset_test.yaml create mode 100644 chart/tests/headless-service_test.yaml create mode 100644 chart/tests/ingress_test.yaml create mode 100644 chart/tests/limitrange_test.yaml create mode 100644 chart/tests/networkpolicy_test.yaml create mode 100644 chart/tests/resourcequota_test.yaml create mode 100644 chart/tests/role_test.yaml create mode 100644 chart/tests/rolebinding_test.yaml create mode 100644 chart/tests/service-monitor_test.yaml create mode 100644 chart/tests/service_test.yaml create mode 100644 chart/tests/serviceaccount_test.yaml create mode 100644 chart/tests/statefulset_test.yaml create mode 100644 chart/tests/workload-serviceaccount_test.yaml create mode 100644 chart/values.yaml create mode 100644 config/README.md create mode 100644 config/config.go rename pkg/{options => config}/controller_context.go (76%) create mode 100644 pkg/config/defaults.go rename pkg/{options => config}/flags.go (82%) rename pkg/{options => config}/options.go (74%) rename pkg/{options => config}/pro_options.go (87%) create mode 100644 pkg/config/validation.go rename pkg/{pro => config}/validation_test.go (70%) create mode 100644 pkg/genericsyncconfig/config.go rename pkg/{config => genericsyncconfig}/config_test.go (96%) create mode 100644 pkg/genericsyncconfig/parse.go rename pkg/{config => }/helmvalues/values_test.go (100%) delete mode 100644 pkg/pro/flags.go delete mode 100644 pkg/pro/validation.go diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 000000000..b6a3eb5b3 --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store + +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ + +# Common backup files +*.swp +*.bak +*.tmp +*~ + +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 000000000..516d9b124 --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: vcluster +description: vcluster - Virtual Kubernetes Clusters +home: https://vcluster.com +icon: https://static.loft.sh/branding/logos/vcluster/vertical/vcluster_vertical.svg +keywords: + - developer + - development + - sharing + - share + - multi-tenancy + - tenancy + - cluster + - space + - namespace + - vcluster + - vclusters +maintainers: + - name: Loft Labs, Inc. + email: info@loft.sh + url: https://twitter.com/loft_sh +sources: + - https://github.com/loft-sh/vcluster +type: application + +version: 0.0.1 # version is auto-generated by release pipeline diff --git a/chart/README.md b/chart/README.md new file mode 100644 index 000000000..5125c17c0 --- /dev/null +++ b/chart/README.md @@ -0,0 +1,64 @@ + +# vcluster + +## **[GitHub](https://github.com/loft-sh/vcluster)** • **[Website](https://www.vcluster.com)** • **[Quickstart](https://www.vcluster.com/docs/getting-started/setup)** • **[Documentation](https://www.vcluster.com/docs/what-are-virtual-clusters)** • **[Blog](https://loft.sh/blog)** • **[Twitter](https://twitter.com/loft_sh)** • **[Slack](https://slack.loft.sh/)** + +Create fully functional virtual Kubernetes clusters - Each vcluster runs inside a namespace of the underlying k8s cluster. It's cheaper than creating separate full-blown clusters and it offers better multi-tenancy and isolation than regular namespaces. + +## Prerequisites + +- Kubernetes 1.18+ +- Helm 3+ + +## Get Helm Repository Info + +```bash +helm repo add loft-sh https://charts.loft.sh +helm repo update +``` + +See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation. + +## Install Helm Chart + +```bash +helm upgrade [RELEASE_NAME] loft-sh/vcluster -n [RELEASE_NAMESPACE] --create-namespace --install +``` + +See [vcluster docs](https://vcluster.com/docs) for configuration options. + +See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation. + +## Connect to the vcluster + +In order to connect to the installed vcluster, please install [vcluster cli](https://www.vcluster.com/docs/getting-started/setup) and run: + +```bash +vcluster connect [RELEASE_NAME] -n [RELEASE_NAMESPACE] +``` + +## Uninstall Helm Chart + +```bash +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation. + +### Why Virtual Kubernetes Clusters? + +- **Cluster Scoped Resources**: much more powerful than simple namespaces (virtual clusters allow users to use CRDs, namespaces, cluster roles etc.) +- **Ease of Use**: usable in any Kubernetes cluster and created in seconds either via a single command or [cluster-api](https://github.com/loft-sh/cluster-api-provider-vcluster) +- **Cost Efficient**: much cheaper and efficient than "real" clusters (single pod and shared resources just like for namespaces) +- **Lightweight**: built upon the ultra-fast k3s distribution with minimal overhead per virtual cluster (other distributions work as well) +- **Strict isolation**: complete separate Kubernetes control plane and access point for each vcluster while still being able to share certain services of the underlying host cluster +- **Cluster Wide Permissions**: allow users to install apps which require cluster-wide permissions while being limited to actually just one namespace within the host cluster +- **Great for Testing**: allow you to test different Kubernetes versions inside a single host cluster which may have a different version than the virtual clusters + +Learn more on [www.vcluster.com](https://vcluster.com). + +![vcluster Intro](https://github.com/loft-sh/vcluster/raw/main/docs/static/media/vcluster-comparison.png) + +Learn more in the [documentation](https://vcluster.com/docs/what-are-virtual-clusters). diff --git a/chart/templates/_backingstore.tpl b/chart/templates/_backingstore.tpl new file mode 100644 index 000000000..870bef478 --- /dev/null +++ b/chart/templates/_backingstore.tpl @@ -0,0 +1,22 @@ +{{/* + is external etcd enabled? +*/}} +{{- define "vcluster.externalEtcd.enabled" -}} +{{- if and (eq (include "vcluster.distro" .) "k8s") (not .Values.controlPlane.backingStore.embeddedEtcd.enabled) -}} +{{- true -}} +{{- else if and (eq (include "vcluster.distro" .) "eks") (not .Values.controlPlane.backingStore.embeddedEtcd.enabled) -}} +{{- true -}} +{{- else if .Values.controlPlane.backingStore.externalEtcd.enabled -}} +{{- true -}} +{{- end -}} +{{- end -}} + +{{/* + migrate from external etcd? +*/}} +{{- define "vcluster.externalEtcd.migrate" -}} +{{- if and .Values.controlPlane.backingStore.embeddedEtcd.enabled .Values.controlPlane.backingStore.embeddedEtcd.migrateFromExternalEtcd -}} +{{- true -}} +{{- end -}} +{{- end -}} + diff --git a/chart/templates/_coredns.tpl b/chart/templates/_coredns.tpl new file mode 100644 index 000000000..41101bf1c --- /dev/null +++ b/chart/templates/_coredns.tpl @@ -0,0 +1,50 @@ +{{/* + Define a common coredns config +*/}} +{{- define "vcluster.corefile" -}} +Corefile: |- + {{- if .Values.controlPlane.coredns.overwriteConfig }} +{{ .Values.controlPlane.coredns.overwriteConfig | indent 8 }} + {{- else }} + .:1053 { + errors + health + ready + rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + {{- if .Values.controlPlane.coredns.embedded }} + kubeconfig /data/vcluster/admin.conf + {{- end }} + pods insecure + {{- if or .Values.networking.advanced.fallbackHostCluster (and .Values.controlPlane.coredns.embedded .Values.networking.resolveServices) }} + fallthrough cluster.local in-addr.arpa ip6.arpa + {{- else }} + fallthrough in-addr.arpa ip6.arpa + {{- end }} + } + {{- if and .Values.controlPlane.coredns.embedded .Values.networking.resolveServices }} + vcluster {{ toYaml .Values.networking.resolveServices | b64enc }} + {{- end }} + hosts /etc/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + {{- if .Values.networking.advanced.fallbackHostCluster }} + forward . {{`{{.HOST_CLUSTER_DNS}}`}} + {{- else if .Values.policies.networkPolicy.enabled }} + forward . /etc/resolv.conf {{ .Values.policies.networkPolicy.fallbackDns }} { + policy sequential + } + {{- else }} + forward . /etc/resolv.conf + {{- end }} + cache 30 + loop + loadbalance + } + + import /etc/coredns/custom/*.server + {{- end }} +{{- end -}} diff --git a/chart/templates/_distro.tpl b/chart/templates/_distro.tpl new file mode 100644 index 000000000..902d337a5 --- /dev/null +++ b/chart/templates/_distro.tpl @@ -0,0 +1,39 @@ +{{- define "vcluster.distro.env" -}} +{{- if and (eq (include "vcluster.distro" .) "k3s") .Values.controlPlane.distro.k3s.env -}} +{{ .Values.controlPlane.distro.k3s.env }} +{{- else if and (eq (include "vcluster.distro" .) "k8s") .Values.controlPlane.distro.k8s.env -}} +{{ .Values.controlPlane.distro.k8s.env }} +{{- else if and (eq (include "vcluster.distro" .) "k0s") .Values.controlPlane.distro.k0s.env -}} +{{ .Values.controlPlane.distro.k0s.env }} +{{- else if and (eq (include "vcluster.distro" .) "eks") .Values.controlPlane.distro.eks.env -}} +{{ .Values.controlPlane.distro.eks.env }} +{{- end -}} +{{- end -}} + +{{/* + vCluster Distro +*/}} +{{- define "vcluster.distro" -}} +{{- $distros := 0 -}} +{{- if .Values.controlPlane.distro.k3s.enabled -}} +k3s +{{- $distros = add1 $distros -}} +{{- end -}} +{{- if .Values.controlPlane.distro.k0s.enabled -}} +k0s +{{- $distros = add1 $distros -}} +{{- end -}} +{{- if .Values.controlPlane.distro.k8s.enabled -}} +k8s +{{- $distros = add1 $distros -}} +{{- end -}} +{{- if .Values.controlPlane.distro.eks.enabled -}} +eks +{{- $distros = add1 $distros -}} +{{- end -}} +{{- if eq $distros 0 -}} +k3s +{{- else if gt $distros 1 -}} +{{- fail "you can only enable one distro at the same time" -}} +{{- end -}} +{{- end -}} diff --git a/chart/templates/_helper.tpl b/chart/templates/_helper.tpl new file mode 100644 index 000000000..272f98912 --- /dev/null +++ b/chart/templates/_helper.tpl @@ -0,0 +1,7 @@ +{{- define "vcluster.controlPlane.image" -}} +{{- if .Values.controlPlane.statefulSet.image.tag -}} +{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.statefulSet.image.repository }}:{{ .Values.controlPlane.statefulSet.image.tag }} +{{- else -}} +{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.statefulSet.image.repository }}:{{ .Chart.Version }}-pro +{{- end -}} +{{- end -}} diff --git a/chart/templates/_init-containers.tpl b/chart/templates/_init-containers.tpl new file mode 100644 index 000000000..0a5733ff3 --- /dev/null +++ b/chart/templates/_init-containers.tpl @@ -0,0 +1,325 @@ +{{- define "vcluster.initContainers" -}} +{{- if eq (include "vcluster.distro" .) "k3s" -}} +{{ include "vcluster.k3s.initContainers" . }} +{{- else if eq (include "vcluster.distro" .) "k8s" -}} +{{ include "vcluster.k8s.initContainers" . }} +{{- else if eq (include "vcluster.distro" .) "k0s" -}} +{{ include "vcluster.k0s.initContainers" . }} +{{- else if eq (include "vcluster.distro" .) "eks" -}} +{{ include "vcluster.eks.initContainers" . }} +{{- end -}} +{{- end -}} + +{{- define "vcluster.eks.initContainers" -}} +{{- include "vcluster.oldPlugins.initContainers" . }} +{{- include "vcluster.plugins.initContainers" . }} +# this is needed because the k8s containers are distroless and thus we don't have any +# way of copying the binaries otherwise +- name: vcluster-copy + image: {{ include "vcluster.controlPlane.image" . | quote }} + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /bin/sh + args: + - -c + - "cp /vcluster /binaries/vcluster" + {{- if .Values.controlPlane.statefulSet.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.statefulSet.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.eks.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.eks.resources | indent 4 }} +{{- if not .Values.controlPlane.distro.eks.controllerManager.disabled }} +- name: kube-controller-manager + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.eks.controllerManager.image.repository }}:{{ .Values.controlPlane.distro.eks.controllerManager.image.tag }}" + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /binaries/vcluster + args: + - cp + - /usr/local/bin/kube-controller-manager + - /binaries/kube-controller-manager + {{- if .Values.controlPlane.distro.eks.controllerManager.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.eks.controllerManager.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.eks.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.eks.resources | indent 4 }} +{{- end }} +{{- if .Values.controlPlane.virtualScheduler.enabled }} +- name: kube-scheduler-manager + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.eks.scheduler.image.repository }}:{{ .Values.controlPlane.distro.eks.scheduler.image.tag }}" + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /binaries/vcluster + args: + - cp + - /usr/local/bin/kube-scheduler + - /binaries/kube-scheduler + {{- if .Values.controlPlane.distro.eks.scheduler.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.eks.scheduler.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.eks.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.eks.resources | indent 4 }} +{{- end }} +{{- if not .Values.controlPlane.distro.eks.apiServer.disabled }} +- name: kube-apiserver + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.eks.apiServer.image.repository }}:{{ .Values.controlPlane.distro.eks.apiServer.image.tag }}" + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /binaries/vcluster + args: + - cp + - /usr/local/bin/kube-apiserver + - /binaries/kube-apiserver + {{- if .Values.controlPlane.distro.eks.apiServer.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.eks.apiServer.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.eks.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.eks.resources | indent 4 }} +{{- end }} +{{- end -}} + +{{- define "vcluster.k8s.initContainers" -}} +{{- include "vcluster.oldPlugins.initContainers" . }} +{{- include "vcluster.plugins.initContainers" . }} +# this is needed because the k8s containers are distroless and thus we don't have any +# way of copying the binaries otherwise +- name: vcluster-copy + image: {{ include "vcluster.controlPlane.image" . | quote }} + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /bin/sh + args: + - -c + - "cp /vcluster /binaries/vcluster" + {{- if .Values.controlPlane.statefulSet.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.statefulSet.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.k8s.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.k8s.resources | indent 4 }} +{{- if not .Values.controlPlane.distro.k8s.controllerManager.disabled }} +- name: kube-controller-manager + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.k8s.controllerManager.image.repository }}:{{ .Values.controlPlane.distro.k8s.controllerManager.image.tag }}" + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /binaries/vcluster + args: + - cp + - /usr/local/bin/kube-controller-manager + - /binaries/kube-controller-manager + {{- if .Values.controlPlane.distro.k8s.controllerManager.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.k8s.controllerManager.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.k8s.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.k8s.resources | indent 4 }} +{{- end }} +{{- if .Values.controlPlane.virtualScheduler.enabled }} +- name: kube-scheduler-manager + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.k8s.scheduler.image.repository }}:{{ .Values.controlPlane.distro.k8s.scheduler.image.tag }}" + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /binaries/vcluster + args: + - cp + - /usr/local/bin/kube-scheduler + - /binaries/kube-scheduler + {{- if .Values.controlPlane.distro.k8s.scheduler.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.k8s.scheduler.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.k8s.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.k8s.resources | indent 4 }} +{{- end }} +{{- if not .Values.controlPlane.distro.k8s.apiServer.disabled }} +- name: kube-apiserver + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.k8s.apiServer.image.repository }}:{{ .Values.controlPlane.distro.k8s.apiServer.image.tag }}" + volumeMounts: + - mountPath: /binaries + name: binaries + command: + - /binaries/vcluster + args: + - cp + - /usr/local/bin/kube-apiserver + - /binaries/kube-apiserver + {{- if .Values.controlPlane.distro.k8s.apiServer.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.k8s.apiServer.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.k8s.securityContext | indent 4 }} + resources: +{{ toYaml .Values.controlPlane.distro.k8s.resources | indent 4 }} +{{- end }} +{{- end -}} + +{{- define "vcluster.k3s.initContainers" -}} +{{- include "vcluster.oldPlugins.initContainers" . }} +{{- include "vcluster.plugins.initContainers" . }} +- image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.k3s.image.repository }}:{{ .Values.controlPlane.distro.k3s.image.tag }}" + name: vcluster + command: + - /bin/sh + args: + - -c + - "cp /bin/k3s /binaries/k3s" + {{- if .Values.controlPlane.distro.k3s.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.k3s.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.k3s.securityContext | indent 4 }} + volumeMounts: + - name: binaries + mountPath: /binaries + resources: +{{ toYaml .Values.controlPlane.distro.k3s.resources | indent 4 }} +{{- end -}} + +{{- define "vcluster.k0s.initContainers" -}} +{{- include "vcluster.oldPlugins.initContainers" . }} +{{- include "vcluster.plugins.initContainers" . }} +- image: {{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.distro.k0s.image.repository }}:{{ .Values.controlPlane.distro.k0s.image.tag }} + name: vcluster + command: + - /bin/sh + args: + - -c + - "cp /usr/local/bin/k0s /binaries/k0s" + {{- if .Values.controlPlane.distro.k0s.imagePullPolicy }} + imagePullPolicy: {{ .Values.controlPlane.distro.k0s.imagePullPolicy }} + {{- end }} + securityContext: +{{ toYaml .Values.controlPlane.distro.k0s.securityContext | indent 4 }} + volumeMounts: + - name: binaries + mountPath: /binaries + resources: +{{ toYaml .Values.controlPlane.distro.k0s.resources | indent 4 }} +{{- end -}} + +{{/* + Plugin init container definition +*/}} +{{- define "vcluster.plugins.initContainers" -}} +{{- range $key, $container := .Values.plugins }} +{{- if not $container.image }} +{{- continue }} +{{- end }} +- image: {{ $.Values.controlPlane.advanced.defaultImageRegistry }}{{ $container.image }} + {{- if $container.name }} + name: {{ $container.name | quote }} + {{- else }} + name: {{ $key | quote }} + {{- end }} + {{- if $container.imagePullPolicy }} + imagePullPolicy: {{ $container.imagePullPolicy }} + {{- end }} + {{- if or $container.command $container.args }} + {{- if $container.command }} + command: + {{- range $commandIndex, $command := $container.command }} + - {{ $command | quote }} + {{- end }} + {{- end }} + {{- if $container.args }} + args: + {{- range $argIndex, $arg := $container.args }} + - {{ $arg | quote }} + {{- end }} + {{- end }} + {{- else }} + command: ["sh"] + args: ["-c", "cp -r /plugin /plugins/{{ $key }}"] + {{- end }} + securityContext: +{{ toYaml $container.securityContext | indent 4 }} + {{- if $container.volumeMounts }} + volumeMounts: +{{ toYaml $container.volumeMounts | indent 4 }} + {{- else }} + volumeMounts: + - mountPath: /plugins + name: plugins + {{- end }} + {{- if $container.resources }} + resources: +{{ toYaml $container.resources | indent 4 }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* + Old Plugin init container definition +*/}} +{{- define "vcluster.oldPlugins.initContainers" -}} +{{- range $key, $container := .Values.plugin }} +{{- if or (ne $container.version "v2") (not $container.image) -}} +{{- continue -}} +{{- end -}} +- image: {{ $.Values.controlPlane.advanced.defaultImageRegistry }}{{ $container.image }} + {{- if $container.name }} + name: {{ $container.name | quote }} + {{- else }} + name: {{ $key | quote }} + {{- end }} + {{- if $container.imagePullPolicy }} + imagePullPolicy: {{ $container.imagePullPolicy }} + {{- end }} + {{- if or $container.command $container.args }} + {{- if $container.command }} + command: + {{- range $commandIndex, $command := $container.command }} + - {{ $command | quote }} + {{- end }} + {{- end }} + {{- if $container.args }} + args: + {{- range $argIndex, $arg := $container.args }} + - {{ $arg | quote }} + {{- end }} + {{- end }} + {{- else }} + command: ["sh"] + args: ["-c", "cp -r /plugin /plugins/{{ $key }}"] + {{- end }} + securityContext: +{{ toYaml $container.securityContext | indent 4 }} + {{- if $container.volumeMounts }} + volumeMounts: +{{ toYaml $container.volumeMounts | indent 4 }} + {{- else }} + volumeMounts: + - mountPath: /plugins + name: plugins + {{- end }} + {{- if $container.resources }} + resources: +{{ toYaml $container.resources | indent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/chart/templates/_persistence.tpl b/chart/templates/_persistence.tpl new file mode 100644 index 000000000..b90a23c8b --- /dev/null +++ b/chart/templates/_persistence.tpl @@ -0,0 +1,41 @@ +{{/* + StatefulSet Persistence Options +*/}} +{{- define "vcluster.persistence" -}} +{{- if and .Values.controlPlane.backingStore.embeddedEtcd.enabled (include "vcluster.externalEtcd.enabled" .) -}} +{{- fail "embeddedEtcd and externalEtcd cannot be enabled at the same time together" }} +{{- end -}} +{{- if .Values.controlPlane.statefulSet.persistence.volumeClaimTemplates }} +{{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} +persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.controlPlane.statefulSet.persistence.volumeClaim.retentionPolicy }} +{{- end }} +volumeClaimTemplates: +{{ toYaml .Values.controlPlane.statefulSet.persistence.volumeClaimTemplates | indent 2 }} +{{- else if include "vcluster.persistence.volumeClaim.enabled" . }} +{{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} +persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.controlPlane.statefulSet.persistence.volumeClaim.retentionPolicy }} +{{- end }} +volumeClaimTemplates: +- metadata: + name: data + spec: + accessModes: {{ .Values.controlPlane.statefulSet.persistence.volumeClaim.accessModes }} + {{- if .Values.controlPlane.statefulSet.persistence.volumeClaim.storageClass }} + storageClassName: {{ .Values.controlPlane.statefulSet.persistence.volumeClaim.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.controlPlane.statefulSet.persistence.volumeClaim.size }} +{{- end }} +{{- end -}} + +{{/* + is persistence enabled? +*/}} +{{- define "vcluster.persistence.volumeClaim.enabled" -}} +{{- if and (not .Values.controlPlane.statefulSet.persistence.volumeClaim.disabled) (not (include "vcluster.externalEtcd.enabled" .)) -}} +{{- true -}} +{{- end -}} +{{- end -}} diff --git a/chart/templates/_plugin.tpl b/chart/templates/_plugin.tpl new file mode 100644 index 000000000..c2ca4ba2a --- /dev/null +++ b/chart/templates/_plugin.tpl @@ -0,0 +1,122 @@ +{{/* + Plugin volume mount definition +*/}} +{{- define "vcluster.plugins.volumeMounts" -}} +{{- $pluginFound := false -}} +{{- range $key, $container := .Values.plugin }} +{{- if or (ne $container.version "v2") (not $container.image) }} +{{- continue }} +{{- end }} +{{ $pluginFound = true }} +- mountPath: /plugins + name: plugins +{{- break }} +{{- end }} +{{- if eq $pluginFound false }} +{{- range $key, $container := .Values.plugins }} +{{- if not $container.image }} +{{- continue }} +{{- end }} +- mountPath: /plugins + name: plugins +{{- break }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Plugin volume definition +*/}} +{{- define "vcluster.plugins.volumes" -}} +{{- $pluginFound := false -}} +{{- range $key, $container := .Values.plugin }} +{{- if or (ne $container.version "v2") (not $container.image) }} +{{- continue }} +{{- end }} +{{ $pluginFound = true }} +- name: plugins + emptyDir: {} +{{- break }} +{{- end }} +{{- if eq $pluginFound false }} +{{- range $key, $container := .Values.plugins }} +{{- if not $container.image }} +{{- continue }} +{{- end }} +- name: plugins + emptyDir: {} +{{- break }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Sidecar container definition for the legacy syncer parts +*/}} +{{- define "vcluster.legacyPlugins.containers" -}} +{{- $counter := -1 -}} +{{- range $key, $container := .Values.plugin }} +{{- if eq $container.version "v2" }} +{{ continue }} +{{- end }} +{{- $counter = add1 $counter }} +- image: {{ $.Values.controlPlane.advanced.defaultImageRegistry }}{{ $container.image }} + {{- if $container.name }} + name: {{ $container.name | quote }} + {{- else }} + name: {{ $key | quote }} + {{- end }} + {{- if $container.imagePullPolicy }} + imagePullPolicy: {{ $container.imagePullPolicy }} + {{- end }} + {{- if $container.workingDir }} + workingDir: {{ $container.workingDir }} + {{- end }} + {{- if $container.command }} + command: + {{- range $commandIndex, $command := $container.command }} + - {{ $command | quote }} + {{- end }} + {{- end }} + {{- if $container.args }} + args: + {{- range $argIndex, $arg := $container.args }} + - {{ $arg | quote }} + {{- end }} + {{- end }} + {{- if $container.terminationMessagePath }} + terminationMessagePath: {{ $container.terminationMessagePath }} + {{- end }} + {{- if $container.terminationMessagePolicy }} + terminationMessagePolicy: {{ $container.terminationMessagePolicy }} + {{- end }} + env: + - name: VCLUSTER_PLUGIN_ADDRESS + value: "localhost:{{ add 14000 $counter }}" + - name: VCLUSTER_PLUGIN_NAME + value: "{{ $key }}" + {{- if $container.env }} +{{ toYaml $container.env | indent 4 }} + {{- end }} + envFrom: +{{ toYaml $container.envFrom | indent 4 }} + securityContext: +{{ toYaml $container.securityContext | indent 4 }} + lifecycle: +{{ toYaml $container.lifecycle | indent 4 }} + livenessProbe: +{{ toYaml $container.livenessProbe | indent 4 }} + readinessProbe: +{{ toYaml $container.readinessProbe | indent 4 }} + startupProbe: +{{ toYaml $container.startupProbe | indent 4 }} + volumeDevices: +{{ toYaml $container.volumeDevices | indent 4 }} + volumeMounts: +{{ toYaml $container.volumeMounts | indent 4 }} + {{- if $container.resources }} + resources: +{{ toYaml $container.resources | indent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/templates/_rbac.tpl b/chart/templates/_rbac.tpl new file mode 100644 index 000000000..4e501e772 --- /dev/null +++ b/chart/templates/_rbac.tpl @@ -0,0 +1,138 @@ +{{- define "vcluster.clusterRoleName" -}} +{{- printf "vc-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "vcluster.clusterRoleNameMultinamespace" -}} +{{- printf "vc-mn-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + Whether to create a cluster role or not +*/}} +{{- define "vcluster.createClusterRole" -}} +{{- if not .Values.rbac.clusterRole.disabled -}} +{{- if or + .Values.rbac.clusterRole.overwriteRules + (not (empty (include "vcluster.rbac.clusterRoleExtraRules" . ))) + (not (empty (include "vcluster.plugin.clusterRoleExtraRules" . ))) + (not (empty (include "vcluster.generic.clusterRoleExtraRules" . ))) + .Values.networking.replicateServices.fromHost + .Values.pro + .Values.sync.toHost.storageClasses.enabled + .Values.sync.toHost.persistentVolumes.enabled + .Values.sync.toHost.priorityClasses.enabled + .Values.sync.toHost.volumeSnapshots.enabled + .Values.sync.fromHost.ingressClasses.enabled + .Values.sync.fromHost.storageClasses.enabled + .Values.sync.fromHost.nodes.real.enabled + .Values.observability.metrics.proxy.nodes.enabled + .Values.experimental.multiNamespaceMode.enabled + .Values.networking.resolveServices -}} +{{- true -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* + Role rules defined on global level +*/}} +{{- define "vcluster.rbac.roleExtraRules" -}} +{{- if .Values.rbac.role.extraRules }} +{{- range $ruleIndex, $rule := .Values.rbac.role.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Role rules defined by plugins +*/}} +{{- define "vcluster.plugin.roleExtraRules" -}} +{{- range $key, $container := .Values.plugin }} +{{- if $container.rbac }} +{{- if $container.rbac.role }} +{{- if $container.rbac.role.extraRules }} +{{- range $ruleIndex, $rule := $container.rbac.role.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- range $key, $container := .Values.plugins }} +{{- if $container.rbac }} +{{- if $container.rbac.role }} +{{- if $container.rbac.role.extraRules }} +{{- range $ruleIndex, $rule := $container.rbac.role.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Cluster role rules defined by plugins +*/}} +{{- define "vcluster.plugin.clusterRoleExtraRules" -}} +{{- range $key, $container := .Values.plugin }} +{{- if $container.rbac }} +{{- if $container.rbac.clusterRole }} +{{- if $container.rbac.clusterRole.extraRules }} +{{- range $ruleIndex, $rule := $container.rbac.clusterRole.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- range $key, $container := .Values.plugins }} +{{- if $container.rbac }} +{{- if $container.rbac.clusterRole }} +{{- if $container.rbac.clusterRole.extraRules }} +{{- range $ruleIndex, $rule := $container.rbac.clusterRole.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Role rules defined in generic syncer +*/}} +{{- define "vcluster.generic.roleExtraRules" -}} +{{- if .Values.experimental.genericSync.role }} +{{- if .Values.experimental.genericSync.role.extraRules }} +{{- range $ruleIndex, $rule := .Values.experimental.genericSync.role.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Cluster role rules defined in generic syncer +*/}} +{{- define "vcluster.generic.clusterRoleExtraRules" -}} +{{- if .Values.experimental.genericSync.clusterRole }} +{{- if .Values.experimental.genericSync.clusterRole.extraRules }} +{{- range $ruleIndex, $rule := .Values.experimental.genericSync.clusterRole.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Cluster Role rules defined on global level +*/}} +{{- define "vcluster.rbac.clusterRoleExtraRules" -}} +{{- if .Values.rbac.clusterRole.extraRules }} +{{- range $ruleIndex, $rule := .Values.rbac.clusterRole.extraRules }} +- {{ toJson $rule }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml new file mode 100644 index 000000000..9c276d858 --- /dev/null +++ b/chart/templates/clusterrole.yaml @@ -0,0 +1,96 @@ +{{- if (include "vcluster.createClusterRole" . ) -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "vcluster.clusterRoleName" . }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +rules: + {{- if .Values.rbac.clusterRole.overwriteRules }} +{{ toYaml .Values.rbac.clusterRole.overwriteRules | indent 2 }} + {{- else }} + {{- if .Values.pro }} + - apiGroups: ["cluster.loft.sh", "storage.loft.sh"] + resources: ["features", "virtualclusters"] + verbs: ["get", "list", "watch"] + {{- end }} + {{- if or .Values.pro .Values.sync.fromHost.nodes.real.enabled }} + - apiGroups: [""] + resources: ["pods", "nodes", "nodes/status", "nodes/metrics", "nodes/stats", "nodes/proxy"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- if .Values.networking.resolveServices }} + - apiGroups: [""] + resources: [ "pods"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- if and .Values.sync.fromHost.nodes.real.enabled .Values.sync.fromHost.nodes.real.syncLabelsTaints }} + - apiGroups: [""] + resources: ["nodes", "nodes/status"] + verbs: ["update", "patch"] + {{- end }} + {{- if .Values.controlPlane.virtualScheduler.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses","csinodes","csidrivers","csistoragecapacities"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- if .Values.sync.toHost.persistentVolumes.enabled }} + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] + {{- end }} + {{- if .Values.sync.fromHost.ingressClasses.enabled }} + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- if .Values.sync.toHost.storageClasses.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] + {{- end }} + {{- if .Values.sync.fromHost.storageClasses.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- if .Values.sync.toHost.priorityClasses.enabled }} + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.sync.toHost.volumeSnapshots.enabled }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.networking.replicateServices.fromHost }} + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- if .Values.experimental.multiNamespaceMode.enabled }} + - apiGroups: [""] + resources: ["namespaces", "serviceaccounts"] + verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] + {{- end }} + {{- if .Values.observability.metrics.proxy.nodes.enabled }} + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes"] + verbs: ["get", "list"] + {{- end }} + {{- include "vcluster.plugin.clusterRoleExtraRules" . | indent 2 }} + {{- include "vcluster.generic.clusterRoleExtraRules" . | indent 2 }} + {{- include "vcluster.rbac.clusterRoleExtraRules" . | indent 2 }} + {{- end }} +{{- end }} diff --git a/chart/templates/clusterrolebinding.yaml b/chart/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..a1166ea68 --- /dev/null +++ b/chart/templates/clusterrolebinding.yaml @@ -0,0 +1,27 @@ +{{- if (include "vcluster.createClusterRole" . ) -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "vcluster.clusterRoleName" . }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + {{- if .Values.controlPlane.advanced.serviceAccount.name }} + name: {{ .Values.controlPlane.advanced.serviceAccount.name }} + {{- else }} + name: vc-{{ .Release.Name }} + {{- end }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "vcluster.clusterRoleName" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/templates/config-secret.yaml b/chart/templates/config-secret.yaml new file mode 100644 index 000000000..eb8170e5e --- /dev/null +++ b/chart/templates/config-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "vc-config-{{ .Release.Name }}" + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +type: Opaque +data: + config.yaml: {{ .Values | toYaml | b64enc | quote }} diff --git a/chart/templates/coredns-configmap.yaml b/chart/templates/coredns-configmap.yaml new file mode 100644 index 000000000..2f30eda4e --- /dev/null +++ b/chart/templates/coredns-configmap.yaml @@ -0,0 +1,279 @@ +{{- if and .Values.controlPlane.coredns.enabled (not .Values.experimental.isolatedControlPlane.headless) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: vc-coredns-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +data: +{{- if .Values.controlPlane.coredns.overwriteManifests }} + coredns.yaml: |- +{{ .Values.controlPlane.coredns.overwriteManifests | indent 4 }} +{{- else if .Values.controlPlane.coredns.embedded }} +{{ include "vcluster.corefile" . | indent 2 }} + coredns.yaml: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + NodeHosts: "" + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + {{- if .Values.controlPlane.coredns.service.annotations }} +{{ toYaml .Values.controlPlane.coredns.service.annotations | indent 8 }} + {{- end }} + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- if .Values.controlPlane.coredns.service.labels }} +{{ toYaml .Values.controlPlane.coredns.service.labels | indent 8 }} + {{- end }} + spec: +{{ toYaml .Values.controlPlane.coredns.service.spec | indent 6 }} + {{- if not .Values.controlPlane.coredns.service.spec.ports }} + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + {{- end }} +{{- else }} + coredns.yaml: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: +{{ include "vcluster.corefile" . | indent 6 }} + NodeHosts: "" + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + {{- if .Values.controlPlane.coredns.deployment.annotations }} + annotations: +{{ toYaml .Values.controlPlane.coredns.deployment.annotations | indent 8 }} + {{- end }} + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + {{- if .Values.controlPlane.coredns.deployment.labels }} +{{ toYaml .Values.controlPlane.coredns.deployment.labels | indent 8 }} + {{- end }} + spec: + replicas: {{ .Values.controlPlane.coredns.deployment.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + {{- if .Values.controlPlane.coredns.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.controlPlane.coredns.deployment.podAnnotations | indent 12 }} + {{- end }} + labels: + k8s-app: kube-dns + {{- if .Values.controlPlane.coredns.deployment.podLabels }} +{{ toYaml .Values.controlPlane.coredns.deployment.podLabels | indent 12 }} + {{- end }} + spec: + priorityClassName: "system-cluster-critical" + serviceAccountName: coredns + nodeSelector: + kubernetes.io/os: linux + {{- if .Values.controlPlane.coredns.deployment.nodeSelector }} +{{ toYaml .Values.controlPlane.coredns.deployment.nodeSelector | indent 12 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + k8s-app: kube-dns + {{- if .Values.policies.podSecurityStandard }} + securityContext: + seccompProfile: + type: RuntimeDefault + {{- end }} + containers: + - name: coredns + {{- if .Values.controlPlane.coredns.deployment.image }} + image: {{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ .Values.controlPlane.coredns.deployment.image }} + {{- else }} + image: {{`{{.IMAGE}}`}} + {{- end }} + imagePullPolicy: IfNotPresent + {{- if .Values.controlPlane.coredns.deployment.resources }} + resources: +{{ toYaml .Values.controlPlane.coredns.deployment.resources | indent 16 }} + {{- end }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + securityContext: + runAsNonRoot: true + runAsUser: {{`{{.RUN_AS_USER}}`}} + runAsGroup: {{`{{.RUN_AS_GROUP}}`}} + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + {{- if .Values.controlPlane.coredns.service.annotations }} +{{ toYaml .Values.controlPlane.coredns.service.annotations | indent 8 }} + {{- end }} + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + {{- if .Values.controlPlane.coredns.service.labels }} +{{ toYaml .Values.controlPlane.coredns.service.labels | indent 8 }} + {{- end }} + spec: +{{ toYaml .Values.controlPlane.coredns.service.spec | indent 6 }} + {{- if not .Values.controlPlane.coredns.service.spec.selector }} + selector: + k8s-app: kube-dns + {{- end }} + {{- if not .Values.controlPlane.coredns.service.spec.ports }} + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + {{- end }} +{{- end }} +{{- end }} diff --git a/chart/templates/etcd-headless-service.yaml b/chart/templates/etcd-headless-service.yaml new file mode 100644 index 000000000..4e8d439e6 --- /dev/null +++ b/chart/templates/etcd-headless-service.yaml @@ -0,0 +1,34 @@ +{{- if not .Values.experimental.isolatedControlPlane.headless }} +{{- if or (include "vcluster.externalEtcd.enabled" .) (include "vcluster.externalEtcd.migrate" .) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-etcd-headless + namespace: {{ .Release.Namespace }} + labels: + app: vcluster-etcd + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.controlPlane.backingStore.externalEtcd.headlessService.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + publishNotReadyAddresses: true + ports: + - name: etcd + port: 2379 + targetPort: 2379 + protocol: TCP + - name: peer + port: 2380 + targetPort: 2380 + protocol: TCP + clusterIP: None + selector: + app: vcluster-etcd + release: "{{ .Release.Name }}" +{{- end }} +{{- end }} diff --git a/chart/templates/etcd-service.yaml b/chart/templates/etcd-service.yaml new file mode 100644 index 000000000..e5a78d0c8 --- /dev/null +++ b/chart/templates/etcd-service.yaml @@ -0,0 +1,33 @@ +{{- if not .Values.experimental.isolatedControlPlane.headless }} +{{- if or (include "vcluster.externalEtcd.enabled" .) (include "vcluster.externalEtcd.migrate" .) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-etcd + namespace: {{ .Release.Namespace }} + labels: + app: vcluster-etcd + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.controlPlane.backingStore.externalEtcd.service.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: etcd + port: 2379 + targetPort: 2379 + protocol: TCP + - name: peer + port: 2380 + targetPort: 2380 + protocol: TCP + selector: + app: vcluster-etcd + release: {{ .Release.Name }} +{{- end }} +{{- end }} diff --git a/chart/templates/etcd-statefulset.yaml b/chart/templates/etcd-statefulset.yaml new file mode 100644 index 000000000..5b7758266 --- /dev/null +++ b/chart/templates/etcd-statefulset.yaml @@ -0,0 +1,210 @@ +{{- if not .Values.experimental.isolatedControlPlane.headless }} +{{- if or (include "vcluster.externalEtcd.enabled" .) (include "vcluster.externalEtcd.migrate" .) }} +{{- $externalEtcd := .Values.controlPlane.backingStore.externalEtcd.statefulSet }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }}-etcd + namespace: {{ .Release.Namespace }} + labels: + app: vcluster-etcd + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +{{- if $externalEtcd.labels }} +{{ toYaml $externalEtcd.labels | indent 4 }} +{{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations $externalEtcd.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + replicas: {{ $externalEtcd.highAvailability.replicas }} + podManagementPolicy: {{ $externalEtcd.scheduling.podManagementPolicy }} + serviceName: {{ .Release.Name }}-etcd-headless + {{- if eq $externalEtcd.persistence.volumeClaim.retentionPolicy "Delete" }} + {{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ $externalEtcd.persistence.volumeClaim.retentionPolicy }} + {{- end }} + {{- end }} + selector: + matchLabels: + app: vcluster-etcd + release: {{ .Release.Name }} + {{- if $externalEtcd.persistence.volumeClaimTemplates }} + volumeClaimTemplates: +{{ toYaml $externalEtcd.persistence.volumeClaimTemplates | indent 4 }} + {{- else if not $externalEtcd.persistence.volumeClaim.disabled }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: {{ $externalEtcd.persistence.volumeClaim.accessModes }} + {{- if $externalEtcd.persistence.volumeClaim.className }} + storageClassName: {{ $externalEtcd.persistence.volumeClaim.className }} + {{- end }} + resources: + requests: + storage: {{ $externalEtcd.persistence.volumeClaim.size }} + {{- end }} + template: + metadata: + {{- if $externalEtcd.pods.annotations }} + annotations: +{{ toYaml $externalEtcd.pods.annotations | indent 8 }} + {{- end }} + labels: + app: vcluster-etcd + release: {{ .Release.Name }} + {{- range $k, $v := $externalEtcd.pods.labels }} + {{ $k }}: {{ $v | quote }} + {{- end }} + spec: + terminationGracePeriodSeconds: 10 + {{- if $externalEtcd.scheduling.affinity }} + affinity: +{{ toYaml $externalEtcd.scheduling.affinity | indent 8 }} + {{- else }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + # if possible avoid scheduling more than one pod on one node + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vcluster-etcd + - key: release + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + # if possible avoid scheduling pod onto node that is in the same zone as one or more vcluster pods are running + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vcluster-etcd + - key: release + operator: In + values: + - {{ .Release.Name }} + topologyKey: topology.kubernetes.io/zone + {{- end }} + {{- if $externalEtcd.scheduling.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml $externalEtcd.scheduling.topologySpreadConstraints | indent 8 }} + {{- end }} + nodeSelector: +{{ toYaml $externalEtcd.scheduling.nodeSelector | indent 8 }} + tolerations: +{{ toYaml $externalEtcd.scheduling.tolerations | indent 8 }} + automountServiceAccountToken: false + {{- if .Values.controlPlane.advanced.serviceAccount.name }} + serviceAccountName: {{ .Values.controlPlane.advanced.serviceAccount.name }} + {{- else }} + serviceAccountName: vc-{{ .Release.Name }} + {{- end }} + volumes: + - name: certs + secret: + secretName: {{ .Release.Name }}-certs + {{- if $externalEtcd.persistence.addVolumes }} +{{ toYaml $externalEtcd.persistence.addVolumes | indent 8 }} + {{- end }} + {{- if $externalEtcd.persistence.volumeClaim.disabled }} + - name: data + emptyDir: {} + {{- end }} + {{- if $externalEtcd.scheduling.priorityClassName }} + priorityClassName: {{ $externalEtcd.scheduling.priorityClassName }} + {{- end }} + {{- if $externalEtcd.security.podSecurityContext }} + securityContext: +{{ toYaml $externalEtcd.security.podSecurityContext | indent 8 }} + {{- end }} + containers: + - name: etcd + image: "{{ .Values.controlPlane.advanced.defaultImageRegistry }}{{ $externalEtcd.image.repository }}:{{ $externalEtcd.image.tag }}" + command: + - etcd + - '--cert-file=/run/config/pki/etcd-server.crt' + - '--client-cert-auth=true' + - '--data-dir=/var/lib/etcd' + - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2379' + - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380' + {{- $releaseName := .Release.Name -}} + {{- $releaseNamespace := .Release.Namespace }} + - '--initial-cluster={{ range $index := untilStep 0 (int $externalEtcd.highAvailability.replicas) 1 }}{{ if (ne (int $index) 0) }},{{ end }}{{ $releaseName }}-etcd-{{ $index }}=https://{{ $releaseName }}-etcd-{{ $index }}.{{ $releaseName }}-etcd-headless.{{ $releaseNamespace }}:2380{{ end }}' + - '--initial-cluster-token={{ .Release.Name }}' + - '--initial-cluster-state=new' + - '--listen-client-urls=https://0.0.0.0:2379' + - '--listen-metrics-urls=http://0.0.0.0:2381' + - '--listen-peer-urls=https://0.0.0.0:2380' + - '--key-file=/run/config/pki/etcd-server.key' + - '--name=$(NAME)' + - '--peer-cert-file=/run/config/pki/etcd-peer.crt' + - '--peer-client-cert-auth=true' + - '--peer-key-file=/run/config/pki/etcd-peer.key' + - '--peer-trusted-ca-file=/run/config/pki/etcd-ca.crt' + - '--snapshot-count=10000' + - '--trusted-ca-file=/run/config/pki/etcd-ca.crt' + {{- range $f := $externalEtcd.extraArgs }} + - {{ $f | quote }} + {{- end }} + {{- if $externalEtcd.security.containerSecurityContext }} + securityContext: +{{ toYaml $externalEtcd.security.containerSecurityContext | indent 10 }} + {{- end }} + env: + - name: NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- if $externalEtcd.env }} +{{ toYaml $externalEtcd.env | indent 10 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /var/lib/etcd + - mountPath: /run/config/pki + name: certs + readOnly: true + {{- if $externalEtcd.persistence.addVolumeMounts }} +{{ toYaml $externalEtcd.persistence.addVolumeMounts | indent 10 }} + {{- end }} + resources: +{{ toYaml $externalEtcd.resources | indent 10 }} + livenessProbe: + httpGet: + path: /health + port: 2381 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 15 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 8 + startupProbe: + httpGet: + path: /health + port: 2381 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 15 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 24 + {{- if $externalEtcd.imagePullPolicy }} + imagePullPolicy: {{ $externalEtcd.imagePullPolicy }} + {{- end }} +{{- end }} +{{- end }} diff --git a/chart/templates/headless-service.yaml b/chart/templates/headless-service.yaml new file mode 100644 index 000000000..4f88162c4 --- /dev/null +++ b/chart/templates/headless-service.yaml @@ -0,0 +1,41 @@ +{{- if not .Values.experimental.isolatedControlPlane.headless }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-headless + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.headlessService.labels }} +{{ toYaml .Values.controlPlane.advanced.headlessService.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.headlessService.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + publishNotReadyAddresses: true + ports: + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + {{- if .Values.controlPlane.backingStore.embeddedEtcd.enabled }} + - name: etcd + port: 2379 + targetPort: 2379 + protocol: TCP + - name: peer + port: 2380 + targetPort: 2380 + protocol: TCP + {{- end }} + clusterIP: None + selector: + app: vcluster + release: "{{ .Release.Name }}" +{{- end }} diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml new file mode 100644 index 000000000..4ff847bbd --- /dev/null +++ b/chart/templates/ingress.yaml @@ -0,0 +1,37 @@ +{{- if .Values.controlPlane.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + {{- $annotations := merge dict .Values.controlPlane.ingress.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- if $annotations }} + annotations: + {{- toYaml $annotations | nindent 4 }} + {{- end }} + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.ingress.labels }} +{{ toYaml .Values.controlPlane.ingress.labels | indent 4 }} + {{- end }} +spec: + {{- if .Values.controlPlane.ingress.spec }} +{{ toYaml .Values.controlPlane.ingress.spec | indent 2 }} + {{- end }} + {{- if not .Values.controlPlane.ingress.spec.rules }} + rules: + - host: {{ .Values.controlPlane.ingress.host | quote }} + http: + paths: + - backend: + service: + name: {{ .Release.Name }} + port: + name: https + path: / + pathType: {{ .Values.controlPlane.ingress.pathType }} + {{- end }} +{{- end }} diff --git a/chart/templates/limitrange.yaml b/chart/templates/limitrange.yaml new file mode 100644 index 000000000..f057b28e2 --- /dev/null +++ b/chart/templates/limitrange.yaml @@ -0,0 +1,35 @@ +{{- if .Values.policies.limitRange.enabled }} +apiVersion: v1 +kind: LimitRange +metadata: + name: vc-{{ .Release.Name }} + {{- if .Values.experimental.syncSettings.targetNamespace }} + namespace: {{ .Values.experimental.syncSettings.targetNamespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.policies.limitRange.labels }} +{{ toYaml .Values.policies.limitRange.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.policies.limitRange.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + limits: + - default: + {{- range $key, $val := .Values.policies.limitRange.default }} + {{ $key }}: {{ $val | quote }} + {{- end }} + defaultRequest: + {{- range $key, $val := .Values.policies.limitRange.defaultRequest }} + {{ $key }}: {{ $val | quote }} + {{- end }} + type: Container +{{- end }} diff --git a/chart/templates/networkpolicy.yaml b/chart/templates/networkpolicy.yaml new file mode 100644 index 000000000..411aa544d --- /dev/null +++ b/chart/templates/networkpolicy.yaml @@ -0,0 +1,100 @@ +{{- if .Values.policies.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: vc-work-{{ .Release.Name }} + {{- if .Values.experimental.syncSettings.targetNamespace }} + namespace: {{ .Values.experimental.syncSettings.targetNamespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.policies.networkPolicy.labels }} +{{ toYaml .Values.policies.networkPolicy.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.policies.networkPolicy.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + vcluster.loft.sh/managed-by: {{ .Release.Name }} + egress: + # Allows outgoing connections to the vcluster control plane + - ports: + - port: 443 + - port: 8443 + to: + - podSelector: + matchLabels: + release: {{ .Release.Name }} + # Allows outgoing connections to DNS server + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # Allows outgoing connections to the internet or + # other vcluster workloads + - to: + - podSelector: + matchLabels: + vcluster.loft.sh/managed-by: {{ .Release.Name }} + - ipBlock: + cidr: {{ .Values.policies.networkPolicy.outgoingConnections.ipBlock.cidr }} + except: + {{- range .Values.policies.networkPolicy.outgoingConnections.ipBlock.except }} + - {{ . }} + {{- end }} + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: vc-cp-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.policies.networkPolicy.labels }} +{{ toYaml .Values.policies.networkPolicy.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.policies.networkPolicy.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + release: {{ .Release.Name }} + egress: + # Allows outgoing connections to all pods with + # port 443, 8443 or 6443. This is needed for host Kubernetes + # access + - ports: + - port: 443 + - port: 8443 + - port: 6443 + # Allows outgoing connections to all vcluster workloads + # or kube system dns server + - to: + - podSelector: {} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: 'kube-system' + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Egress + {{- end }} diff --git a/chart/templates/resourcequota.yaml b/chart/templates/resourcequota.yaml new file mode 100644 index 000000000..e5d1a689a --- /dev/null +++ b/chart/templates/resourcequota.yaml @@ -0,0 +1,43 @@ +{{- if .Values.policies.resourceQuota.enabled }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: vc-{{ .Release.Name }} + {{- if .Values.experimental.syncSettings.targetNamespace }} + namespace: {{ .Values.experimental.syncSettings.targetNamespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.policies.resourceQuota.labels }} +{{ toYaml .Values.policies.resourceQuota.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.policies.resourceQuota.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + {{- if .Values.policies.resourceQuota.quota }} + hard: + {{- range $key, $val := .Values.policies.resourceQuota.quota }} + {{ $key }}: {{ $val | quote }} + {{- end }} + {{- end }} + + {{- if .Values.policies.resourceQuota.scopeSelector.matchExpressions }} + scopeSelector: + matchExpressions: +{{- toYaml .Values.policies.resourceQuota.scopeSelector.matchExpressions | nindent 6 }} + {{- end }} + + {{- if .Values.policies.resourceQuota.scopes }} + scopes: +{{- toYaml .Values.policies.resourceQuota.scopes | nindent 4 }} + {{- end}} + +{{- end }} diff --git a/chart/templates/role.yaml b/chart/templates/role.yaml new file mode 100644 index 000000000..cde2b6c49 --- /dev/null +++ b/chart/templates/role.yaml @@ -0,0 +1,84 @@ +{{- if .Values.rbac.role.enabled }} +{{- if .Values.experimental.multiNamespaceMode.enabled }} +kind: ClusterRole +{{- else -}} +kind: Role +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +metadata: +{{- if .Values.experimental.multiNamespaceMode.enabled }} + name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} +{{- else }} + name: vc-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +{{- end }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +rules: + {{- if .Values.rbac.role.overwriteRules }} +{{ toYaml .Values.rbac.role.overwriteRules | indent 2 }} + {{- else }} + - apiGroups: [""] + resources: ["configmaps", "secrets", "services", "pods", "pods/attach", "pods/portforward", "pods/exec", "persistentvolumeclaims"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + - apiGroups: [""] + resources: ["pods/status", "pods/ephemeralcontainers"] + verbs: ["patch", "update"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "deployments"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["endpoints", "events", "pods/log"] + verbs: ["get", "list", "watch"] + {{- if or .Values.sync.toHost.endpoints.enabled .Values.experimental.isolatedControlPlane.headless }} + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["create", "delete", "patch", "update"] + {{- end }} + {{- if gt (int .Values.controlPlane.statefulSet.highAvailability.replicas) 1 }} + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.observability.metrics.proxy.pods.enabled }} + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + {{- end }} + {{- if .Values.sync.toHost.ingresses.enabled}} + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.sync.toHost.networkPolicies.enabled }} + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.sync.toHost.volumeSnapshots.enabled }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.sync.toHost.serviceAccounts.enabled }} + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- if .Values.sync.toHost.podDisruptionBudgets.enabled }} + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + {{- end }} + {{- include "vcluster.plugin.roleExtraRules" . | indent 2 }} + {{- include "vcluster.generic.roleExtraRules" . | indent 2 }} + {{- include "vcluster.rbac.roleExtraRules" . | indent 2 }} + {{- end }} +{{- end }} diff --git a/chart/templates/rolebinding.yaml b/chart/templates/rolebinding.yaml new file mode 100644 index 000000000..029510186 --- /dev/null +++ b/chart/templates/rolebinding.yaml @@ -0,0 +1,41 @@ +{{- if .Values.rbac.role.enabled }} +{{- if .Values.experimental.multiNamespaceMode.enabled }} +kind: ClusterRoleBinding +{{- else -}} +kind: RoleBinding +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +metadata: +{{- if .Values.experimental.multiNamespaceMode.enabled }} + name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} +{{- else }} + name: vc-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +{{- end }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + {{- if .Values.controlPlane.advanced.serviceAccount.name }} + name: {{ .Values.controlPlane.advanced.serviceAccount.name }} + {{- else }} + name: vc-{{ .Release.Name }} + {{- end }} + namespace: {{ .Release.Namespace }} +roleRef: +{{- if .Values.experimental.multiNamespaceMode.enabled }} + kind: ClusterRole + name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} +{{- else }} + kind: Role + name: vc-{{ .Release.Name }} +{{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/templates/service-monitor.yaml b/chart/templates/service-monitor.yaml new file mode 100644 index 000000000..ee172c4a4 --- /dev/null +++ b/chart/templates/service-monitor.yaml @@ -0,0 +1,44 @@ +{{- if .Values.controlPlane.observability.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: vc-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.observability.serviceMonitor.labels }} +{{ toYaml .Values.controlPlane.observability.serviceMonitor.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.observability.serviceMonitor.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: vcluster + release: "{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: "{{ .Release.Service }}" + endpoints: + - interval: 30s + port: https + path: /metrics + scheme: https + tlsConfig: + ca: + secret: + name: vc-{{ .Release.Name }} + key: certificate-authority + cert: + secret: + name: vc-{{ .Release.Name }} + key: client-certificate + keySecret: + name: vc-{{ .Release.Name }} + key: client-key +{{- end }} diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml new file mode 100644 index 000000000..1d836de8e --- /dev/null +++ b/chart/templates/service.yaml @@ -0,0 +1,44 @@ +{{- if .Values.controlPlane.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.service.labels }} +{{ toYaml .Values.controlPlane.service.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.globalMetadata.annotations .Values.controlPlane.service.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: +{{ toYaml .Values.controlPlane.service.spec | indent 2 }} + {{- if not .Values.controlPlane.service.spec.ports }} + ports: + - name: https + port: 443 + {{- if not .Values.experimental.isolatedControlPlane.headless }} + targetPort: 8443 + {{- end }} + nodePort: {{ .Values.controlPlane.service.httpsNodePort }} + protocol: TCP + - name: kubelet + port: 10250 + {{- if not .Values.experimental.isolatedControlPlane.headless }} + targetPort: 8443 + {{- end }} + nodePort: {{ .Values.controlPlane.service.kubeletNodePort }} + protocol: TCP + {{- end }} + {{- if and (not .Values.controlPlane.service.spec.selector) (not .Values.experimental.isolatedControlPlane.headless) }} + selector: + app: vcluster + release: {{ .Release.Name }} + {{- end }} +{{- end }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml new file mode 100644 index 000000000..6faa4d790 --- /dev/null +++ b/chart/templates/serviceaccount.yaml @@ -0,0 +1,28 @@ +{{- if .Values.controlPlane.advanced.serviceAccount.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + {{- if .Values.controlPlane.advanced.serviceAccount.name }} + name: {{ .Values.controlPlane.advanced.serviceAccount.name | quote }} + {{- else }} + name: vc-{{ .Release.Name }} + {{- end }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.serviceAccount.labels }} +{{ toYaml .Values.controlPlane.advanced.serviceAccount.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.serviceAccount.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- if $annotations }} + annotations: +{{- toYaml $annotations | nindent 4 }} + {{- end }} +{{- if .Values.controlPlane.advanced.serviceAccount.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.controlPlane.advanced.serviceAccount.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/chart/templates/statefulset.yaml b/chart/templates/statefulset.yaml new file mode 100644 index 000000000..6eeffb4c1 --- /dev/null +++ b/chart/templates/statefulset.yaml @@ -0,0 +1,215 @@ +{{- if not .Values.experimental.isolatedControlPlane.headless }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.statefulSet.labels }} +{{ toYaml .Values.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.statefulSet.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- if $annotations }} + annotations: +{{ toYaml $annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: vcluster + release: {{ .Release.Name }} + podManagementPolicy: {{ .Values.controlPlane.statefulSet.scheduling.podManagementPolicy }} + {{ include "vcluster.persistence" . | indent 2 }} + replicas: {{ .Values.controlPlane.statefulSet.highAvailability.replicas }} + template: + metadata: + {{- if .Values.controlPlane.statefulSet.pods.annotations }} + annotations: +{{ toYaml .Values.controlPlane.statefulSet.pods.annotations | indent 8 }} + {{- end }} + labels: + app: vcluster + release: {{ .Release.Name }} + {{- if .Values.controlPlane.statefulSet.pods.labels }} +{{ toYaml .Values.controlPlane.statefulSet.pods.labels | indent 8 }} + {{- end }} + spec: + terminationGracePeriodSeconds: 10 + {{- if .Values.controlPlane.statefulSet.scheduling.priorityClassName }} + priorityClassName: {{ .Values.controlPlane.statefulSet.scheduling.priorityClassName }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.security.podSecurityContext }} + securityContext: +{{ toYaml .Values.controlPlane.statefulSet.security.podSecurityContext | indent 8 }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.scheduling.nodeSelector }} + nodeSelector: +{{ toYaml .Values.controlPlane.statefulSet.scheduling.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.scheduling.affinity }} + affinity: +{{ toYaml .Values.controlPlane.statefulSet.scheduling.affinity | indent 8 }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.scheduling.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.controlPlane.statefulSet.scheduling.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.scheduling.tolerations }} + tolerations: +{{ toYaml .Values.controlPlane.statefulSet.scheduling.tolerations | indent 8 }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.scheduling.priorityClassName }} + priorityClassName: {{ .Values.controlPlane.statefulSet.scheduling.priorityClassName }} + {{- end }} + {{- if .Values.controlPlane.advanced.serviceAccount.name }} + serviceAccountName: {{ .Values.controlPlane.advanced.serviceAccount.name }} + {{- else }} + serviceAccountName: vc-{{ .Release.Name }} + {{- end }} + volumes: +{{- include "vcluster.plugins.volumes" . | indent 8 }} + - name: helm-cache + emptyDir: {} + - name: binaries + emptyDir: {} + - name: tmp + emptyDir: {} + - name: certs + emptyDir: {} + {{- if eq (include "vcluster.distro" .) "k0s" }} + - name: run-k0s + emptyDir: {} + {{- end }} + {{- if eq (include "vcluster.distro" .) "k3s" }} + - name: k3s-config + emptyDir: {} + {{- end }} + - name: vcluster-config + secret: + secretName: vc-config-{{ .Release.Name }} + {{- if .Values.controlPlane.coredns.enabled }} + - name: coredns + configMap: + name: vc-coredns-{{ .Release.Name }} + # - name: custom-config-volume + # configMap: + # name: coredns-custom + # optional: true + {{- end }} + {{- if not (include "vcluster.persistence.volumeClaim.enabled" .) }} + - name: data + emptyDir: {} + {{- end }} + {{- if .Values.controlPlane.statefulSet.persistence.addVolumes }} +{{ toYaml .Values.controlPlane.statefulSet.persistence.addVolumes | indent 8 }} + {{- end }} + {{- if (not .Values.experimental.syncSettings.disableSync) }} + initContainers: +{{ include "vcluster.initContainers" . | indent 8 }} + {{- end }} + containers: + - name: syncer + image: {{ include "vcluster.controlPlane.image" . | quote }} + imagePullPolicy: {{ .Values.controlPlane.statefulSet.imagePullPolicy }} + {{- if .Values.controlPlane.statefulSet.workingDir }} + workingDir: {{ .Values.controlPlane.statefulSet.workingDir }} + {{- end }} + {{- if .Values.controlPlane.statefulSet.command }} + command: +{{ toYaml .Values.controlPlane.statefulSet.command | indent 12 }} + {{- end }} + args: + {{- if .Values.controlPlane.statefulSet.args }} +{{ toYaml .Values.controlPlane.statefulSet.args | indent 12 }} + {{- else }} + - --config=/var/vcluster/config.yaml + {{- end }} + {{- if .Values.controlPlane.statefulSet.probes.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + failureThreshold: 60 + initialDelaySeconds: 60 + periodSeconds: 2 + {{- end }} + {{- if .Values.controlPlane.statefulSet.probes.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + failureThreshold: 60 + periodSeconds: 2 + {{- end }} + {{- if .Values.controlPlane.statefulSet.probes.startupProbe.enabled }} + startupProbe: + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + failureThreshold: 300 + periodSeconds: 6 + {{- end }} + {{- if .Values.controlPlane.statefulSet.security.containerSecurityContext }} + securityContext: +{{ toYaml .Values.controlPlane.statefulSet.security.containerSecurityContext | indent 12 }} + {{- end }} + resources: +{{ toYaml .Values.controlPlane.statefulSet.resources | indent 12 }} + env: + - name: VCLUSTER_NAME + value: "{{ .Release.Name }}" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.controlPlane.statefulSet.env }} +{{ toYaml .Values.controlPlane.statefulSet.env | indent 12 }} + {{- end }} +{{ include "vcluster.distro.env" . | indent 12 }} + volumeMounts: +{{- include "vcluster.plugins.volumeMounts" . | indent 12 }} + - name: data + mountPath: /data + - name: binaries + mountPath: /binaries + - name: certs + mountPath: /pki + - name: helm-cache + mountPath: /.cache/helm + {{- if eq (include "vcluster.distro" .) "k0s" }} + - name: run-k0s + mountPath: /run/k0s + {{- end }} + {{- if eq (include "vcluster.distro" .) "k3s" }} + - name: k3s-config + mountPath: /etc/rancher + {{- end }} + - name: vcluster-config + mountPath: /var/vcluster + - name: tmp + mountPath: /tmp + {{- if .Values.controlPlane.coredns.enabled }} + - name: coredns + mountPath: /manifests/coredns + readOnly: true + {{- end }} + {{- if .Values.controlPlane.statefulSet.persistence.addVolumeMounts }} +{{ toYaml .Values.controlPlane.statefulSet.persistence.addVolumeMounts | indent 12 }} + {{- end }} +{{- include "vcluster.legacyPlugins.containers" . | indent 8 }} +{{- end }} diff --git a/chart/templates/workload-serviceaccount.yaml b/chart/templates/workload-serviceaccount.yaml new file mode 100644 index 000000000..91a24fd0a --- /dev/null +++ b/chart/templates/workload-serviceaccount.yaml @@ -0,0 +1,29 @@ +{{- if .Values.controlPlane.advanced.workloadServiceAccount.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + {{- if .Values.controlPlane.advanced.workloadServiceAccount.name }} + name: {{ .Values.controlPlane.advanced.workloadServiceAccount.name | quote }} + {{- else }} + name: vc-workload-{{ .Release.Name }} + {{- end }} + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.workloadServiceAccount.labels }} +{{ toYaml .Values.controlPlane.advanced.workloadServiceAccount.labels | indent 4 }} + {{- end }} + {{- $annotations := merge dict .Values.controlPlane.advanced.workloadServiceAccount.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- if $annotations }} + annotations: +{{- toYaml $annotations | nindent 4 }} + {{- end }} +{{- $pullSecrets := concat .Values.controlPlane.advanced.serviceAccount.imagePullSecrets .Values.controlPlane.advanced.workloadServiceAccount.imagePullSecrets }} +{{- if $pullSecrets }} +imagePullSecrets: +{{ toYaml $pullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/chart/tests/README.md b/chart/tests/README.md new file mode 100644 index 000000000..4b87ed1ea --- /dev/null +++ b/chart/tests/README.md @@ -0,0 +1,9 @@ +Add [unittest plugin](https://github.com/helm-unittest/helm-unittest) via: +``` +helm plugin install https://github.com/helm-unittest/helm-unittest.git +``` + +Run tests via: +``` +helm unittest chart -d +``` diff --git a/chart/tests/clusterrole_test.yaml b/chart/tests/clusterrole_test.yaml new file mode 100644 index 000000000..91b1a74a9 --- /dev/null +++ b/chart/tests/clusterrole_test.yaml @@ -0,0 +1,176 @@ +suite: ClusterRoleBinding +templates: + - clusterrole.yaml + +tests: + - it: disable by default + asserts: + - hasDocuments: + count: 0 + + - it: enable by multi namespace mode + set: + experimental: + multiNamespaceMode: + enabled: true + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 1 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "namespaces", "serviceaccounts" ] + verbs: [ "create", "delete", "patch", "update", "get", "watch", "list" ] + + - it: override rules + set: + rbac: + clusterRole: + extraRules: + - apiGroups: [""] + resources: ["test123"] + verbs: ["test123"] + overwriteRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 1 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "test" ] + verbs: [ "test" ] + + - it: extra rules + set: + sync: + toHost: + priorityClasses: + enabled: true + rbac: + clusterRole: + extraRules: + - apiGroups: [ "" ] + resources: [ "test123" ] + verbs: [ "test123" ] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 2 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "test123" ] + verbs: [ "test123" ] + + - it: plugin rules + set: + plugin: + myTest: + rbac: + clusterRole: + extraRules: + - apiGroups: [ "" ] + resources: [ "test123" ] + verbs: [ "test123" ] + plugins: + myTest2: + rbac: + clusterRole: + extraRules: + - apiGroups: [ "" ] + resources: [ "test1234" ] + verbs: [ "test1234" ] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 2 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "test123" ] + verbs: [ "test123" ] + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "test1234" ] + verbs: [ "test1234" ] + + - it: replicate services + set: + networking: + replicateServices: + fromHost: + - abc: test + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 1 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "services", "endpoints" ] + verbs: [ "get", "watch", "list" ] + + - it: real nodes + set: + sync: + fromHost: + nodes: + real: + enabled: true + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 1 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "pods", "nodes", "nodes/status", "nodes/metrics", "nodes/stats", "nodes/proxy" ] + verbs: [ "get", "watch", "list" ] + + - it: legacy pro + set: + pro: true + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 2 + - contains: + path: rules + content: + apiGroups: [ "" ] + resources: [ "pods", "nodes", "nodes/status", "nodes/metrics", "nodes/stats", "nodes/proxy" ] + verbs: [ "get", "watch", "list" ] + - contains: + path: rules + content: + apiGroups: [ "cluster.loft.sh", "storage.loft.sh" ] + resources: [ "features", "virtualclusters" ] + verbs: [ "get", "list", "watch" ] + diff --git a/chart/tests/clusterrolebinding_test.yaml b/chart/tests/clusterrolebinding_test.yaml new file mode 100644 index 000000000..89a751b1b --- /dev/null +++ b/chart/tests/clusterrolebinding_test.yaml @@ -0,0 +1,141 @@ +suite: ClusterRoleBinding +templates: + - clusterrolebinding.yaml + +tests: + - it: disable by default + asserts: + - hasDocuments: + count: 0 + + - it: enable by multi namespace mode + set: + experimental: + multiNamespaceMode: + enabled: true + asserts: + - hasDocuments: + count: 1 + + - it: enable by from syncer + set: + sync: + fromHost: + ingressClasses: + enabled: true + asserts: + - hasDocuments: + count: 1 + + - it: enable by generic sync + set: + experimental: + genericSync: + clusterRole: + extraRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + asserts: + - hasDocuments: + count: 1 + + - it: enable by plugins + set: + plugins: + test: + rbac: + clusterRole: + extraRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + + - it: enable by plugin + set: + plugin: + test: + rbac: + clusterRole: + extraRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + + - it: enable by legacy api key + set: + pro: true + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: ClusterRoleBinding + - equal: + path: metadata.name + value: vc-my-release-v-my-namespace + - notExists: + path: metadata.namespace + + - it: enable by extra rules + set: + rbac: + clusterRole: + extraRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: ClusterRoleBinding + - equal: + path: metadata.name + value: vc-my-release-v-my-namespace + - notExists: + path: metadata.namespace + + - it: enable by overwrite rules + set: + rbac: + clusterRole: + overwriteRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: ClusterRoleBinding + - equal: + path: metadata.name + value: vc-my-release-v-my-namespace + - notExists: + path: metadata.namespace + + diff --git a/chart/tests/coredns-configmap_test.yaml b/chart/tests/coredns-configmap_test.yaml new file mode 100644 index 000000000..0b984e7db --- /dev/null +++ b/chart/tests/coredns-configmap_test.yaml @@ -0,0 +1,328 @@ +suite: CoreDNS Configmap +templates: + - coredns-configmap.yaml + +tests: + - it: should create configmap + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: vc-coredns-my-release + - equal: + path: metadata.namespace + value: my-namespace + + - it: should create correct external coredns config + asserts: + - hasDocuments: + count: 1 + - notExists: + path: data.Corefile + - equal: + path: data["coredns.yaml"] + value: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: |- + .:1053 { + errors + health + ready + rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough cluster.local in-addr.arpa ip6.arpa + } + hosts /etc/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + forward . {{.HOST_CLUSTER_DNS}} + cache 30 + loop + loadbalance + } + + import /etc/coredns/custom/*.server + NodeHosts: "" + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + priorityClassName: "system-cluster-critical" + serviceAccountName: coredns + nodeSelector: + kubernetes.io/os: linux + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + k8s-app: kube-dns + containers: + - name: coredns + image: {{.IMAGE}} + imagePullPolicy: IfNotPresent + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + securityContext: + runAsNonRoot: true + runAsUser: {{.RUN_AS_USER}} + runAsGroup: {{.RUN_AS_GROUP}} + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + type: ClusterIP + selector: + k8s-app: kube-dns + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + + - it: should create correct custom configmap + set: + controlPlane: + coredns: + embedded: true + overwriteManifests: |- + abc + asserts: + - hasDocuments: + count: 1 + - equal: + path: data["coredns.yaml"] + value: |- + abc + + - it: should create correct custom configmap + set: + controlPlane: + coredns: + embedded: true + overwriteConfig: |- + abc + asserts: + - hasDocuments: + count: 1 + - equal: + path: data.Corefile + value: |- + abc + + - it: should create correct embedded configmap + set: + controlPlane: + coredns: + embedded: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: data.Corefile + value: |- + .:1053 { + errors + health + ready + rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + kubeconfig /data/vcluster/admin.conf + pods insecure + fallthrough cluster.local in-addr.arpa ip6.arpa + } + hosts /etc/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + forward . {{.HOST_CLUSTER_DNS}} + cache 30 + loop + loadbalance + } + + import /etc/coredns/custom/*.server + - equal: + path: data["coredns.yaml"] + value: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + NodeHosts: "" + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + type: ClusterIP + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP diff --git a/chart/tests/etcd-headless-service_test.yaml b/chart/tests/etcd-headless-service_test.yaml new file mode 100644 index 000000000..acd9b7ddb --- /dev/null +++ b/chart/tests/etcd-headless-service_test.yaml @@ -0,0 +1,119 @@ +suite: External etcd headless Service +templates: + - etcd-headless-service.yaml + +tests: + - it: check disabled + asserts: + - hasDocuments: + count: 0 + + - it: enable for k3s & defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + backingStore: + externalEtcd: + enabled: true + headlessService: + annotations: + test: test + distro: + k3s: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-etcd-headless + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: metadata.annotations.test + value: test + + - it: enable for k0s & defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + backingStore: + externalEtcd: + enabled: true + headlessService: + annotations: + test: test + distro: + k0s: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-etcd-headless + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: metadata.annotations.test + value: test + + - it: enable for eks & defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + backingStore: + externalEtcd: + headlessService: + annotations: + test: test + distro: + eks: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-etcd-headless + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: metadata.annotations.test + value: test + + - it: enable for k8s & defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + backingStore: + externalEtcd: + headlessService: + annotations: + test: test + distro: + k8s: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-etcd-headless + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: metadata.annotations.test + value: test diff --git a/chart/tests/etcd-service_test.yaml b/chart/tests/etcd-service_test.yaml new file mode 100644 index 000000000..8fa039922 --- /dev/null +++ b/chart/tests/etcd-service_test.yaml @@ -0,0 +1,36 @@ +suite: External etcd Service +templates: + - etcd-service.yaml + +tests: + - it: check disabled + asserts: + - hasDocuments: + count: 0 + + - it: enable for k8s & defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + backingStore: + externalEtcd: + service: + annotations: + test: test + distro: + k8s: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-etcd + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: metadata.annotations.test + value: test diff --git a/chart/tests/etcd-statefulset_test.yaml b/chart/tests/etcd-statefulset_test.yaml new file mode 100644 index 000000000..7cf049a3d --- /dev/null +++ b/chart/tests/etcd-statefulset_test.yaml @@ -0,0 +1,121 @@ +suite: External etcd StatefulSet +templates: + - etcd-statefulset.yaml + +tests: + - it: check disabled + asserts: + - hasDocuments: + count: 0 + + - it: check disabled headless + set: + controlPlane: + distro: + k8s: + enabled: true + experimental: + isolatedControlPlane: + headless: true + asserts: + - hasDocuments: + count: 0 + + - it: enabled for k3s & non persistent + set: + controlPlane: + backingStore: + externalEtcd: + enabled: true + statefulSet: + extraArgs: + - "extra-arg" + env: + - name: my-new-env + persistence: + volumeClaim: + disabled: true + addVolumes: + - name: my-new-volume + addVolumeMounts: + - name: my-new-volume + asserts: + - hasDocuments: + count: 1 + - contains: + path: spec.template.spec.volumes + content: + name: "data" + emptyDir: {} + count: 1 + - notExists: + path: spec.volumeClaimTemplates + - contains: + path: spec.template.spec.volumes + content: + name: "my-new-volume" + count: 1 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: "my-new-volume" + count: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: "my-new-env" + count: 1 + - contains: + path: spec.template.spec.containers[0].command + content: "extra-arg" + count: 1 + + - it: enable for k8s & defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + backingStore: + externalEtcd: + statefulSet: + highAvailability: + replicas: 3 + annotations: + test: test + distro: + k8s: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release-etcd + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: metadata.annotations.test + value: test + - equal: + path: spec.replicas + value: 3 + - lengthEqual: + path: spec.volumeClaimTemplates + count: 1 + - lengthEqual: + path: spec.template.spec.volumes + count: 1 + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + - lengthEqual: + path: spec.template.spec.containers[0].env + count: 1 + - notExists: + path: spec.template.spec.containers[0].args + - contains: + path: spec.template.spec.containers[0].command + content: "--initial-cluster=my-release-etcd-0=https://my-release-etcd-0.my-release-etcd-headless.my-namespace:2380,my-release-etcd-1=https://my-release-etcd-1.my-release-etcd-headless.my-namespace:2380,my-release-etcd-2=https://my-release-etcd-2.my-release-etcd-headless.my-namespace:2380" + count: 1 diff --git a/chart/tests/headless-service_test.yaml b/chart/tests/headless-service_test.yaml new file mode 100644 index 000000000..f5302843d --- /dev/null +++ b/chart/tests/headless-service_test.yaml @@ -0,0 +1,50 @@ +suite: ControlPlane StatefulSet +templates: + - headless-service.yaml + +tests: + - it: should not create control-plane + set: + experimental: + isolatedControlPlane: + headless: true + asserts: + - hasDocuments: + count: 0 + + - it: name + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.ports + count: 1 + - equal: + path: metadata.name + value: my-release-headless + - equal: + path: metadata.namespace + value: my-namespace + + - it: embedded-etcd + set: + controlPlane: + backingStore: + embeddedEtcd: + enabled: true + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.ports + count: 3 + - equal: + path: spec.ports[1].name + value: etcd + - equal: + path: spec.ports[2].name + value: peer + diff --git a/chart/tests/ingress_test.yaml b/chart/tests/ingress_test.yaml new file mode 100644 index 000000000..c5a11d8c6 --- /dev/null +++ b/chart/tests/ingress_test.yaml @@ -0,0 +1,56 @@ +suite: ControlPlane Ingress +templates: + - ingress.yaml + +tests: + - it: should not create ingress by default + asserts: + - hasDocuments: + count: 0 + + - it: ingress defaults + set: + controlPlane: + ingress: + enabled: true + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release + - equal: + path: metadata.namespace + value: my-namespace + + - it: overwrite ingress tls + set: + controlPlane: + ingress: + enabled: true + host: my-host + spec: + tls: + - hosts: + - ingress-demo.example.com + secretName: ingress-demo-tls + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.tls + count: 1 + - equal: + path: spec.rules[0].host + value: my-host + - contains: + path: spec.tls + count: 1 + content: + hosts: + - ingress-demo.example.com + secretName: ingress-demo-tls + diff --git a/chart/tests/limitrange_test.yaml b/chart/tests/limitrange_test.yaml new file mode 100644 index 000000000..aec52a717 --- /dev/null +++ b/chart/tests/limitrange_test.yaml @@ -0,0 +1,30 @@ +suite: LimitRange +templates: + - limitrange.yaml + +tests: + - it: should not create limit range by default + asserts: + - hasDocuments: + count: 0 + + - it: check defaults + release: + name: my-release + namespace: my-namespace + set: + policies: + limitRange: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: vc-my-release + - equal: + path: metadata.namespace + value: my-namespace + - lengthEqual: + path: spec.limits + count: 1 diff --git a/chart/tests/networkpolicy_test.yaml b/chart/tests/networkpolicy_test.yaml new file mode 100644 index 000000000..21f1a1f58 --- /dev/null +++ b/chart/tests/networkpolicy_test.yaml @@ -0,0 +1,45 @@ +suite: NetworkPolicy +templates: + - networkpolicy.yaml + +tests: + - it: should not create network policy by default + asserts: + - hasDocuments: + count: 0 + + - it: check defaults + release: + name: my-release + namespace: my-namespace + set: + policies: + networkPolicy: + enabled: true + asserts: + - hasDocuments: + count: 2 + - documentIndex: 0 + equal: + path: metadata.name + value: vc-work-my-release + - documentIndex: 1 + equal: + path: metadata.name + value: vc-cp-my-release + - documentIndex: 0 + equal: + path: metadata.namespace + value: my-namespace + - documentIndex: 1 + equal: + path: metadata.namespace + value: my-namespace + - documentIndex: 0 + lengthEqual: + path: spec.egress + count: 3 + - documentIndex: 1 + lengthEqual: + path: spec.egress + count: 2 diff --git a/chart/tests/resourcequota_test.yaml b/chart/tests/resourcequota_test.yaml new file mode 100644 index 000000000..8f36f6d06 --- /dev/null +++ b/chart/tests/resourcequota_test.yaml @@ -0,0 +1,30 @@ +suite: ResourceQuota +templates: + - resourcequota.yaml + +tests: + - it: should not create resource quota by default + asserts: + - hasDocuments: + count: 0 + + - it: check defaults + release: + name: my-release + namespace: my-namespace + set: + policies: + resourceQuota: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: vc-my-release + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: spec.hard["requests.cpu"] + value: "10" diff --git a/chart/tests/role_test.yaml b/chart/tests/role_test.yaml new file mode 100644 index 000000000..e12bde988 --- /dev/null +++ b/chart/tests/role_test.yaml @@ -0,0 +1,154 @@ +suite: Role +templates: + - role.yaml + +tests: + - it: check disabled + set: + rbac: + role: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: check overwrite rules + set: + rbac: + role: + overwriteRules: + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "create" ] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 1 + - contains: + path: rules + count: 1 + content: + apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "create" ] + + - it: check plugin extra rules + set: + plugin: + test123: + rbac: + role: + extraRules: + - apiGroups: [ "" ] + resources: [ "test123" ] + verbs: [ "test123" ] + plugins: + test: + rbac: + role: + extraRules: + - apiGroups: [ "" ] + resources: [ "test" ] + verbs: [ "test" ] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 7 + - contains: + path: rules + count: 1 + content: + apiGroups: [ "" ] + resources: [ "test123" ] + verbs: [ "test123" ] + - contains: + path: rules + count: 1 + content: + apiGroups: [ "" ] + resources: [ "test" ] + verbs: [ "test" ] + + - it: check generic sync + set: + experimental: + genericSync: + role: + extraRules: + - apiGroups: [ "" ] + resources: [ "test" ] + verbs: [ "test" ] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 6 + - contains: + path: rules + count: 1 + content: + apiGroups: [ "" ] + resources: [ "test" ] + verbs: [ "test" ] + + - it: check extra rules + set: + rbac: + role: + extraRules: + - apiGroups: [""] + resources: ["test"] + verbs: ["test"] + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: rules + count: 6 + - contains: + path: rules + count: 1 + content: + apiGroups: [ "" ] + resources: [ "test" ] + verbs: [ "test" ] + + - it: check defaults + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: Role + - equal: + path: metadata.name + value: vc-my-release + - equal: + path: metadata.namespace + value: my-namespace + + - it: multi-namespace mode + set: + experimental: + multiNamespaceMode: + enabled: true + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: ClusterRole + - equal: + path: metadata.name + value: vc-mn-my-release-v-my-namespace diff --git a/chart/tests/rolebinding_test.yaml b/chart/tests/rolebinding_test.yaml new file mode 100644 index 000000000..6695445be --- /dev/null +++ b/chart/tests/rolebinding_test.yaml @@ -0,0 +1,59 @@ +suite: RoleBinding +templates: + - rolebinding.yaml + +tests: + - it: check defaults + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: RoleBinding + - equal: + path: metadata.name + value: vc-my-release + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: subjects[0].name + value: vc-my-release + - equal: + path: roleRef.kind + value: Role + - equal: + path: roleRef.name + value: vc-my-release + + - it: multi-namespace mode + set: + experimental: + multiNamespaceMode: + enabled: true + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: ClusterRoleBinding + - equal: + path: metadata.name + value: vc-mn-my-release-v-my-namespace + - notExists: + path: metadata.namespace + - equal: + path: subjects[0].name + value: vc-my-release + - equal: + path: roleRef.kind + value: ClusterRole + - equal: + path: roleRef.name + value: vc-mn-my-release-v-my-namespace diff --git a/chart/tests/service-monitor_test.yaml b/chart/tests/service-monitor_test.yaml new file mode 100644 index 000000000..122b0d81e --- /dev/null +++ b/chart/tests/service-monitor_test.yaml @@ -0,0 +1,34 @@ +suite: ServiceMonitor +templates: + - service-monitor.yaml + +tests: + - it: should not create service monitor by default + asserts: + - hasDocuments: + count: 0 + + - it: check defaults + release: + name: my-release + namespace: my-namespace + set: + controlPlane: + observability: + serviceMonitor: + enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: vc-my-release + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: spec.selector.matchLabels.app + value: vcluster + - lengthEqual: + path: spec.endpoints + count: 1 diff --git a/chart/tests/service_test.yaml b/chart/tests/service_test.yaml new file mode 100644 index 000000000..d32fd5763 --- /dev/null +++ b/chart/tests/service_test.yaml @@ -0,0 +1,60 @@ +suite: ControlPlane Service +templates: + - service.yaml + +tests: + - it: should not create service + set: + controlPlane: + service: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: service defaults + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: my-release + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.selector.app + value: vcluster + - lengthEqual: + path: spec.ports + count: 2 + + - it: isolated control plane + release: + name: my-release + namespace: my-namespace + set: + experimental: + isolatedControlPlane: + headless: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.type + value: ClusterIP + - lengthEqual: + path: spec.ports + count: 2 + - notExists: + path: spec.ports[0].targetPort + - notExists: + path: spec.ports[1].targetPort + - notExists: + path: spec.selector diff --git a/chart/tests/serviceaccount_test.yaml b/chart/tests/serviceaccount_test.yaml new file mode 100644 index 000000000..4441181e8 --- /dev/null +++ b/chart/tests/serviceaccount_test.yaml @@ -0,0 +1,61 @@ +suite: ControlPlane ServiceAccount +templates: + - serviceaccount.yaml + +tests: + - it: should not create service account + set: + controlPlane: + advanced: + serviceAccount: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should create service account + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: vc-my-release + - equal: + path: metadata.namespace + value: my-namespace + + - it: should create service account with name + set: + controlPlane: + advanced: + serviceAccount: + name: test + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: test + + - it: should create image pull secrets + set: + controlPlane: + advanced: + serviceAccount: + imagePullSecrets: + - name: test1 + workloadServiceAccount: + imagePullSecrets: + - name: test2 + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: imagePullSecrets + count: 1 + - equal: + path: imagePullSecrets[0].name + value: test1 diff --git a/chart/tests/statefulset_test.yaml b/chart/tests/statefulset_test.yaml new file mode 100644 index 000000000..53246afb4 --- /dev/null +++ b/chart/tests/statefulset_test.yaml @@ -0,0 +1,239 @@ +suite: ControlPlane StatefulSet +templates: + - statefulset.yaml + +tests: + - it: should not create control-plane + set: + experimental: + isolatedControlPlane: + headless: true + asserts: + - hasDocuments: + count: 0 + + - it: name & defaults + release: + name: my-release + namespace: my-namespace + capabilities: + majorVersion: 1 + minorVersion: 29 + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.template.spec.containers + count: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: VCLUSTER_NAME + value: my-release + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: data + mountPath: /data + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: binaries + mountPath: /binaries + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: certs + mountPath: /pki + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: helm-cache + mountPath: /.cache/helm + - equal: + path: metadata.name + value: my-release + - equal: + path: metadata.namespace + value: my-namespace + - equal: + path: spec.podManagementPolicy + value: Parallel + - equal: + path: spec.persistentVolumeClaimRetentionPolicy.whenDeleted + value: Retain + - equal: + path: spec.replicas + value: 1 + - equal: + path: spec.template.metadata.labels.app + value: vcluster + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 10 + - equal: + path: spec.volumeClaimTemplates[0].spec.accessModes[0] + value: ReadWriteOnce + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 5Gi + + - it: fail when both backing stores are enabled + set: + controlPlane: + backingStore: + embeddedEtcd: + enabled: true + externalEtcd: + enabled: true + asserts: + - failedTemplate: + errorMessage: "embeddedEtcd and externalEtcd cannot be enabled at the same time together" + + - it: not persistent when external etcd is enabled + set: + controlPlane: + backingStore: + externalEtcd: + enabled: true + asserts: + - notExists: + path: spec.volumeClaimTemplates + + - it: not persistent when k8s + set: + controlPlane: + distro: + k8s: + enabled: true + asserts: + - notExists: + path: spec.volumeClaimTemplates + + - it: persistent when k8s and embedded etcd + set: + controlPlane: + backingStore: + embeddedEtcd: + enabled: true + distro: + k8s: + enabled: true + asserts: + - lengthEqual: + path: spec.volumeClaimTemplates + count: 1 + + - it: plugin 1 + set: + plugins: + test: + image: test + plugin: + test123: + version: v2 + image: test + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 8 + - lengthEqual: + path: spec.template.spec.initContainers + count: 3 + + - it: plugin volumes 2 + set: + controlPlane: + distro: + k0s: + enabled: true + plugin: + test: + version: v2 + image: test + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 8 + - lengthEqual: + path: spec.template.spec.initContainers + count: 2 + + - it: plugin volumes 3 + set: + plugin: + test: + image: test + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 7 + - lengthEqual: + path: spec.template.spec.initContainers + count: 1 + + - it: add volumes + set: + controlPlane: + statefulSet: + persistence: + addVolumes: + - name: myVolume + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: myVolume + - lengthEqual: + path: spec.template.spec.volumes + count: 8 + + - it: enable k8s + set: + controlPlane: + distro: + k8s: + enabled: true + asserts: + - notExists: + path: spec.volumeClaimTemplates + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + + - it: enable eks + set: + controlPlane: + distro: + eks: + enabled: true + asserts: + - notExists: + path: spec.volumeClaimTemplates + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + + - it: enable k0s + set: + controlPlane: + backingStore: + externalEtcd: + enabled: true + distro: + k0s: + enabled: true + asserts: + - notExists: + path: spec.volumeClaimTemplates + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + + diff --git a/chart/tests/workload-serviceaccount_test.yaml b/chart/tests/workload-serviceaccount_test.yaml new file mode 100644 index 000000000..8b2ced477 --- /dev/null +++ b/chart/tests/workload-serviceaccount_test.yaml @@ -0,0 +1,64 @@ +suite: Workload ServiceAccount +templates: + - workload-serviceaccount.yaml + +tests: + - it: should not create service account + set: + controlPlane: + advanced: + workloadServiceAccount: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should create service account + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: vc-workload-my-release + - equal: + path: metadata.namespace + value: my-namespace + + - it: should create service account with name + set: + controlPlane: + advanced: + workloadServiceAccount: + name: test + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: test + + - it: should create image pull secrets + set: + controlPlane: + advanced: + serviceAccount: + imagePullSecrets: + - name: test1 + workloadServiceAccount: + imagePullSecrets: + - name: test2 + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: imagePullSecrets + count: 2 + - equal: + path: imagePullSecrets[0].name + value: test1 + - equal: + path: imagePullSecrets[1].name + value: test2 diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 000000000..7b3482158 --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,462 @@ +# Sync Options +sync: + toHost: + services: + enabled: true + endpoints: + enabled: true + persistentVolumeClaims: + enabled: true + configMaps: + enabled: true + all: true + secrets: + enabled: true + all: true + pods: + enabled: true + translateImage: {} + enforceTolerations: [] + useSecretsForSATokens: false + rewriteHosts: + enabled: true + initContainerImage: "library/alpine:3.13.1" + ingresses: + enabled: false + priorityClasses: + enabled: false + networkPolicies: + enabled: false + volumeSnapshots: + enabled: false + podDisruptionBudgets: + enabled: false + serviceAccounts: + enabled: false + storageClasses: + enabled: false + persistentVolumes: + enabled: false + + fromHost: + events: + enabled: true + csiDrivers: + enabled: false + csiNodes: + enabled: false + csiStorageCapacities: + enabled: false + ingressClasses: + enabled: false + storageClasses: + enabled: false + nodes: + pseudo: + enabled: true + real: + enabled: false + syncLabelsTaints: false + clearImageStatus: false + selector: + # name: "" + labels: {} + +# Control Plane Options +controlPlane: + # What distro to use for vCluster, if none is specified, k3s is used + distro: + k3s: + enabled: false + imagePullPolicy: "" + image: + repository: "rancher/k3s" + tag: "v1.29.0-k3s1" + securityContext: {} + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 40m + memory: 64Mi + + k0s: + enabled: false + imagePullPolicy: "" + image: + repository: "k0sproject/k0s" + tag: "v1.29.1-k0s.0" + securityContext: {} + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 40m + memory: 64Mi + + k8s: + enabled: false + apiServer: + disabled: false + imagePullPolicy: "" + image: + repository: "registry.k8s.io/kube-apiserver" + tag: "v1.29.0" + controllerManager: + disabled: false + imagePullPolicy: "" + image: + repository: "registry.k8s.io/kube-controller-manager" + tag: "v1.29.0" + scheduler: + imagePullPolicy: "" + image: + repository: "registry.k8s.io/kube-scheduler" + tag: "v1.29.0" + env: [] + securityContext: {} + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 40m + memory: 64Mi + + eks: + enabled: false + apiServer: + disabled: false + imagePullPolicy: "" + image: + repository: "public.ecr.aws/eks-distro/kubernetes/kube-apiserver" + tag: "v1.28.2-eks-1-28-6" + controllerManager: + disabled: false + imagePullPolicy: "" + image: + repository: "public.ecr.aws/eks-distro/kubernetes/kube-controller-manager" + tag: "v1.28.2-eks-1-28-6" + scheduler: + imagePullPolicy: "" + image: + repository: "public.ecr.aws/eks-distro/kubernetes/kube-scheduler" + tag: "v1.28.2-eks-1-28-6" + env: [] + securityContext: {} + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 40m + memory: 64Mi + + backingStore: + embeddedEtcd: + enabled: false + migrateFromExternalEtcd: false + externalEtcd: + enabled: false + statefulSet: + annotations: {} + labels: {} + image: + repository: "registry.k8s.io/etcd" + tag: "3.5.10-0" + imagePullPolicy: "" + extraArgs: [] + env: [] + resources: + requests: + cpu: 20m + memory: 150Mi + pods: + annotations: {} + labels: {} + highAvailability: + replicas: 1 + scheduling: + podManagementPolicy: Parallel + nodeSelector: {} + affinity: {} + tolerations: [] + topologySpreadConstraints: [] + priorityClassName: "" + security: + podSecurityContext: {} + containerSecurityContext: {} + persistence: + volumeClaim: + disabled: false + # Defines if the PVC should get automatically deleted when the StatefulSet is deleted. Can be either Delete or Retain + retentionPolicy: Retain + size: 5Gi + storageClass: "" + accessModes: [ "ReadWriteOnce" ] + volumeClaimTemplates: [] + addVolumes: [] + addVolumeMounts: [] + service: + annotations: {} + headlessService: + annotations: {} + + virtualScheduler: + enabled: false + + proxy: + bindAddress: "0.0.0.0" + port: 8443 + + coredns: + enabled: true + embedded: false + overwriteManifests: "" + overwriteConfig: "" + service: + annotations: {} + labels: {} + spec: + type: ClusterIP + deployment: + annotations: {} + labels: {} + image: "" + replicas: 1 + podLabels: {} + podAnnotations: {} + nodeSelector: {} + resources: {} + + service: + enabled: true + labels: {} + annotations: {} + kubeletNodePort: 0 + httpsNodePort: 0 + spec: + type: ClusterIP + + ingress: + enabled: false + labels: {} + annotations: {} + spec: {} + + statefulSet: + labels: {} + annotations: {} + imagePullPolicy: "" + image: + repository: "ghcr.io/loft-sh/vcluster" + tag: "" + + workingDir: "" + command: [] + args: [] + env: [] + + resources: + limits: + ephemeral-storage: 8Gi + memory: 2Gi + requests: + ephemeral-storage: 200Mi + cpu: 200m + memory: 256Mi + + pods: + labels: {} + annotations: {} + + highAvailability: + replicas: 1 + leaseDuration: 60 + renewDeadline: 40 + retryPeriod: 15 + + security: + podSecurityContext: {} + containerSecurityContext: {} + + persistence: + volumeClaim: + disabled: false + # Defines if the PVC should get automatically deleted when the StatefulSet is deleted. Can be either Delete or Retain + retentionPolicy: Retain + size: 5Gi + storageClass: "" + accessModes: [ "ReadWriteOnce" ] + volumeClaimTemplates: [] + addVolumeMounts: [] + addVolumes: [] + + scheduling: + podManagementPolicy: Parallel + topologySpreadConstraints: [] + priorityClassName: "" + nodeSelector: {} + affinity: {} + tolerations: [] + + probes: + livenessProbe: + enabled: true + readinessProbe: + enabled: true + startupProbe: + enabled: true + + observability: + serviceMonitor: + enabled: false + labels: {} + annotations: {} + + advanced: + serviceAccount: + enabled: true + name: "" + imagePullSecrets: [] + labels: {} + annotations: {} + + workloadServiceAccount: + enabled: true + name: "" + imagePullSecrets: [] + annotations: {} + labels: {} + + headlessService: + labels: {} + annotations: {} + + globalMetadata: + annotations: {} + +rbac: + role: + enabled: true + overwriteRules: [] + extraRules: [] + clusterRole: + disabled: false + overwriteRules: [] + extraRules: [] + +observability: + metrics: + proxy: + nodes: + enabled: false + pods: + enabled: false + +networking: + # Embedded CoreDNS plugin config + replicateServices: + toHost: [] + fromHost: [] + resolveServices: [] + advanced: + clusterDomain: "cluster.local" + fallbackHostCluster: true + +policies: + resourceQuota: + enabled: false + labels: {} + annotations: {} + quota: + requests.cpu: 10 + requests.memory: 20Gi + requests.storage: "100Gi" + requests.ephemeral-storage: 60Gi + limits.cpu: 20 + limits.memory: 40Gi + limits.ephemeral-storage: 160Gi + services.nodeports: 0 + services.loadbalancers: 1 + count/endpoints: 40 + count/pods: 20 + count/services: 20 + count/secrets: 100 + count/configmaps: 100 + count/persistentvolumeclaims: 20 + scopeSelector: + matchExpressions: [] + scopes: {} + + limitRange: + enabled: false + labels: {} + annotations: {} + default: + ephemeral-storage: 8Gi + memory: 512Mi + cpu: "1" + defaultRequest: + ephemeral-storage: 3Gi + memory: 128Mi + cpu: 100m + + networkPolicy: + enabled: false + labels: {} + annotations: {} + fallbackDns: 8.8.8.8 + outgoingConnections: + ipBlock: + cidr: 0.0.0.0/0 + except: + - 100.64.0.0/10 + - 127.0.0.0/8 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + +# Export vCluster Kube Config +exportKubeConfig: + context: "" + server: "" + secret: + name: "" + namespace: "" + +# What plugins should get used +plugins: {} + +# Functionality that is likely to change, use with caution! +experimental: + multiNamespaceMode: + enabled: false + + syncSettings: + targetNamespace: "" + + isolatedControlPlane: + headless: false + + deploy: + enabled: false + manifests: "" + manifestsTemplate: '' + helm: [] + + genericSync: + config: "" + clusterRole: + extraRules: [] + role: + extraRules: [] + +platform: + apiKey: + value: "" + secretRef: + name: "" + namespace: "" diff --git a/cmd/vcluster/cmd/start.go b/cmd/vcluster/cmd/start.go index 6a8319961..797bf332c 100644 --- a/cmd/vcluster/cmd/start.go +++ b/cmd/vcluster/cmd/start.go @@ -6,8 +6,8 @@ import ( "os" "runtime/debug" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/leaderelection" - "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/plugin" "github.com/loft-sh/vcluster/pkg/pro" "github.com/loft-sh/vcluster/pkg/scheme" @@ -19,8 +19,12 @@ import ( "k8s.io/client-go/kubernetes" ) +type StartOptions struct { + Config string +} + func NewStartCommand() *cobra.Command { - vClusterOptions := &options.VirtualClusterOptions{} + startOptions := &StartOptions{} cmd := &cobra.Command{ Use: "start", Short: "Execute the vcluster", @@ -40,52 +44,48 @@ func NewStartCommand() *cobra.Command { } }() + // parse vCluster config + vClusterConfig, err := config.ParseConfig(startOptions.Config) + if err != nil { + return err + } + // execute command - return ExecuteStart(cobraCmd.Context(), vClusterOptions) + return ExecuteStart(cobraCmd.Context(), vClusterConfig) }, } - options.AddFlags(cmd.Flags(), vClusterOptions) - pro.AddProFlags(cmd.Flags(), vClusterOptions) + cmd.Flags().StringVar(&startOptions.Config, "config", "", "The path where to find the vCluster config to load") return cmd } -func ExecuteStart(ctx context.Context, options *options.VirtualClusterOptions) error { - err := pro.ValidateProOptions(options) - if err != nil { - return err - } - - // set suffix - translate.VClusterName = options.Name - if translate.VClusterName == "" { - translate.VClusterName = options.DeprecatedSuffix - } +func ExecuteStart(ctx context.Context, vConfig *config.VirtualClusterConfig) error { + // set global vCluster name if translate.VClusterName == "" { - translate.VClusterName = "vcluster" + translate.VClusterName = vConfig.Name } // set service name - if options.ServiceName == "" { - options.ServiceName = translate.VClusterName + if vConfig.ServiceName == "" { + vConfig.ServiceName = translate.VClusterName } // get current namespace - controlPlaneConfig, controlPlaneNamespace, controlPlaneService, workloadConfig, workloadNamespace, workloadService, err := pro.GetRemoteClient(options) + controlPlaneConfig, controlPlaneNamespace, controlPlaneService, workloadConfig, workloadNamespace, workloadService, err := pro.GetRemoteClient(vConfig) if err != nil { return err } - options.ServiceName = workloadService + vConfig.ServiceName = workloadService err = os.Setenv("NAMESPACE", workloadNamespace) if err != nil { return fmt.Errorf("set NAMESPACE env var: %w", err) } // init telemetry - telemetry.Collector.Init(controlPlaneConfig, controlPlaneNamespace, options) + telemetry.Collector.Init(controlPlaneConfig, controlPlaneNamespace, vConfig) // initialize feature gate from environment - err = pro.LicenseInit(ctx, controlPlaneConfig, controlPlaneNamespace, options.ProOptions.ProLicenseSecret) + err = pro.LicenseInit(ctx, controlPlaneConfig, controlPlaneNamespace, vConfig.Platform.ApiKey.Value, vConfig.Platform.ApiKey.SecretRef.Namespace, vConfig.Platform.ApiKey.SecretRef.Name) if err != nil { return fmt.Errorf("init license: %w", err) } @@ -141,7 +141,7 @@ func ExecuteStart(ctx context.Context, options *options.VirtualClusterOptions) e } // start integrated coredns - if controllerCtx.Options.ProOptions.IntegratedCoredns { + if vConfig.ControlPlane.CoreDNS.Embedded { err = pro.StartIntegratedCoreDNS(controllerCtx) if err != nil { return fmt.Errorf("start integrated core dns: %w", err) @@ -160,9 +160,9 @@ func ExecuteStart(ctx context.Context, options *options.VirtualClusterOptions) e return nil } -func StartLeaderElection(ctx *options.ControllerContext, startLeading func() error) error { +func StartLeaderElection(ctx *config.ControllerContext, startLeading func() error) error { var err error - if ctx.Options.LeaderElect { + if ctx.Config.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 { err = leaderelection.StartLeaderElection(ctx, scheme.Scheme, func() error { return startLeading() }) diff --git a/config/README.md b/config/README.md new file mode 100644 index 000000000..0003a6dfd --- /dev/null +++ b/config/README.md @@ -0,0 +1 @@ +This file will get synced to [github.com/loft-sh/vcluster-config](https://github.com/loft-sh/vcluster-config). Please do not add any dependencies here. diff --git a/config/config.go b/config/config.go new file mode 100644 index 000000000..179e45e9b --- /dev/null +++ b/config/config.go @@ -0,0 +1,890 @@ +package config + +import "regexp" + +type Config struct { + ExportKubeConfig ExportKubeConfig `json:"exportKubeConfig,omitempty"` + Sync Sync `json:"sync,omitempty"` + Observability Observability `json:"observability,omitempty"` + Networking Networking `json:"networking,omitempty"` + Plugin map[string]Plugin `json:"plugin,omitempty"` + Plugins map[string]Plugins `json:"plugins,omitempty"` + ControlPlane ControlPlane `json:"controlPlane,omitempty"` + Policies Policies `json:"policies,omitempty"` + RBAC RBAC `json:"rbac,omitempty"` + + // Telemetry is the configuration related to telemetry gathered about vcluster usage. + Telemetry Telemetry `json:"telemetry,omitempty"` + Experimental Experimental `json:"experimental,omitempty"` + Platform Platform `json:"platform,omitempty"` +} + +type ExportKubeConfig struct { + Context string `json:"context"` + Server string `json:"server"` + Secret SecretReference `json:"secret,omitempty"` +} + +// SecretReference represents a Secret Reference. It has enough information to retrieve secret +// in any namespace +type SecretReference struct { + // name is unique within a namespace to reference a secret resource. + Name string `json:"name,omitempty"` + + // namespace defines the space within which the secret name must be unique. + Namespace string `json:"namespace,omitempty"` +} + +type Sync struct { + ToHost SyncToHost `json:"toHost,omitempty"` + FromHost SyncFromHost `json:"fromHost,omitempty"` +} + +type SyncToHost struct { + Services EnableSwitch `json:"services,omitempty"` + Endpoints EnableSwitch `json:"endpoints,omitempty"` + Ingresses EnableSwitch `json:"ingresses,omitempty"` + PriorityClasses EnableSwitch `json:"priorityClasses,omitempty"` + NetworkPolicies EnableSwitch `json:"networkPolicies,omitempty"` + VolumeSnapshots EnableSwitch `json:"volumeSnapshots,omitempty"` + PodDisruptionBudgets EnableSwitch `json:"podDisruptionBudgets,omitempty"` + ServiceAccounts EnableSwitch `json:"serviceAccounts,omitempty"` + StorageClasses EnableSwitch `json:"storageClasses,omitempty"` + PersistentVolumes EnableSwitch `json:"persistentVolumes,omitempty"` + PersistentVolumeClaims EnableSwitch `json:"persistentVolumeClaims,omitempty"` + ConfigMaps SyncAllResource `json:"configMaps,omitempty"` + Secrets SyncAllResource `json:"secrets,omitempty"` + Pods SyncPods `json:"pods,omitempty"` +} + +type SyncFromHost struct { + CSIDrivers EnableSwitch `json:"csiDrivers,omitempty"` + CSINodes EnableSwitch `json:"csiNodes,omitempty"` + CSIStorageCapacities EnableSwitch `json:"csiStorageCapacities,omitempty"` + IngressClasses EnableSwitch `json:"ingressClasses,omitempty"` + Events EnableSwitch `json:"events,omitempty"` + StorageClasses EnableSwitch `json:"storageClasses,omitempty"` + Nodes SyncNodes `json:"nodes,omitempty"` +} + +type EnableSwitch struct { + Enabled bool `json:"enabled,omitempty"` +} + +type SyncAllResource struct { + Enabled bool `json:"enabled,omitempty"` + All bool `json:"all,omitempty"` +} + +type SyncPods struct { + Enabled bool `json:"enabled,omitempty"` + + WorkloadServiceAccount string `json:"workloadServiceAccount,omitempty"` + TranslateImage map[string]string `json:"translateImage,omitempty"` + EnforceTolerations []string `json:"enforceTolerations,omitempty"` // validate format + UseSecretsForSATokens bool `json:"useSecretsForSATokens,omitempty"` + RewriteHosts SyncRewriteHosts `json:"rewriteHosts,omitempty"` +} + +type SyncRewriteHosts struct { + Enabled bool `json:"enabled,omitempty"` + InitContainerImage string `json:"initContainerImage,omitempty"` +} + +type SyncNodes struct { + Real SyncRealNodes `json:"real,omitempty"` + Pseudo EnableSwitch `json:"pseudo,omitempty"` +} + +type SyncRealNodes struct { + Enabled bool `json:"enabled,omitempty"` + + SyncAll bool `json:"syncAll,omitempty"` + SyncLabelsTaints bool `json:"syncLabelsTaints,omitempty"` + ClearImageStatus bool `json:"clearImageStatus,omitempty"` + Selector SyncNodeSelector `json:"selector,omitempty"` +} + +type SyncNodeSelector struct { + Labels map[string]string `json:"labels,omitempty"` +} + +type Observability struct { + Metrics ObservabilityMetrics `json:"metrics,omitempty"` +} + +type ControlPlaneObservability struct { + ServiceMonitor EnableSwitch `json:"serviceMonitor,omitempty"` +} + +type ObservabilityMetrics struct { + Proxy MetricsProxy `json:"proxy,omitempty"` +} + +type MetricsProxy struct { + Nodes EnableSwitch `json:"nodes,omitempty"` + Pods EnableSwitch `json:"pods,omitempty"` +} + +type Networking struct { + ReplicateServices ReplicateServices `json:"replicateServices,omitempty"` + ResolveServices ResolveServices `json:"resolveServices,omitempty"` + Advanced NetworkingAdvanced `json:"advanced,omitempty"` +} + +type ReplicateServices struct { + ToHost ServiceMapping `json:"toHost,omitempty"` + FromHost ServiceMapping `json:"fromHost,omitempty"` +} + +type ServiceMapping struct { + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` +} + +type ResolveServices struct { + Service string `json:"service,omitempty"` + Target ResolveServiceTarget `json:"target,omitempty"` +} + +type ResolveServiceTarget struct { + VCluster ResolveServiceService `json:"vcluster,omitempty"` + Host ResolveServiceService `json:"host,omitempty"` + External ResolveServiceHostname `json:"external,omitempty"` +} + +type ResolveServiceService struct { + Service string `json:"service,omitempty"` +} + +type ResolveServiceHostname struct { + Hostname string `json:"hostname,omitempty"` +} + +type NetworkingAdvanced struct { + ClusterDomain string `json:"clusterDomain,omitempty"` + FallBack []NetworkDNSFallback `json:"fallback,omitempty"` + ProxyKubelets NetworkProxyKubelets `json:"proxyKubelets,omitempty"` +} + +type NetworkDNSFallback struct { + IP string `json:"ip,omitempty"` + HostCluster bool `json:"hostCluster,omitempty"` +} + +type NetworkProxyKubelets struct { + ByHostname bool `json:"byHostname,omitempty"` + ByIP bool `json:"byIP,omitempty"` +} + +type Plugin struct { + Plugins `json:",inline"` + + Version string `json:"version,omitempty"` + Env []interface{} `json:"env,omitempty"` + EnvFrom []interface{} `json:"envFrom,omitempty"` + Lifecycle map[string]interface{} `json:"lifecycle,omitempty"` + LivenessProbe map[string]interface{} `json:"livenessProbe,omitempty"` + ReadinessProbe map[string]interface{} `json:"readinessProbe,omitempty"` + StartupProbe map[string]interface{} `json:"startupProbe,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` + Optional bool `json:"optional,omitempty"` +} + +type Plugins struct { + Name string `json:"name,omitempty"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Image string `json:"image,omitempty"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + Config map[string]interface{} `json:"config,omitempty"` + SecurityContext map[string]interface{} `json:"securityContext,omitempty"` + Resources map[string]interface{} `json:"resources,omitempty"` + VolumeMounts []interface{} `json:"volumeMounts,omitempty"` + RBAC PluginsRBAC `json:"rbac,omitempty"` +} + +type PluginsRBAC struct { + Role []RBACPolicyRule `json:"role,omitempty"` + ClusterRole []RBACPolicyRule `json:"clusterRole,omitempty"` +} + +type RBACPolicyRule struct { + // Verbs is a list of Verbs that apply to ALL the ResourceKinds contained in this rule. '*' represents all verbs. + Verbs []string `json:"verbs"` + + // APIGroups is the name of the APIGroup that contains the resources. If multiple API groups are specified, any action requested against one of + // the enumerated resources in any API group will be allowed. "" represents the core API group and "*" represents all API groups. + APIGroups []string `json:"apiGroups,omitempty"` + // Resources is a list of resources this rule applies to. '*' represents all resources. + Resources []string `json:"resources,omitempty"` + // ResourceNames is an optional white list of names that the rule applies to. An empty set means that everything is allowed. + ResourceNames []string `json:"resourceNames,omitempty"` + + // NonResourceURLs is a set of partial urls that a user should have access to. *s are allowed, but only as the full, final step in the path + // Since non-resource URLs are not namespaced, this field is only applicable for ClusterRoles referenced from a ClusterRoleBinding. + // Rules can either apply to API resources (such as "pods" or "secrets") or non-resource URL paths (such as "/api"), but not both. + NonResourceURLs []string `json:"nonResourceURLs,omitempty"` +} + +type ControlPlane struct { + Distro Distro `json:"distro,omitempty"` + HostPathMapper HostPathMapper `json:"hostPathMapper,omitempty"` + CoreDNS CoreDNS `json:"coredns,omitempty"` + BackingStore BackingStore `json:"backingStore,omitempty"` + VirtualScheduler EnableSwitch `json:"virtualScheduler,omitempty" product:"pro"` + Proxy ControlPlaneProxy `json:"proxy,omitempty"` + Service ControlPlaneService `json:"service,omitempty"` + Ingress ControlPlaneIngress `json:"ingress,omitempty"` + StatefulSet ControlPlaneStatefulSet `json:"statefulSet,omitempty"` + Observability ControlPlaneObservability `json:"observability,omitempty"` + Advanced ControlPlaneAdvanced `json:"advanced,omitempty"` +} + +type ControlPlaneStatefulSet struct { + Image Image `json:"image,omitempty"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + Probes ControlPlaneProbes `json:"probes,omitempty"` + Security ControlPlaneSecurity `json:"security,omitempty"` + Persistence ControlPlanePersistence `json:"persistence,omitempty"` + Scheduling ControlPlaneScheduling `json:"scheduling,omitempty"` + HighAvailability ControlPlaneHighAvailability `json:"highAvailability,omitempty"` +} + +type Distro struct { + K3S DistroK3s `json:"k3s,omitempty"` + K8S DistroK8s `json:"k8s,omitempty"` + K0S DistroK0s `json:"k0s,omitempty"` + EKS DistroEks `json:"eks,omitempty"` +} + +type DistroK3s struct { + Enabled bool `json:"enabled,omitempty"` + DistroCommon `json:",inline"` + DistroContainer `json:",inline"` +} + +type DistroK8s struct { + Enabled bool `json:"enabled,omitempty"` + DistroCommon `json:",inline"` + APIServer DistroContainer `json:"apiServer,omitempty"` + ControllerManager DistroContainer `json:"controllerManager,omitempty"` + Scheduler DistroContainer `json:"scheduler,omitempty"` +} + +type DistroK0s struct { + Enabled bool `json:"enabled,omitempty"` + DistroCommon `json:",inline"` + DistroContainer `json:",inline"` +} + +type DistroEks struct { + Enabled bool `json:"enabled,omitempty"` + DistroCommon `json:",inline"` + APIServer DistroContainer `json:"apiServer,omitempty"` + ControllerManager DistroContainer `json:"controllerManager,omitempty"` + Scheduler DistroContainer `json:"scheduler,omitempty"` +} + +type DistroCommon struct { + Env []map[string]interface{} `json:"env,omitempty"` + SecurityContext map[string]interface{} `json:"securityContext,omitempty"` + Resources map[string]interface{} `json:"resources,omitempty"` +} + +type DistroContainer struct { + Image Image `json:"image,omitempty"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + Command []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + ExtraArgs []string `json:"extraArgs,omitempty"` +} + +type Image struct { + Repository string `json:"repository,omitempty"` + Tag string `json:"tag,omitempty"` +} + +// LocalObjectReference contains enough information to let you locate the +// referenced object inside the same namespace. +type LocalObjectReference struct { + // Name of the referent. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + Name string `json:"name,omitempty"` +} + +type VirtualClusterKubeConfig struct { + KubeConfig string `json:"kubeConfig,omitempty"` + ServerCAKey string `json:"serverCAKey,omitempty"` + ServerCACert string `json:"serverCACert,omitempty"` + ClientCACert string `json:"clientCACert,omitempty"` + RequestHeaderCACert string `json:"requestHeaderCACert,omitempty"` +} + +type BackingStore struct { + EmbeddedEtcd EmbeddedEtcd `json:"embeddedEtcd,omitempty" product:"pro"` + ExternalEtcd ExternalEtcd `json:"externalEtcd,omitempty"` +} + +type EmbeddedEtcd struct { + Enabled bool `json:"enabled,omitempty"` + MigrateFromSqlite bool `json:"migrateFromSqlite,omitempty"` +} + +type ExternalEtcd struct { + Enabled bool `json:"enabled,omitempty"` + Image string `json:"image,omitempty"` + Replicas uint8 `json:"replicas,omitempty"` + Security ControlPlaneSecurity `json:"security,omitempty"` + Scheduling ControlPlaneScheduling `json:"scheduling,omitempty"` + Persistence ControlPlanePersistence `json:"persistence,omitempty"` + Metadata ExternalEtcdMetadata `json:"metadata,omitempty"` +} + +type ExternalEtcdMetadata struct { + LabelsAndAnnotations `json:",inline"` + PodLabels map[string]string `json:"podLabels,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` +} + +type HostPathMapper struct { + EnableSwitch `json:",inline"` + Central bool `json:"central,omitempty" product:"pro"` +} + +type CoreDNS struct { + EnableSwitch `json:",inline"` + Embedded bool `json:"embedded,omitempty" product:"pro"` + Service CoreDNSService `json:"service,omitempty"` + Deployment CoreDNSDeployment `json:"deployment,omitempty"` +} + +type CoreDNSService struct { + LabelsAndAnnotations `json:",inline"` + Spec map[string]interface{} `json:"spec,omitempty"` +} + +type CoreDNSDeployment struct { + LabelsAndAnnotations `json:",inline"` + Spec map[string]interface{} `json:"spec,omitempty"` +} + +type ControlPlaneProxy struct { + BindAddress string `json:"bindAddress,omitempty"` + Port int `json:"port,omitempty"` + TLS ControlPlaneProxyTLS `json:"tls,omitempty"` +} + +type ControlPlaneProxyTLS struct { + ExtraSANs []string `json:"extraSANs,omitempty"` +} + +type ControlPlaneService struct { + LabelsAndAnnotations `json:",inline"` + + Name string `json:"name,omitempty"` + Spec map[string]interface{} `json:"spec,omitempty"` +} + +type ControlPlaneIngress struct { + EnableSwitch `json:",inline"` + LabelsAndAnnotations `json:",inline"` + + Name string `json:"name,omitempty"` + Spec map[string]interface{} `json:"spec,omitempty"` +} + +type ControlPlaneHighAvailability struct { + Replicas int32 `json:"replicas,omitempty"` + LeaseDuration int `json:"leaseDuration,omitempty"` + RenewDeadline int `json:"renewDeadline,omitempty"` + RetryPeriod int `json:"retryPeriod,omitempty"` +} + +type ControlPlaneAdvanced struct { + DefaultImageRegistry string `json:"defaultImageRegistry,omitempty"` + ServiceAccount ControlPlaneServiceAccount `json:"serviceAccount,omitempty"` + WorkloadServiceAccount ControlPlaneWorkloadServiceAccount `json:"workloadServiceAccount,omitempty"` + Metadata ControlPlaneMetadata `json:"metadata,omitempty"` +} + +type ControlPlanePersistence struct { + EnableSwitch `json:",inline"` + RetentionPolicy string `json:"retentionPolicy,omitempty"` + Size string `json:"size,omitempty"` + StorageClass string `json:"storageClass,omitempty"` + AddVolumeMounts []VolumeMount `json:"addVolumeMounts,omitempty"` + OverwriteVolumeMounts []VolumeMount `json:"overwriteVolumeMounts,omitempty"` +} + +// VolumeMount describes a mounting of a Volume within a container. +type VolumeMount struct { + // This must match the Name of a Volume. + Name string `json:"name" protobuf:"bytes,1,opt,name=name"` + // Mounted read-only if true, read-write otherwise (false or unspecified). + // Defaults to false. + ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,2,opt,name=readOnly"` + // Path within the container at which the volume should be mounted. Must + // not contain ':'. + MountPath string `json:"mountPath" protobuf:"bytes,3,opt,name=mountPath"` + // Path within the volume from which the container's volume should be mounted. + // Defaults to "" (volume's root). + SubPath string `json:"subPath,omitempty" protobuf:"bytes,4,opt,name=subPath"` + // mountPropagation determines how mounts are propagated from the host + // to container and the other way around. + // When not set, MountPropagationNone is used. + // This field is beta in 1.10. + MountPropagation *string `json:"mountPropagation,omitempty" protobuf:"bytes,5,opt,name=mountPropagation,casttype=MountPropagationMode"` + // Expanded path within the volume from which the container's volume should be mounted. + // Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + // Defaults to "" (volume's root). + // SubPathExpr and SubPath are mutually exclusive. + SubPathExpr string `json:"subPathExpr,omitempty" protobuf:"bytes,6,opt,name=subPathExpr"` +} + +type ControlPlaneScheduling struct { + NodeSelector map[string]interface{} `json:"nodeSelector,omitempty"` + Affinity map[string]interface{} `json:"affinity,omitempty"` + Tolerations map[string]interface{} `json:"tolerations,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` +} + +type ControlPlaneServiceAccount struct { + Enabled bool `json:"enabled,omitempty"` + Name string `json:"name,omitempty"` + ImagePullSecrets []LocalObjectReference `json:"imagePullSecrets,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type ControlPlaneWorkloadServiceAccount struct { + Enabled bool `json:"enabled,omitempty"` + Name string `json:"name,omitempty"` + ImagePullSecrets []LocalObjectReference `json:"imagePullSecrets,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +type ControlPlaneProbes struct { + LivenessProbe EnableSwitch `json:"livenessProbe,omitempty"` + ReadinessProbe EnableSwitch `json:"readinessProbe,omitempty"` + StartupProbe EnableSwitch `json:"startupProbe,omitempty"` +} + +type ControlPlaneSecurity struct { + PodSecurityContext map[string]interface{} `json:"podSecurityContext,omitempty"` + ContainerSecurityContext ControlPlaneContainerSecurityContext `json:"containerSecurityContext,omitempty"` +} + +type ControlPlaneContainerSecurityContext struct { + AllowPrivilegeEscalation bool `json:"allowPrivilegeEscalation,omitempty"` + Capabilities map[string]interface{} `json:"capabilities,omitempty"` + RunAsUser int64 `json:"runAsUser,omitempty"` + RunAsGroup int64 `json:"runAsGroup,omitempty"` +} + +type ControlPlaneMetadata struct { + StatefulSet LabelsAndAnnotations `json:"statefulSet,omitempty"` + Pods LabelsAndAnnotations `json:"pods,omitempty"` + AllResources LabelsAndAnnotations `json:"allResources,omitempty"` +} + +type LabelsAndAnnotations struct { + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +type Policies struct { + PodSecurityStandard string `json:"podSecurityStandard,omitempty"` + ResourceQuota ResourceQuota `json:"resourceQuota,omitempty"` + LimitRange LimitRange `json:"limitRange,omitempty"` + NetworkPolicy NetworkPolicy `json:"networkPolicy,omitempty"` + CentralAdmissionControl CentralAdmissionControl `json:"centralAdmissionControl,omitempty" product:"pro"` +} + +type ResourceQuota struct { + EnableSwitch `json:",inline"` + Quota map[string]string `json:"quota,omitempty"` + ScopeSelector ScopeSelector `json:"scopeSelector,omitempty"` + Scopes []string `json:"scopes,omitempty"` +} + +type ScopeSelector struct { + MatchExpressions []LabelSelectorRequirement `json:"matchExpressions,omitempty"` +} + +type LabelSelectorRequirement struct { + // key is the label key that the selector applies to. + Key string `json:"key"` + // operator represents a key's relationship to a set of values. + // Valid operators are In, NotIn, Exists and DoesNotExist. + Operator string `json:"operator"` + // values is an array of string values. If the operator is In or NotIn, + // the values array must be non-empty. If the operator is Exists or DoesNotExist, + // the values array must be empty. This array is replaced during a strategic + // merge patch. + Values []string `json:"values,omitempty"` +} + +type LimitRange struct { + EnableSwitch `json:",inline"` + Default LimitRangeLimits `json:"default,omitempty"` + DefaultRequest LimitRangeLimits `json:"defaultRequest,omitempty"` +} + +type LimitRangeLimits struct { + EphemeralStorage string `json:"ephemeral-storage,omitempty"` + Memory string `json:"memory,omitempty"` + CPU string `json:"cpu,omitempty"` +} + +type NetworkPolicy struct { + EnableSwitch `json:",inline"` + OutgoingConnections OutgoingConnections `json:"outgoingConnections,omitempty"` +} + +type OutgoingConnections struct { + IPBlock IPBlock `json:"ipBlock,omitempty"` +} + +// IPBlock describes a particular CIDR (Ex. "192.168.1.0/24","2001:db8::/64") that is allowed +// to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs +// that should not be included within this rule. +type IPBlock struct { + // cidr is a string representing the IPBlock + // Valid examples are "192.168.1.0/24" or "2001:db8::/64" + CIDR string `json:"cidr"` + + // except is a slice of CIDRs that should not be included within an IPBlock + // Valid examples are "192.168.1.0/24" or "2001:db8::/64" + // Except values will be rejected if they are outside the cidr range + // +optional + Except []string `json:"except,omitempty"` +} + +type CentralAdmissionControl struct { + ValidatingWebhooks []interface{} `json:"validatingWebhooks,omitempty"` + MutatingWebhooks []interface{} `json:"mutatingWebhooks,omitempty"` +} + +type RBAC struct { + ClusterRole RBACClusterRole `json:"clusterRole,omitempty"` + RBACRole RBACRole `json:"role,omitempty"` +} + +type RBACClusterRole struct { + Create bool `json:"create,omitempty"` + ExtraRules interface{} `json:"extraRules,omitempty"` +} + +type RBACRole struct { + RBACClusterRole `json:",inline"` + ExcludedApiResources []string `json:"excludedApiResources,omitempty"` +} + +type Telemetry struct { + Disabled bool `json:"disabled,omitempty"` + InstanceCreator string `json:"instanceCreators,omitempty"` + PlatformUserID string `json:"platformUserID,omitempty"` + PlatformInstanceID string `json:"platformInstanceID,omitempty"` + MachineID string `json:"machineID,omitempty"` +} + +type Experimental struct { + Extended map[string]interface{} `json:",inline"` + + IsolatedControlPlane ExperimentalIsolatedControlPlane `json:"isolatedControlPlane,omitempty"` + ControlPlaneSettings ExperimentalControlPlaneSettings `json:"controlPlaneSettings,omitempty"` + SyncSettings ExperimentalSyncSettings `json:"syncSettings,omitempty"` + GenericSync ExperimentalGenericSync `json:"genericSync,omitempty"` + Deploy ExperimentalDeploy `json:"deploy,omitempty"` + MultiNamespaceMode ExperimentalMultiNamespaceMode `json:"multiNamespaceMode,omitempty"` + + VirtualClusterKubeConfig VirtualClusterKubeConfig `json:"virtualClusterKubeConfig,omitempty"` +} + +type ExperimentalMultiNamespaceMode struct { + Enabled bool `json:"enabled,omitempty"` +} + +type ExperimentalIsolatedControlPlane struct { + Enabled bool `json:"enabled,omitempty"` + + KubeConfig string `json:"kubeConfig,omitempty"` +} + +type ExperimentalControlPlaneSettings struct { + RewriteKubernetesService bool `json:"rewriteKubernetesService,omitempty"` +} + +type ExperimentalSyncSettings struct { + DisableSync bool `json:"disableSync,omitempty"` + Target ExperimentalSyncSettingsTarget `json:"target,omitempty"` +} + +type ExperimentalSyncSettingsTarget struct { + Namespace string `json:"namespace,omitempty"` +} + +type ExperimentalDeploy struct { + Manifests string `json:"manifests,omitempty"` + ManifestsTemplate string `json:"manifestsTemplate,omitempty"` + Helm []interface{} `json:"helm,omitempty"` +} + +type Platform struct { + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + Project string `json:"project,omitempty"` + ApiKey SecretKeyReference `json:"apiKey,omitempty"` +} + +type SecretKeyReference struct { + Value string `json:"value,omitempty"` + SecretRef SecretReference `json:"secretRef,omitempty"` +} + +type Template struct { + // Name is the name of the template used to populate the virtual cluster + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` +} + +type Access struct { + Audit AccessAudit `json:"audit,omitempty" product:"platform"` +} + +type AccessAudit struct { + Enabled bool `json:"enabled,omitempty"` + Level int `json:"level,omitempty"` + Policy AccessAuditPolicy `json:"policy,omitempty"` +} + +type AccessAuditPolicy struct { + Rules []AuditPolicyRule `json:"rules,omitempty"` +} + +// AuditPolicyRule maps requests based off metadata to an audit Level. +// Requests must match the rules of every field (an intersection of rules). +type AuditPolicyRule struct { + // The Level that requests matching this rule are recorded at. + Level string `json:"level"` + + // The users (by authenticated user name) this rule applies to. + // An empty list implies every user. + Users []string `json:"users,omitempty"` + // The user groups this rule applies to. A user is considered matching + // if it is a member of any of the UserGroups. + // An empty list implies every user group. + UserGroups []string `json:"userGroups,omitempty"` + + // The verbs that match this rule. + // An empty list implies every verb. + Verbs []string `json:"verbs,omitempty"` + + // Rules can apply to API resources (such as "pods" or "secrets"), + // non-resource URL paths (such as "/api"), or neither, but not both. + // If neither is specified, the rule is treated as a default for all URLs. + + // Resources that this rule matches. An empty list implies all kinds in all API groups. + Resources []AuditGroupResources `json:"resources,omitempty"` + // Namespaces that this rule matches. + // The empty string "" matches non-namespaced resources. + // An empty list implies every namespace. + Namespaces []string `json:"namespaces,omitempty"` + + // NonResourceURLs is a set of URL paths that should be audited. + // `*`s are allowed, but only as the full, final step in the path. + // Examples: + // - `/metrics` - Log requests for apiserver metrics + // - `/healthz*` - Log all health checks + NonResourceURLs []string `json:"nonResourceURLs,omitempty"` + + // OmitStages is a list of stages for which no events are created. Note that this can also + // be specified policy wide in which case the union of both are omitted. + // An empty list means no restrictions will apply. + OmitStages []string `json:"omitStages,omitempty"` + + // OmitManagedFields indicates whether to omit the managed fields of the request + // and response bodies from being written to the API audit log. + // - a value of 'true' will drop the managed fields from the API audit log + // - a value of 'false' indicates that the managed fileds should be included + // in the API audit log + // Note that the value, if specified, in this rule will override the global default + // If a value is not specified then the global default specified in + // Policy.OmitManagedFields will stand. + OmitManagedFields *bool `json:"omitManagedFields,omitempty"` +} + +// AuditGroupResources represents resource kinds in an API group. +type AuditGroupResources struct { + // Group is the name of the API group that contains the resources. + // The empty string represents the core API group. + Group string `json:"group,omitempty"` + // Resources is a list of resources this rule applies to. + // + // For example: + // - `pods` matches pods. + // - `pods/log` matches the log subresource of pods. + // - `*` matches all resources and their subresources. + // - `pods/*` matches all subresources of pods. + // - `*/scale` matches all scale subresources. + // + // If wildcard is present, the validation rule will ensure resources do not + // overlap with each other. + // + // An empty list implies all resources and subresources in this API groups apply. + Resources []string `json:"resources,omitempty"` + // ResourceNames is a list of resource instance names that the policy matches. + // Using this field requires Resources to be specified. + // An empty list implies that every instance of the resource is matched. + ResourceNames []string `json:"resourceNames,omitempty"` +} + +type ExperimentalGenericSync struct { + // Version is the config version + Version string `json:"version,omitempty" yaml:"version,omitempty"` + + // Exports syncs a resource from the virtual cluster to the host + Exports []*Export `json:"export,omitempty" yaml:"export,omitempty"` + + // Imports syncs a resource from the host cluster to virtual cluster + Imports []*Import `json:"import,omitempty" yaml:"import,omitempty"` + + // Hooks are hooks that can be used to inject custom patches before syncing + Hooks *Hooks `json:"hooks,omitempty" yaml:"hooks,omitempty"` +} + +type Hooks struct { + // HostToVirtual is a hook that is executed before syncing from the host to the virtual cluster + HostToVirtual []*Hook `json:"hostToVirtual,omitempty" yaml:"hostToVirtual,omitempty"` + + // VirtualToHost is a hook that is executed before syncing from the virtual to the host cluster + VirtualToHost []*Hook `json:"virtualToHost,omitempty" yaml:"virtualToHost,omitempty"` +} + +type Hook struct { + TypeInformation + + // Verbs are the verbs that the hook should mutate + Verbs []string `json:"verbs,omitempty" yaml:"verbs,omitempty"` + + // Patches are the patches to apply on the object to be synced + Patches []*Patch `json:"patches,omitempty" yaml:"patches,omitempty"` +} + +type Import struct { + SyncBase `json:",inline" yaml:",inline"` +} + +type SyncBase struct { + TypeInformation `json:",inline" yaml:",inline"` + + Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` + + // ReplaceWhenInvalid determines if the controller should try to recreate the object + // if there is a problem applying + ReplaceWhenInvalid bool `json:"replaceOnConflict,omitempty" yaml:"replaceOnConflict,omitempty"` + + // Patches are the patches to apply on the virtual cluster objects + // when syncing them from the host cluster + Patches []*Patch `json:"patches,omitempty" yaml:"patches,omitempty"` + + // ReversePatches are the patches to apply to host cluster objects + // after it has been synced to the virtual cluster + ReversePatches []*Patch `json:"reversePatches,omitempty" yaml:"reversePatches,omitempty"` +} + +type Export struct { + SyncBase `json:",inline" yaml:",inline"` + + // Selector is a label selector to select the synced objects in the virtual cluster. + // If empty, all objects will be synced. + Selector *Selector `json:"selector,omitempty" yaml:"selector,omitempty"` +} + +type TypeInformation struct { + // APIVersion of the object to sync + APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty"` + + // Kind of the object to sync + Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` +} + +type Selector struct { + // LabelSelector are the labels to select the object from + LabelSelector map[string]string `json:"labelSelector,omitempty" yaml:"labelSelector,omitempty"` +} + +type Patch struct { + // Operation is the type of the patch + Operation PatchType `json:"op,omitempty" yaml:"op,omitempty"` + + // FromPath is the path from the other object + FromPath string `json:"fromPath,omitempty" yaml:"fromPath,omitempty"` + + // Path is the path of the patch + Path string `json:"path,omitempty" yaml:"path,omitempty"` + + // NamePath is the path to the name of a child resource within Path + NamePath string `json:"namePath,omitempty" yaml:"namePath,omitempty"` + + // NamespacePath is path to the namespace of a child resource within Path + NamespacePath string `json:"namespacePath,omitempty" yaml:"namespacePath,omitempty"` + + // Value is the new value to be set to the path + Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` + + // Regex - is regular expresion used to identify the Name, + // and optionally Namespace, parts of the field value that + // will be replaced with the rewritten Name and/or Namespace + Regex string `json:"regex,omitempty" yaml:"regex,omitempty"` + ParsedRegex *regexp.Regexp `json:"-" yaml:"-"` + + // Conditions are conditions that must be true for + // the patch to get executed + Conditions []*PatchCondition `json:"conditions,omitempty" yaml:"conditions,omitempty"` + + // Ignore determines if the path should be ignored if handled as a reverse patch + Ignore *bool `json:"ignore,omitempty" yaml:"ignore,omitempty"` + + // Sync defines if a specialized syncer should be initialized using values + // from the rewriteName operation as Secret/Configmap names to be synced + Sync *PatchSync `json:"sync,omitempty" yaml:"sync,omitempty"` +} + +type PatchType string + +const ( + PatchTypeRewriteName PatchType = "rewriteName" + PatchTypeRewriteLabelKey PatchType = "rewriteLabelKey" + PatchTypeRewriteLabelSelector PatchType = "rewriteLabelSelector" + PatchTypeRewriteLabelExpressionsSelector PatchType = "rewriteLabelExpressionsSelector" + + PatchTypeCopyFromObject PatchType = "copyFromObject" + PatchTypeAdd PatchType = "add" + PatchTypeReplace PatchType = "replace" + PatchTypeRemove PatchType = "remove" +) + +type PatchCondition struct { + // Path is the path within the object to select + Path string `json:"path,omitempty" yaml:"path,omitempty"` + + // SubPath is the path below the selected object to select + SubPath string `json:"subPath,omitempty" yaml:"subPath,omitempty"` + + // Equal is the value the path should be equal to + Equal interface{} `json:"equal,omitempty" yaml:"equal,omitempty"` + + // NotEqual is the value the path should not be equal to + NotEqual interface{} `json:"notEqual,omitempty" yaml:"notEqual,omitempty"` + + // Empty means that the path value should be empty or unset + Empty *bool `json:"empty,omitempty" yaml:"empty,omitempty"` +} + +type PatchSync struct { + Secret *bool `json:"secret,omitempty" yaml:"secret,omitempty"` + ConfigMap *bool `json:"configmap,omitempty" yaml:"configmap,omitempty"` +} diff --git a/pkg/config/config.go b/pkg/config/config.go index c1b8f0ce9..d3a449b4b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,155 +1,114 @@ package config import ( - "regexp" -) - -const Version = "v1beta1" - -type Config struct { - // Version is the config version - Version string `json:"version,omitempty" yaml:"version,omitempty"` - - // Exports syncs a resource from the virtual cluster to the host - Exports []*Export `json:"export,omitempty" yaml:"export,omitempty"` - - // Imports syncs a resource from the host cluster to virtual cluster - Imports []*Import `json:"import,omitempty" yaml:"import,omitempty"` - - // Hooks are hooks that can be used to inject custom patches before syncing - Hooks *Hooks `json:"hooks,omitempty" yaml:"hooks,omitempty"` -} - -type Hooks struct { - // HostToVirtual is a hook that is executed before syncing from the host to the virtual cluster - HostToVirtual []*Hook `json:"hostToVirtual,omitempty" yaml:"hostToVirtual,omitempty"` - - // VirtualToHost is a hook that is executed before syncing from the virtual to the host cluster - VirtualToHost []*Hook `json:"virtualToHost,omitempty" yaml:"virtualToHost,omitempty"` -} - -type Hook struct { - TypeInformation - - // Verbs are the verbs that the hook should mutate - Verbs []string `json:"verbs,omitempty" yaml:"verbs,omitempty"` - - // Patches are the patches to apply on the object to be synced - Patches []*Patch `json:"patches,omitempty" yaml:"patches,omitempty"` -} - -type Import struct { - SyncBase `json:",inline" yaml:",inline"` -} - -type SyncBase struct { - TypeInformation `json:",inline" yaml:",inline"` - - Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` - - // ReplaceWhenInvalid determines if the controller should try to recreate the object - // if there is a problem applying - ReplaceWhenInvalid bool `json:"replaceOnConflict,omitempty" yaml:"replaceOnConflict,omitempty"` - - // Patches are the patches to apply on the virtual cluster objects - // when syncing them from the host cluster - Patches []*Patch `json:"patches,omitempty" yaml:"patches,omitempty"` - - // ReversePatches are the patches to apply to host cluster objects - // after it has been synced to the virtual cluster - ReversePatches []*Patch `json:"reversePatches,omitempty" yaml:"reversePatches,omitempty"` -} + "strings" -type Export struct { - SyncBase `json:",inline" yaml:",inline"` - - // Selector is a label selector to select the synced objects in the virtual cluster. - // If empty, all objects will be synced. - Selector *Selector `json:"selector,omitempty" yaml:"selector,omitempty"` -} - -type TypeInformation struct { - // APIVersion of the object to sync - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty"` - - // Kind of the object to sync - Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` -} - -type Selector struct { - // LabelSelector are the labels to select the object from - LabelSelector map[string]string `json:"labelSelector,omitempty" yaml:"labelSelector,omitempty"` -} - -type Patch struct { - // Operation is the type of the patch - Operation PatchType `json:"op,omitempty" yaml:"op,omitempty"` - - // FromPath is the path from the other object - FromPath string `json:"fromPath,omitempty" yaml:"fromPath,omitempty"` - - // Path is the path of the patch - Path string `json:"path,omitempty" yaml:"path,omitempty"` - - // NamePath is the path to the name of a child resource within Path - NamePath string `json:"namePath,omitempty" yaml:"namePath,omitempty"` - - // NamespacePath is path to the namespace of a child resource within Path - NamespacePath string `json:"namespacePath,omitempty" yaml:"namespacePath,omitempty"` - - // Value is the new value to be set to the path - Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` - - // Regex - is regular expresion used to identify the Name, - // and optionally Namespace, parts of the field value that - // will be replaced with the rewritten Name and/or Namespace - Regex string `json:"regex,omitempty" yaml:"regex,omitempty"` - ParsedRegex *regexp.Regexp `json:"-" yaml:"-"` - - // Conditions are conditions that must be true for - // the patch to get executed - Conditions []*PatchCondition `json:"conditions,omitempty" yaml:"conditions,omitempty"` - - // Ignore determines if the path should be ignored if handled as a reverse patch - Ignore *bool `json:"ignore,omitempty" yaml:"ignore,omitempty"` - - // Sync defines if a specialized syncer should be initialized using values - // from the rewriteName operation as Secret/Configmap names to be synced - Sync *PatchSync `json:"sync,omitempty" yaml:"sync,omitempty"` -} - -type PatchType string - -const ( - PatchTypeRewriteName = "rewriteName" - PatchTypeRewriteLabelKey = "rewriteLabelKey" - PatchTypeRewriteLabelSelector = "rewriteLabelSelector" - PatchTypeRewriteLabelExpressionsSelector = "rewriteLabelExpressionsSelector" - - PatchTypeCopyFromObject = "copyFromObject" - PatchTypeAdd = "add" - PatchTypeReplace = "replace" - PatchTypeRemove = "remove" + "github.com/loft-sh/vcluster/config" ) -type PatchCondition struct { - // Path is the path within the object to select - Path string `json:"path,omitempty" yaml:"path,omitempty"` - - // SubPath is the path below the selected object to select - SubPath string `json:"subPath,omitempty" yaml:"subPath,omitempty"` +type VirtualClusterConfig struct { + // Holds the vCluster config + config.Config `json:",inline"` - // Equal is the value the path should be equal to - Equal interface{} `json:"equal,omitempty" yaml:"equal,omitempty"` + // Name is the name of the vCluster + Name string `json:"name,omitempty"` - // NotEqual is the value the path should not be equal to - NotEqual interface{} `json:"notEqual,omitempty" yaml:"notEqual,omitempty"` + // ServiceName is the name of the service of the vCluster + ServiceName string `json:"serviceName,omitempty"` - // Empty means that the path value should be empty or unset - Empty *bool `json:"empty,omitempty" yaml:"empty,omitempty"` + // TargetNamespace is the namespace where the workloads go + TargetNamespace string `json:"targetNamespace,omitempty"` } -type PatchSync struct { - Secret *bool `json:"secret,omitempty" yaml:"secret,omitempty"` - ConfigMap *bool `json:"configmap,omitempty" yaml:"configmap,omitempty"` +// LegacyOptions converts the config to the legacy cluster options +func (v VirtualClusterConfig) LegacyOptions() (*LegacyVirtualClusterOptions, error) { + legacyPlugins := []string{} + for pluginName, plugin := range v.Plugin { + if plugin.Version != "" && !plugin.Optional { + continue + } + + legacyPlugins = append(legacyPlugins, pluginName) + } + + nodeSelector := "" + if v.Sync.FromHost.Nodes.Real.Enabled { + selectors := []string{} + for k, v := range v.Sync.FromHost.Nodes.Real.Selector.Labels { + selectors = append(selectors, k+"="+v) + } + + nodeSelector = strings.Join(selectors, ",") + } + + retOptions := &LegacyVirtualClusterOptions{ + ProOptions: LegacyVirtualClusterProOptions{ + RemoteKubeConfig: "", + RemoteNamespace: "", + RemoteServiceName: "", + EnforceValidatingHooks: nil, + EnforceMutatingHooks: nil, + IntegratedCoredns: v.ControlPlane.CoreDNS.Embedded, + UseCoreDNSPlugin: false, + EtcdReplicas: 0, + EtcdEmbedded: v.ControlPlane.BackingStore.EmbeddedEtcd.Enabled, + MigrateFrom: "", + NoopSyncer: false, + SyncKubernetesService: false, + }, + Controllers: nil, + ServerCaCert: "", + ServerCaKey: "", + TLSSANs: nil, + RequestHeaderCaCert: "", + ClientCaCert: "", + KubeConfigPath: "", + KubeConfigContextName: "", + KubeConfigSecret: "", + KubeConfigSecretNamespace: "", + KubeConfigServer: "", + Tolerations: nil, + BindAddress: v.ControlPlane.Proxy.BindAddress, + Port: v.ControlPlane.Proxy.Port, + Name: v.Name, + TargetNamespace: v.TargetNamespace, + ServiceName: v.ServiceName, + SetOwner: false, + SyncAllNodes: v.Sync.FromHost.Nodes.Real.SyncAll, + EnableScheduler: v.ControlPlane.VirtualScheduler.Enabled, + DisableFakeKubelets: false, + FakeKubeletIPs: false, + ClearNodeImages: v.Sync.FromHost.Nodes.Real.ClearImageStatus, + TranslateImages: nil, + NodeSelector: nodeSelector, + ServiceAccount: v.Sync.ToHost.Pods.WorkloadServiceAccount, + EnforceNodeSelector: true, + PluginListenAddress: "localhost:10099", + OverrideHosts: v.Sync.ToHost.Pods.RewriteHosts.Enabled, + OverrideHostsContainerImage: v.Sync.ToHost.Pods.RewriteHosts.InitContainerImage, + ServiceAccountTokenSecrets: v.Sync.ToHost.Pods.UseSecretsForSATokens, + ClusterDomain: v.Networking.Advanced.ClusterDomain, + LeaderElect: v.ControlPlane.StatefulSet.HighAvailability.Replicas > 1, + LeaseDuration: v.ControlPlane.StatefulSet.HighAvailability.LeaseDuration, + RenewDeadline: v.ControlPlane.StatefulSet.HighAvailability.RenewDeadline, + RetryPeriod: v.ControlPlane.StatefulSet.HighAvailability.RetryPeriod, + Plugins: legacyPlugins, + DefaultImageRegistry: v.ControlPlane.Advanced.DefaultImageRegistry, + EnforcePodSecurityStandard: v.Policies.PodSecurityStandard, + MapHostServices: nil, + MapVirtualServices: nil, + SyncLabels: nil, + MountPhysicalHostPaths: false, + HostMetricsBindAddress: "0", + VirtualMetricsBindAddress: "0", + MultiNamespaceMode: v.Experimental.MultiNamespaceMode.Enabled, + NamespaceLabels: nil, + SyncAllSecrets: v.Sync.ToHost.Secrets.All, + SyncAllConfigMaps: v.Sync.ToHost.ConfigMaps.All, + ProxyMetricsServer: v.Observability.Metrics.Proxy.Nodes.Enabled || v.Observability.Metrics.Proxy.Pods.Enabled, + + DeprecatedSyncNodeChanges: v.Sync.FromHost.Nodes.Real.SyncLabelsTaints, + } + + return retOptions, nil } diff --git a/pkg/options/controller_context.go b/pkg/config/controller_context.go similarity index 76% rename from pkg/options/controller_context.go rename to pkg/config/controller_context.go index cabf1455e..cccb2cb7c 100644 --- a/pkg/options/controller_context.go +++ b/pkg/config/controller_context.go @@ -1,11 +1,10 @@ -package options +package config import ( "context" "net/http" servertypes "github.com/loft-sh/vcluster/pkg/server/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/version" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ctrl "sigs.k8s.io/controller-runtime" @@ -23,11 +22,10 @@ type ControllerContext struct { CurrentNamespace string CurrentNamespaceClient client.Client - Controllers sets.Set[string] AdditionalServerFilters []servertypes.Filter - Options *VirtualClusterOptions + Config *VirtualClusterConfig StopChan <-chan struct{} - //set of extra services that should handle the traffic or pass it along + // set of extra services that should handle the traffic or pass it along ExtraHandlers []func(http.Handler) http.Handler } diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go new file mode 100644 index 000000000..f1fd1ba79 --- /dev/null +++ b/pkg/config/defaults.go @@ -0,0 +1,5 @@ +package config + +func FillDefaults(config *VirtualClusterConfig) error { + return nil +} diff --git a/pkg/options/flags.go b/pkg/config/flags.go similarity index 82% rename from pkg/options/flags.go rename to pkg/config/flags.go index f002d6406..db4a1db73 100644 --- a/pkg/options/flags.go +++ b/pkg/config/flags.go @@ -1,9 +1,29 @@ -package options +package config import ( "github.com/spf13/pflag" ) +func AddProFlags(flags *pflag.FlagSet, options *VirtualClusterOptions) { + flags.StringVar(&options.ProOptions.ProLicenseSecret, "pro-license-secret", "", "If set, vCluster.Pro will try to find this secret to retrieve the vCluster.Pro license.") + + flags.StringVar(&options.ProOptions.RemoteKubeConfig, "remote-kube-config", "", "If set, will use the remote kube-config instead of the local in-cluster one. Expects a kube config to a headless vcluster installation") + flags.StringVar(&options.ProOptions.RemoteNamespace, "remote-namespace", "", "If set, will use this as the remote namespace") + flags.StringVar(&options.ProOptions.RemoteServiceName, "remote-service-name", "", "If set, will use this as the remote service name") + + flags.BoolVar(&options.ProOptions.IntegratedCoredns, "integrated-coredns", false, "If enabled vcluster will spin an in memory coreDNS inside the syncer container") + flags.BoolVar(&options.ProOptions.UseCoreDNSPlugin, "use-coredns-plugin", false, "If enabled, the vcluster plugin for coredns will be used") + flags.BoolVar(&options.ProOptions.NoopSyncer, "noop-syncer", false, "If enabled will setup a noop Syncer that filters and proxies requests to a specified remote cluster") + flags.BoolVar(&options.ProOptions.SyncKubernetesService, "sync-k8s-service", false, "If enabled will sync the kubernetes service endpoints in the remote cluster with the load balancer ip of this cluster") + + flags.BoolVar(&options.ProOptions.EtcdEmbedded, "etcd-embedded", false, "If true, will start an embedded etcd within vCluster") + flags.StringVar(&options.ProOptions.MigrateFrom, "migrate-from", "", "The url (including protocol) of the original database") + flags.IntVar(&options.ProOptions.EtcdReplicas, "etcd-replicas", 0, "The amount of replicas the etcd has") + + flags.StringArrayVar(&options.ProOptions.EnforceValidatingHooks, "enforce-validating-hook", nil, "A validating hook configuration in yaml format encoded with base64. Can be used multiple times") + flags.StringArrayVar(&options.ProOptions.EnforceMutatingHooks, "enforce-mutating-hook", nil, "A mutating hook configuration in yaml format encoded with base64. Can be used multiple times") +} + func AddFlags(flags *pflag.FlagSet, options *VirtualClusterOptions) { flags.StringVar(&options.KubeConfigContextName, "kube-config-context-name", "", "If set, will override the context name of the generated virtual cluster kube config with this name") flags.StringSliceVar(&options.Controllers, "sync", []string{}, "A list of sync controllers to enable. 'foo' enables the sync controller named 'foo', '-foo' disables the sync controller named 'foo'") diff --git a/pkg/options/options.go b/pkg/config/options.go similarity index 74% rename from pkg/options/options.go rename to pkg/config/options.go index 84f8a6b9d..6b0ced1b1 100644 --- a/pkg/options/options.go +++ b/pkg/config/options.go @@ -1,14 +1,13 @@ -package options +package config const ( DefaultHostsRewriteImage = "library/alpine:3.13.1" - GenericConfig = "CONFIG" ) -// VirtualClusterOptions holds the cmd flags -type VirtualClusterOptions struct { +// LegacyVirtualClusterOptions holds the cmd flags +type LegacyVirtualClusterOptions struct { // PRO Options - ProOptions VirtualClusterProOptions `json:",inline"` + ProOptions LegacyVirtualClusterProOptions `json:",inline"` // OSS Options below Controllers []string `json:"controllers,omitempty"` @@ -52,12 +51,11 @@ type VirtualClusterOptions struct { ClusterDomain string `json:"clusterDomain,omitempty"` - LeaderElect bool `json:"leaderElect,omitempty"` - LeaseDuration int64 `json:"leaseDuration,omitempty"` - RenewDeadline int64 `json:"renewDeadline,omitempty"` - RetryPeriod int64 `json:"retryPeriod,omitempty"` + LeaderElect bool `json:"leaderElect,omitempty"` + LeaseDuration int `json:"leaseDuration,omitempty"` + RenewDeadline int `json:"renewDeadline,omitempty"` + RetryPeriod int `json:"retryPeriod,omitempty"` - DisablePlugins bool `json:"disablePlugins,omitempty"` PluginListenAddress string `json:"pluginListenAddress,omitempty"` Plugins []string `json:"plugins,omitempty"` @@ -74,11 +72,6 @@ type VirtualClusterOptions struct { // this is only needed if using vcluster-hostpath-mapper component // see: https://github.com/loft-sh/vcluster-hostpath-mapper MountPhysicalHostPaths bool `json:"mountPhysicalHostPaths,omitempty"` - // To enable FSMounts functionality - VirtualLogsPath string - VirtualPodLogsPath string - VirtualContainerLogsPath string - VirtualKubeletPodPath string HostMetricsBindAddress string `json:"hostMetricsBindAddress,omitempty"` VirtualMetricsBindAddress string `json:"virtualMetricsBindAddress,omitempty"` @@ -92,14 +85,5 @@ type VirtualClusterOptions struct { ServiceAccountTokenSecrets bool `json:"serviceAccountTokenSecrets,omitempty"` // DEPRECATED FLAGS - RewriteHostPaths bool `json:"rewriteHostPaths,omitempty"` - DeprecatedSyncNodeChanges bool `json:"syncNodeChanges"` - DeprecatedDisableSyncResources string - DeprecatedOwningStatefulSet string - DeprecatedUseFakeNodes bool - DeprecatedUseFakePersistentVolumes bool - DeprecatedEnableStorageClasses bool - DeprecatedEnablePriorityClasses bool - DeprecatedSuffix string - DeprecatedUseFakeKubelets bool + DeprecatedSyncNodeChanges bool `json:"syncNodeChanges"` } diff --git a/pkg/config/parse.go b/pkg/config/parse.go index 9d0ac29f4..74373a13a 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -2,189 +2,49 @@ package config import ( "fmt" + "os" - "github.com/samber/lo" - "sigs.k8s.io/yaml" + "github.com/ghodss/yaml" + "github.com/loft-sh/vcluster/config" ) -var ( - verbs = []string{"get", "list", "create", "update", "patch", "watch", "delete", "deletecollection"} -) - -func Parse(rawConfig string) (*Config, error) { - c := &Config{} - err := yaml.UnmarshalStrict([]byte(rawConfig), c) +func ParseConfig(path string) (*VirtualClusterConfig, error) { + rawFile, err := os.ReadFile(path) if err != nil { return nil, err } - err = validate(c) + rawConfig := &config.Config{} + err = yaml.Unmarshal(rawFile, rawConfig) if err != nil { return nil, err } - return c, nil -} - -func validate(config *Config) error { - if config.Version != Version { - return fmt.Errorf("unsupported configuration version. Only %s is supported currently", Version) - } - - err := validateExportDuplicates(config.Exports) + retConfig, err := Convert(rawConfig) if err != nil { - return err - } - - for idx, exp := range config.Exports { - if exp == nil { - return fmt.Errorf("exports[%d] is required", idx) - } - - if exp.Kind == "" { - return fmt.Errorf("exports[%d].kind is required", idx) - } - - if exp.APIVersion == "" { - return fmt.Errorf("exports[%d].APIVersion is required", idx) - } - - for patchIdx, patch := range exp.Patches { - err := validatePatch(patch) - if err != nil { - return fmt.Errorf("invalid exports[%d].patches[%d]: %w", idx, patchIdx, err) - } - } - - for patchIdx, patch := range exp.ReversePatches { - err := validatePatch(patch) - if err != nil { - return fmt.Errorf("invalid exports[%d].reversPatches[%d]: %w", idx, patchIdx, err) - } - } + return nil, err } - err = validateImportDuplicates(config.Imports) + err = ValidateConfig(retConfig) if err != nil { - return err - } - - for idx, imp := range config.Imports { - if imp == nil { - return fmt.Errorf("imports[%d] is required", idx) - } - - if imp.Kind == "" { - return fmt.Errorf("imports[%d].kind is required", idx) - } - - if imp.APIVersion == "" { - return fmt.Errorf("imports[%d].APIVersion is required", idx) - } - - for patchIdx, patch := range imp.Patches { - err := validatePatch(patch) - if err != nil { - return fmt.Errorf("invalid imports[%d].patches[%d]: %w", idx, patchIdx, err) - } - } - - for patchIdx, patch := range imp.ReversePatches { - err := validatePatch(patch) - if err != nil { - return fmt.Errorf("invalid imports[%d].reversPatches[%d]: %w", idx, patchIdx, err) - } - } - } - - if config.Hooks != nil { - // HostToVirtual validation - for idx, hook := range config.Hooks.HostToVirtual { - for idy, verb := range hook.Verbs { - if err := validateVerb(verb); err != nil { - return fmt.Errorf("invalid hooks.hostToVirtual[%d].verbs[%d]: %w", idx, idy, err) - } - } - - for idy, patch := range hook.Patches { - if err := validatePatch(patch); err != nil { - return fmt.Errorf("invalid hooks.hostToVirtual[%d].patches[%d]: %w", idx, idy, err) - } - } - } - - // VirtualToHost validation - for idx, hook := range config.Hooks.VirtualToHost { - for idy, verb := range hook.Verbs { - if err := validateVerb(verb); err != nil { - return fmt.Errorf("invalid hooks.virtualToHost[%d].verbs[%d]: %w", idx, idy, err) - } - } - - for idy, patch := range hook.Patches { - if err := validatePatch(patch); err != nil { - return fmt.Errorf("invalid hooks.virtualToHost[%d].patches[%d]: %w", idx, idy, err) - } - } - } + return nil, err } - return nil + return retConfig, nil } -func validatePatch(patch *Patch) error { - switch patch.Operation { - case PatchTypeRemove, PatchTypeReplace, PatchTypeAdd: - if patch.FromPath != "" { - return fmt.Errorf("fromPath is not supported for this operation") - } - - return nil - case PatchTypeRewriteName, PatchTypeRewriteLabelSelector, PatchTypeRewriteLabelKey, PatchTypeRewriteLabelExpressionsSelector: - return nil - case PatchTypeCopyFromObject: - if patch.FromPath == "" { - return fmt.Errorf("fromPath is required for this operation") - } - - return nil - default: - return fmt.Errorf("unsupported patch type %s", patch.Operation) +func Convert(config *config.Config) (*VirtualClusterConfig, error) { + vClusterName := os.Getenv("VCLUSTER_NAME") + if vClusterName == "" { + return nil, fmt.Errorf("environment variable VCLUSTER_NAME is not defined") } -} - -func validateVerb(verb string) error { - if !lo.Contains(verbs, verb) { - return fmt.Errorf("invalid verb \"%s\"; expected on of %q", verb, verbs) - } - - return nil -} -func validateExportDuplicates(exports []*Export) error { - gvks := map[string]bool{} - for _, e := range exports { - k := fmt.Sprintf("%s|%s", e.APIVersion, e.Kind) - _, found := gvks[k] - if found { - return fmt.Errorf("duplicate export for APIVersion %s and %s Kind, only one export for each APIVersion+Kind is permitted", e.APIVersion, e.Kind) - } - gvks[k] = true + retConfig := &VirtualClusterConfig{ + Config: *config, + Name: vClusterName, } - return nil -} - -func validateImportDuplicates(imports []*Import) error { - gvks := map[string]bool{} - for _, e := range imports { - k := fmt.Sprintf("%s|%s", e.APIVersion, e.Kind) - _, found := gvks[k] - if found { - return fmt.Errorf("duplicate import for APIVersion %s and %s Kind, only one import for each APIVersion+Kind is permitted", e.APIVersion, e.Kind) - } - gvks[k] = true - } + // convert legacy options - return nil + return retConfig, nil } diff --git a/pkg/options/pro_options.go b/pkg/config/pro_options.go similarity index 87% rename from pkg/options/pro_options.go rename to pkg/config/pro_options.go index b3e01a850..287fabfcb 100644 --- a/pkg/options/pro_options.go +++ b/pkg/config/pro_options.go @@ -1,8 +1,6 @@ -package options - -type VirtualClusterProOptions struct { - ProLicenseSecret string `json:"proLicenseSecret,omitempty"` +package config +type LegacyVirtualClusterProOptions struct { RemoteKubeConfig string `json:"remoteKubeConfig,omitempty"` RemoteNamespace string `json:"remoteNamespace,omitempty"` RemoteServiceName string `json:"remoteServiceName,omitempty"` diff --git a/pkg/config/validation.go b/pkg/config/validation.go new file mode 100644 index 000000000..763595da8 --- /dev/null +++ b/pkg/config/validation.go @@ -0,0 +1,273 @@ +package config + +import ( + "crypto/x509" + "errors" + "fmt" + "net/url" + + "github.com/ghodss/yaml" + "github.com/loft-sh/vcluster/config" + "github.com/samber/lo" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" +) + +func ValidateConfig(config *VirtualClusterConfig) error { + err := validateCentralAdmissionControl(config) + if err != nil { + return err + } + + err = validateGenericSyncConfig(config.Experimental.GenericSync) + if err != nil { + return fmt.Errorf("validate experimental.genericSync") + } + + return nil +} + +var ( + verbs = []string{"get", "list", "create", "update", "patch", "watch", "delete", "deletecollection"} +) + +func validateGenericSyncConfig(config config.ExperimentalGenericSync) error { + err := validateExportDuplicates(config.Exports) + if err != nil { + return err + } + + for idx, exp := range config.Exports { + if exp == nil { + return fmt.Errorf("exports[%d] is required", idx) + } + + if exp.Kind == "" { + return fmt.Errorf("exports[%d].kind is required", idx) + } + + if exp.APIVersion == "" { + return fmt.Errorf("exports[%d].APIVersion is required", idx) + } + + for patchIdx, patch := range exp.Patches { + err := validatePatch(patch) + if err != nil { + return fmt.Errorf("invalid exports[%d].patches[%d]: %w", idx, patchIdx, err) + } + } + + for patchIdx, patch := range exp.ReversePatches { + err := validatePatch(patch) + if err != nil { + return fmt.Errorf("invalid exports[%d].reversPatches[%d]: %w", idx, patchIdx, err) + } + } + } + + err = validateImportDuplicates(config.Imports) + if err != nil { + return err + } + + for idx, imp := range config.Imports { + if imp == nil { + return fmt.Errorf("imports[%d] is required", idx) + } + + if imp.Kind == "" { + return fmt.Errorf("imports[%d].kind is required", idx) + } + + if imp.APIVersion == "" { + return fmt.Errorf("imports[%d].APIVersion is required", idx) + } + + for patchIdx, patch := range imp.Patches { + err := validatePatch(patch) + if err != nil { + return fmt.Errorf("invalid imports[%d].patches[%d]: %w", idx, patchIdx, err) + } + } + + for patchIdx, patch := range imp.ReversePatches { + err := validatePatch(patch) + if err != nil { + return fmt.Errorf("invalid imports[%d].reversPatches[%d]: %w", idx, patchIdx, err) + } + } + } + + if config.Hooks != nil { + // HostToVirtual validation + for idx, hook := range config.Hooks.HostToVirtual { + for idy, verb := range hook.Verbs { + if err := validateVerb(verb); err != nil { + return fmt.Errorf("invalid hooks.hostToVirtual[%d].verbs[%d]: %w", idx, idy, err) + } + } + + for idy, patch := range hook.Patches { + if err := validatePatch(patch); err != nil { + return fmt.Errorf("invalid hooks.hostToVirtual[%d].patches[%d]: %w", idx, idy, err) + } + } + } + + // VirtualToHost validation + for idx, hook := range config.Hooks.VirtualToHost { + for idy, verb := range hook.Verbs { + if err := validateVerb(verb); err != nil { + return fmt.Errorf("invalid hooks.virtualToHost[%d].verbs[%d]: %w", idx, idy, err) + } + } + + for idy, patch := range hook.Patches { + if err := validatePatch(patch); err != nil { + return fmt.Errorf("invalid hooks.virtualToHost[%d].patches[%d]: %w", idx, idy, err) + } + } + } + } + + return nil +} + +func validatePatch(patch *config.Patch) error { + switch patch.Operation { + case config.PatchTypeRemove, config.PatchTypeReplace, config.PatchTypeAdd: + if patch.FromPath != "" { + return fmt.Errorf("fromPath is not supported for this operation") + } + + return nil + case config.PatchTypeRewriteName, config.PatchTypeRewriteLabelSelector, config.PatchTypeRewriteLabelKey, config.PatchTypeRewriteLabelExpressionsSelector: + return nil + case config.PatchTypeCopyFromObject: + if patch.FromPath == "" { + return fmt.Errorf("fromPath is required for this operation") + } + + return nil + default: + return fmt.Errorf("unsupported patch type %s", patch.Operation) + } +} + +func validateVerb(verb string) error { + if !lo.Contains(verbs, verb) { + return fmt.Errorf("invalid verb \"%s\"; expected on of %q", verb, verbs) + } + + return nil +} + +func validateExportDuplicates(exports []*config.Export) error { + gvks := map[string]bool{} + for _, e := range exports { + k := fmt.Sprintf("%s|%s", e.APIVersion, e.Kind) + _, found := gvks[k] + if found { + return fmt.Errorf("duplicate export for APIVersion %s and %s Kind, only one export for each APIVersion+Kind is permitted", e.APIVersion, e.Kind) + } + gvks[k] = true + } + + return nil +} + +func validateImportDuplicates(imports []*config.Import) error { + gvks := map[string]bool{} + for _, e := range imports { + k := fmt.Sprintf("%s|%s", e.APIVersion, e.Kind) + _, found := gvks[k] + if found { + return fmt.Errorf("duplicate import for APIVersion %s and %s Kind, only one import for each APIVersion+Kind is permitted", e.APIVersion, e.Kind) + } + gvks[k] = true + } + + return nil +} + +func validateCentralAdmissionControl(config *VirtualClusterConfig) error { + _, _, err := ParseExtraHooks(config.Policies.CentralAdmissionControl.ValidatingWebhooks, config.Policies.CentralAdmissionControl.MutatingWebhooks) + return err +} + +func ParseExtraHooks(valHooks, mutHooks []interface{}) ([]admissionregistrationv1.ValidatingWebhookConfiguration, []admissionregistrationv1.MutatingWebhookConfiguration, error) { + decodedVal := make([]string, 0, len(valHooks)) + for _, v := range valHooks { + bytes, err := yaml.Marshal(v) + if err != nil { + return nil, nil, err + } + decodedVal = append(decodedVal, string(bytes)) + } + decodedMut := make([]string, 0, len(mutHooks)) + for _, v := range mutHooks { + bytes, err := yaml.Marshal(v) + if err != nil { + return nil, nil, err + } + decodedMut = append(decodedMut, string(bytes)) + } + + validateConfs := make([]admissionregistrationv1.ValidatingWebhookConfiguration, 0, len(valHooks)) + mutateConfs := make([]admissionregistrationv1.MutatingWebhookConfiguration, 0, len(mutHooks)) + for _, v := range decodedVal { + var valHook admissionregistrationv1.ValidatingWebhookConfiguration + err := yaml.Unmarshal([]byte(v), &valHook) + if err != nil { + return nil, nil, err + } + for _, v := range valHook.Webhooks { + err := validateWebhookClientCfg(v.ClientConfig) + if err != nil { + return nil, nil, fmt.Errorf("webhook client config was not valid for ValidatingWebhookConfiguration %s: %w", v.Name, err) + } + } + validateConfs = append(validateConfs, valHook) + } + for _, v := range decodedMut { + var mutHook admissionregistrationv1.MutatingWebhookConfiguration + err := yaml.Unmarshal([]byte(v), &mutHook) + if err != nil { + return nil, nil, err + } + for _, v := range mutHook.Webhooks { + err := validateWebhookClientCfg(v.ClientConfig) + if err != nil { + return nil, nil, fmt.Errorf("webhook client config was not valid for MutatingWebhookConfiguration %s: %w", v.Name, err) + } + } + mutateConfs = append(mutateConfs, mutHook) + } + + return validateConfs, mutateConfs, nil +} + +func validateWebhookClientCfg(clientCfg admissionregistrationv1.WebhookClientConfig) error { + if len(clientCfg.CABundle) != 0 { + ok := x509.NewCertPool().AppendCertsFromPEM(clientCfg.CABundle) + if !ok { + return errors.New("could not parse the CABundle") + } + } + + if clientCfg.Service == nil && clientCfg.URL == nil { + return errors.New("there is no service config") + } + + if clientCfg.Service != nil && (clientCfg.Service.Name == "" || clientCfg.Service.Namespace == "") { + return errors.New("namespace or name of the service is missing") + } + + if clientCfg.URL != nil { + _, err := url.Parse(*clientCfg.URL) + if err != nil { + return errors.New("the url was not valid") + } + } + + return nil +} diff --git a/pkg/pro/validation_test.go b/pkg/config/validation_test.go similarity index 70% rename from pkg/pro/validation_test.go rename to pkg/config/validation_test.go index 86f584647..9797ed758 100644 --- a/pkg/pro/validation_test.go +++ b/pkg/config/validation_test.go @@ -1,10 +1,8 @@ -package pro +package config import ( - "encoding/base64" "testing" - "github.com/ghodss/yaml" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" ) @@ -52,54 +50,44 @@ HYMfRsCbvUOZ58SWLs5fyQ== testCases := []struct { name string wantErr string - valHook []string - mutHook []string + valHook []interface{} + mutHook []interface{} }{ - { - name: "not valid base64 validating", - valHook: []string{"hello there"}, - wantErr: "illegal base64 data at input byte 5", - }, - { - name: "not valid base64 mutating", - valHook: []string{"hello there"}, - wantErr: "illegal base64 data at input byte 5", - }, { name: "valid valhook", - valHook: []string{valHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test", Name: "service"}})}, + valHook: []interface{}{valHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test", Name: "service"}})}, }, { name: "valid muthook", - mutHook: []string{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test", Name: "service"}})}, + mutHook: []interface{}{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test", Name: "service"}})}, }, { name: "invalid valhook", - valHook: []string{valHookToBase64(admissionregistrationv1.WebhookClientConfig{})}, + valHook: []interface{}{valHookToBase64(admissionregistrationv1.WebhookClientConfig{})}, wantErr: "webhook client config was not valid for ValidatingWebhookConfiguration test: there is no service config", }, { name: "invalid muthook", - mutHook: []string{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{})}, + mutHook: []interface{}{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{})}, wantErr: "webhook client config was not valid for MutatingWebhookConfiguration test: there is no service config", }, { name: "invalid service", - mutHook: []string{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test"}})}, + mutHook: []interface{}{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test"}})}, wantErr: "webhook client config was not valid for MutatingWebhookConfiguration test: namespace or name of the service is missing", }, { name: "valid url", - mutHook: []string{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{URL: &validURL})}, + mutHook: []interface{}{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{URL: &validURL})}, }, { name: "invalid bundle", - mutHook: []string{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test"}, CABundle: []byte("HAZAA")})}, + mutHook: []interface{}{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test"}, CABundle: []byte("HAZAA")})}, wantErr: "webhook client config was not valid for MutatingWebhookConfiguration test: could not parse the CABundle", }, { name: "valid bundle", - mutHook: []string{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test", Name: "test"}, CABundle: []byte(validCABUNDLE)})}, + mutHook: []interface{}{mutHookToBase64(admissionregistrationv1.WebhookClientConfig{Service: &admissionregistrationv1.ServiceReference{Namespace: "test", Name: "test"}, CABundle: []byte(validCABUNDLE)})}, }, } for _, tt := range testCases { @@ -114,32 +102,24 @@ HYMfRsCbvUOZ58SWLs5fyQ== } } -func valHookToBase64(clientCfg admissionregistrationv1.WebhookClientConfig) string { - hook := admissionregistrationv1.ValidatingWebhookConfiguration{} +func valHookToBase64(clientCfg admissionregistrationv1.WebhookClientConfig) interface{} { + hook := &admissionregistrationv1.ValidatingWebhookConfiguration{} hook.APIVersion = "v1" hook.Kind = "ValidatingWebhookConfiguration" hook.Name = "test" hook.Webhooks = []admissionregistrationv1.ValidatingWebhook{ {Name: "test", ClientConfig: clientCfg}, } - bytes, err := yaml.Marshal(hook) - if err != nil { - return "" - } - return base64.StdEncoding.EncodeToString(bytes) + return hook } -func mutHookToBase64(clientCfg admissionregistrationv1.WebhookClientConfig) string { - hook := admissionregistrationv1.MutatingWebhookConfiguration{} +func mutHookToBase64(clientCfg admissionregistrationv1.WebhookClientConfig) interface{} { + hook := &admissionregistrationv1.MutatingWebhookConfiguration{} hook.APIVersion = "v1" hook.Kind = "MutatingWebhookConfiguration" hook.Name = "test" hook.Webhooks = []admissionregistrationv1.MutatingWebhook{ {Name: "test", ClientConfig: clientCfg}, } - bytes, err := yaml.Marshal(hook) - if err != nil { - return "" - } - return base64.StdEncoding.EncodeToString(bytes) + return hook } diff --git a/pkg/controllers/generic/export_syncer.go b/pkg/controllers/generic/export_syncer.go index 1e0050fbb..b9e486f24 100644 --- a/pkg/controllers/generic/export_syncer.go +++ b/pkg/controllers/generic/export_syncer.go @@ -7,13 +7,13 @@ import ( "strings" "time" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" "github.com/loft-sh/vcluster/pkg/log" - "github.com/loft-sh/vcluster/pkg/config" + config2 "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/controllers/syncer" synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" "github.com/loft-sh/vcluster/pkg/controllers/syncer/translator" @@ -32,7 +32,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -func CreateExporters(ctx *options.ControllerContext, exporterConfig *config.Config) error { +func CreateExporters(ctx *config.ControllerContext) error { + exporterConfig := ctx.Config.Experimental.GenericSync if len(exporterConfig.Exports) == 0 { return nil } @@ -58,9 +59,9 @@ func CreateExporters(ctx *options.ControllerContext, exporterConfig *config.Conf } } - reversePatches := []*config.Patch{ + reversePatches := []*config2.Patch{ { - Operation: config.PatchTypeCopyFromObject, + Operation: config2.PatchTypeCopyFromObject, FromPath: "status", Path: "status", }, @@ -84,7 +85,7 @@ func CreateExporters(ctx *options.ControllerContext, exporterConfig *config.Conf return nil } -func createExporter(ctx *synccontext.RegisterContext, config *config.Export) (syncertypes.Syncer, error) { +func createExporter(ctx *synccontext.RegisterContext, config *config2.Export) (syncertypes.Syncer, error) { obj := &unstructured.Unstructured{} obj.SetKind(config.Kind) obj.SetAPIVersion(config.APIVersion) @@ -127,7 +128,7 @@ type exporter struct { patcher *patcher gvk schema.GroupVersionKind - config *config.Export + config *config2.Export selector labels.Selector name string } @@ -346,7 +347,7 @@ func (r *virtualToHostNameResolver) TranslateNamespaceRef(namespace string) (str return translate.Default.PhysicalNamespace(namespace), nil } -func validateExportConfig(config *config.Export) error { +func validateExportConfig(config *config2.Export) error { for _, p := range append(config.Patches, config.ReversePatches...) { if p.Regex != "" { parsed, err := patchesregex.PrepareRegex(p.Regex) diff --git a/pkg/controllers/generic/import_syncer.go b/pkg/controllers/generic/import_syncer.go index e939bb88a..bf3707aad 100644 --- a/pkg/controllers/generic/import_syncer.go +++ b/pkg/controllers/generic/import_syncer.go @@ -7,11 +7,11 @@ import ( "strings" "time" - "github.com/loft-sh/vcluster/pkg/options" + config2 "github.com/loft-sh/vcluster/config" + "github.com/loft-sh/vcluster/pkg/config" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" - "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/controllers/syncer" synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" @@ -33,13 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -func CreateImporters(ctx *options.ControllerContext, cfg *config.Config) error { +func CreateImporters(ctx *config.ControllerContext) error { + cfg := ctx.Config.Experimental.GenericSync if len(cfg.Imports) == 0 { return nil } registerCtx := util.ToRegisterContext(ctx) - if !registerCtx.Options.MultiNamespaceMode { + if !registerCtx.Config.Experimental.MultiNamespaceMode.Enabled { return fmt.Errorf("invalid configuration, 'import' type sync of the generic CRDs is allowed only in the multi-namespace mode") } @@ -84,7 +85,7 @@ func CreateImporters(ctx *options.ControllerContext, cfg *config.Config) error { return nil } -func createImporter(ctx *synccontext.RegisterContext, config *config.Import, gvkRegister GVKRegister) (syncertypes.Syncer, error) { +func createImporter(ctx *synccontext.RegisterContext, config *config2.Import, gvkRegister GVKRegister) (syncertypes.Syncer, error) { gvk := schema.FromAPIVersionAndKind(config.APIVersion, config.Kind) controllerID := fmt.Sprintf("%s/%s/GenericImport", strings.ToLower(gvk.Kind), strings.ToLower(gvk.GroupVersion().String())) @@ -116,7 +117,7 @@ type importer struct { translator.Translator virtualClient client.Client patcher *patcher - config *config.Import + config *config2.Import syncerOptions *syncertypes.Options gvk schema.GroupVersionKind name string diff --git a/pkg/controllers/generic/patcher.go b/pkg/controllers/generic/patcher.go index 5ef9145fe..89e9c0162 100644 --- a/pkg/controllers/generic/patcher.go +++ b/pkg/controllers/generic/patcher.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/genericsyncconfig" "github.com/loft-sh/vcluster/pkg/log" "github.com/loft-sh/vcluster/pkg/patches" "github.com/pkg/errors" @@ -25,7 +25,7 @@ type patcher struct { statusIsSubresource bool } -func (s *patcher) ApplyPatches(ctx context.Context, fromObj, toObj client.Object, patchesConfig, reversePatchesConfig []*config.Patch, translateMetadata func(vObj client.Object) (client.Object, error), nameResolver patches.NameResolver) (client.Object, error) { +func (s *patcher) ApplyPatches(ctx context.Context, fromObj, toObj client.Object, patchesConfig, reversePatchesConfig []*genericsyncconfig.Patch, translateMetadata func(vObj client.Object) (client.Object, error), nameResolver patches.NameResolver) (client.Object, error) { translatedObject, err := translateMetadata(fromObj) if err != nil { return nil, errors.Wrap(err, "translate object") @@ -76,7 +76,7 @@ func (s *patcher) ApplyPatches(ctx context.Context, fromObj, toObj client.Object return outObject, nil } -func (s *patcher) ApplyReversePatches(ctx context.Context, fromObj, otherObj client.Object, reversePatchConfig []*config.Patch, nameResolver patches.NameResolver) (controllerutil.OperationResult, error) { +func (s *patcher) ApplyReversePatches(ctx context.Context, fromObj, otherObj client.Object, reversePatchConfig []*genericsyncconfig.Patch, nameResolver patches.NameResolver) (controllerutil.OperationResult, error) { originalUnstructured, err := toUnstructured(fromObj) if err != nil { return controllerutil.OperationResultNone, err diff --git a/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go b/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go index 23df1281c..e3c93c350 100644 --- a/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go +++ b/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go @@ -5,8 +5,8 @@ import ( "fmt" "time" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/constants" - "github.com/loft-sh/vcluster/pkg/options" "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/loft-sh/vcluster/pkg/specialservices" @@ -45,11 +45,11 @@ type EndpointController struct { provider provider } -func NewEndpointController(ctx *options.ControllerContext, provider provider) *EndpointController { +func NewEndpointController(ctx *config.ControllerContext, provider provider) *EndpointController { return &EndpointController{ LocalClient: ctx.LocalManager.GetClient(), VirtualClient: ctx.VirtualManager.GetClient(), - ServiceName: ctx.Options.ServiceName, + ServiceName: ctx.Config.ServiceName, ServiceNamespace: ctx.CurrentNamespace, VirtualManagerCache: ctx.VirtualManager.GetCache(), Log: loghelper.New("kubernetes-default-endpoint-controller"), diff --git a/pkg/controllers/k8sdefaultendpoint/register.go b/pkg/controllers/k8sdefaultendpoint/register.go index d9e1bb224..fc6245169 100644 --- a/pkg/controllers/k8sdefaultendpoint/register.go +++ b/pkg/controllers/k8sdefaultendpoint/register.go @@ -1,13 +1,13 @@ package k8sdefaultendpoint import ( - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/discovery" "k8s.io/klog/v2" ) -func Register(ctx *options.ControllerContext) error { +func Register(ctx *config.ControllerContext) error { discoveryClient, err := discovery.NewDiscoveryClientForConfig(ctx.VirtualManager.GetConfig()) if err != nil { return err diff --git a/pkg/controllers/register.go b/pkg/controllers/register.go index 574826b12..008ed4e1e 100644 --- a/pkg/controllers/register.go +++ b/pkg/controllers/register.go @@ -4,18 +4,16 @@ import ( "context" "fmt" "net/http" - "os" "strings" "time" "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "github.com/loft-sh/vcluster/pkg/util/translate" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" - "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/controllers/generic" "github.com/loft-sh/vcluster/pkg/controllers/servicesync" "github.com/loft-sh/vcluster/pkg/controllers/syncer" @@ -89,7 +87,7 @@ var ResourceControllers = map[string][]func(*synccontext.RegisterContext) (synce "persistentvolumes,fake-persistentvolumes": {persistentvolumes.New}, } -func Create(ctx *options.ControllerContext) ([]syncertypes.Object, error) { +func Create(ctx *config.ControllerContext) ([]syncertypes.Object, error) { registerContext := util.ToRegisterContext(ctx) // register controllers for resource synchronization @@ -100,12 +98,12 @@ func Create(ctx *options.ControllerContext) ([]syncertypes.Object, error) { for _, controller := range controllers { if ctx.Controllers.Has(controller) { loghelper.Infof("Start %s sync controller", controller) - ctrl, err := controllerNew(registerContext) + createdController, err := controllerNew(registerContext) if err != nil { return nil, errors.Wrapf(err, "register %s controller", controller) } - syncers = append(syncers, ctrl) + syncers = append(syncers, createdController) break } } @@ -115,7 +113,7 @@ func Create(ctx *options.ControllerContext) ([]syncertypes.Object, error) { return syncers, nil } -func ExecuteInitializers(controllerCtx *options.ControllerContext, syncers []syncertypes.Object) error { +func ExecuteInitializers(controllerCtx *config.ControllerContext, syncers []syncertypes.Object) error { registerContext := util.ToRegisterContext(controllerCtx) // execute in parallel because each one might be time-consuming @@ -138,7 +136,7 @@ func ExecuteInitializers(controllerCtx *options.ControllerContext, syncers []syn return errorGroup.Wait() } -func RegisterIndices(ctx *options.ControllerContext, syncers []syncertypes.Object) error { +func RegisterIndices(ctx *config.ControllerContext, syncers []syncertypes.Object) error { registerContext := util.ToRegisterContext(ctx) for _, s := range syncers { indexRegisterer, ok := s.(syncertypes.IndicesRegisterer) @@ -153,7 +151,7 @@ func RegisterIndices(ctx *options.ControllerContext, syncers []syncertypes.Objec return nil } -func RegisterControllers(ctx *options.ControllerContext, syncers []syncertypes.Object) error { +func RegisterControllers(ctx *config.ControllerContext, syncers []syncertypes.Object) error { registerContext := util.ToRegisterContext(ctx) err := k8sdefaultendpoint.Register(ctx) @@ -162,7 +160,7 @@ func RegisterControllers(ctx *options.ControllerContext, syncers []syncertypes.O } // register controller that maintains pod security standard check - if ctx.Options.EnforcePodSecurityStandard != "" { + if ctx.Config.Policies.PodSecurityStandard != "" { err := RegisterPodSecurityController(ctx) if err != nil { return err @@ -218,31 +216,13 @@ func RegisterControllers(ctx *options.ControllerContext, syncers []syncertypes.O return nil } -func RegisterGenericSyncController(ctx *options.ControllerContext) error { - // first check if a generic CRD config is provided and we actually need - // to create any of these syncer controllers - c := os.Getenv(options.GenericConfig) - if strings.TrimSpace(c) == "" || strings.TrimSpace(c) == "---" { - // empty configuration, no need for creating any syncer controllers - loghelper.Infof("no generic config provided, skipping creating controllers") - - return nil - } - - configuration, err := config.Parse(c) - if err != nil { - loghelper.Infof("error parsing the config %v", err.Error()) - return errors.Wrapf(err, "parsing the config") - } - - loghelper.Infof("generic config provided, parsed successfully") - - err = generic.CreateExporters(ctx, configuration) +func RegisterGenericSyncController(ctx *config.ControllerContext) error { + err := generic.CreateExporters(ctx) if err != nil { return err } - err = generic.CreateImporters(ctx, configuration) + err = generic.CreateImporters(ctx) if err != nil { return err } @@ -250,7 +230,7 @@ func RegisterGenericSyncController(ctx *options.ControllerContext) error { return nil } -func RegisterInitManifestsController(controllerCtx *options.ControllerContext) error { +func RegisterInitManifestsController(controllerCtx *config.ControllerContext) error { vconfig, err := kubeconfig.ConvertRestConfigToClientConfig(controllerCtx.VirtualManager.GetConfig()) if err != nil { return err @@ -296,9 +276,9 @@ func RegisterInitManifestsController(controllerCtx *options.ControllerContext) e return nil } -func RegisterServiceSyncControllers(ctx *options.ControllerContext) error { +func RegisterServiceSyncControllers(ctx *config.ControllerContext) error { hostNamespace := ctx.Options.TargetNamespace - if ctx.Options.MultiNamespaceMode { + if ctx.Config.Experimental.MultiNamespaceMode.Enabled { hostNamespace = ctx.CurrentNamespace } @@ -363,7 +343,7 @@ func RegisterServiceSyncControllers(ctx *options.ControllerContext) error { Log: loghelper.New("map-virtual-service-syncer"), } - if ctx.Options.MultiNamespaceMode { + if ctx.Config.Experimental.MultiNamespaceMode.Enabled { controller.CreateEndpoints = true } @@ -424,7 +404,7 @@ func parseMapping(mappings []string, fromDefaultNamespace, toDefaultNamespace st return ret, nil } -func RegisterCoreDNSController(ctx *options.ControllerContext) error { +func RegisterCoreDNSController(ctx *config.ControllerContext) error { controller := &coredns.NodeHostsReconciler{ Client: ctx.VirtualManager.GetClient(), Log: loghelper.New("corednsnodehosts-controller"), @@ -436,10 +416,10 @@ func RegisterCoreDNSController(ctx *options.ControllerContext) error { return nil } -func RegisterPodSecurityController(ctx *options.ControllerContext) error { +func RegisterPodSecurityController(ctx *config.ControllerContext) error { controller := &podsecurity.Reconciler{ Client: ctx.VirtualManager.GetClient(), - PodSecurityStandard: ctx.Options.EnforcePodSecurityStandard, + PodSecurityStandard: ctx.Config.Policies.PodSecurityStandard, Log: loghelper.New("podSecurity-controller"), } err := controller.SetupWithManager(ctx.VirtualManager) diff --git a/pkg/controllers/resources/namespaces/syncer.go b/pkg/controllers/resources/namespaces/syncer.go index c38f8fa3c..c7c9e8c59 100644 --- a/pkg/controllers/resources/namespaces/syncer.go +++ b/pkg/controllers/resources/namespaces/syncer.go @@ -26,8 +26,8 @@ var excludedAnnotations = []string{ } const ( - VclusterNameAnnotation = "vcluster.loft.sh/vcluster-name" - VclusterNamespaceAnnotation = "vcluster.loft.sh/vcluster-namespace" + VClusterNameAnnotation = "vcluster.loft.sh/vcluster-name" + VClusterNamespaceAnnotation = "vcluster.loft.sh/vcluster-namespace" ) func New(ctx *synccontext.RegisterContext) (syncertypes.Object, error) { @@ -36,8 +36,8 @@ func New(ctx *synccontext.RegisterContext) (syncertypes.Object, error) { return nil, fmt.Errorf("invalid value of the namespace-labels flag: %w", err) } - namespaceLabels[VclusterNameAnnotation] = ctx.Options.Name - namespaceLabels[VclusterNamespaceAnnotation] = ctx.CurrentNamespace + namespaceLabels[VClusterNameAnnotation] = ctx.Options.Name + namespaceLabels[VClusterNamespaceAnnotation] = ctx.CurrentNamespace return &namespaceSyncer{ Translator: translator.NewClusterTranslator(ctx, "namespace", &corev1.Namespace{}, NamespaceNameTranslator, excludedAnnotations...), diff --git a/pkg/controllers/resources/nodes/translate.go b/pkg/controllers/resources/nodes/translate.go index ab44516d1..8925dc055 100644 --- a/pkg/controllers/resources/nodes/translate.go +++ b/pkg/controllers/resources/nodes/translate.go @@ -111,8 +111,8 @@ func (s *nodeSyncer) translateUpdateBackwards(pNode *corev1.Node, vNode *corev1. } // add annotation to prevent scale down of node by cluster-autoscaler - // the env var VCLUSTER_NODE_NAME is set when only one replica of vcluster is running - if nodeName, set := os.LookupEnv("VCLUSTER_NODE_NAME"); set && nodeName == pNode.Name { + // the env var NODE_NAME is set when only one replica of vcluster is running + if nodeName, set := os.LookupEnv("NODE_NAME"); set && nodeName == pNode.Name { annotations["cluster-autoscaler.kubernetes.io/scale-down-disabled"] = "true" } diff --git a/pkg/controllers/resources/pods/translate/image_translator.go b/pkg/controllers/resources/pods/translate/image_translator.go index 6b25d4246..3d59fd96a 100644 --- a/pkg/controllers/resources/pods/translate/image_translator.go +++ b/pkg/controllers/resources/pods/translate/image_translator.go @@ -1,10 +1,5 @@ package translate -import ( - "fmt" - "strings" -) - type ImageTranslator interface { Translate(image string) string } @@ -13,19 +8,9 @@ type imageTranslator struct { translateImages map[string]string } -func NewImageTranslator(translateImages []string) (ImageTranslator, error) { - translateImagesMap := map[string]string{} - for _, t := range translateImages { - i := strings.Split(strings.TrimSpace(t), "=") - if len(i) != 2 { - return nil, fmt.Errorf("error parsing translate image '%s': bad format expected image1=image2", t) - } - - translateImagesMap[i[0]] = i[1] - } - +func NewImageTranslator(translateImages map[string]string) (ImageTranslator, error) { return &imageTranslator{ - translateImages: translateImagesMap, + translateImages: translateImages, }, nil } diff --git a/pkg/controllers/syncer/context/context.go b/pkg/controllers/syncer/context/context.go index 58749c70a..ed691bda5 100644 --- a/pkg/controllers/syncer/context/context.go +++ b/pkg/controllers/syncer/context/context.go @@ -3,9 +3,8 @@ package context import ( "context" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/loghelper" - "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -24,8 +23,7 @@ type SyncContext struct { type RegisterContext struct { Context context.Context - Options *options.VirtualClusterOptions - Controllers sets.Set[string] + Config *config.VirtualClusterConfig CurrentNamespace string CurrentNamespaceClient client.Client diff --git a/pkg/genericsyncconfig/config.go b/pkg/genericsyncconfig/config.go new file mode 100644 index 000000000..9297f6bbf --- /dev/null +++ b/pkg/genericsyncconfig/config.go @@ -0,0 +1,3 @@ +package genericsyncconfig + +const Version = "v1beta1" diff --git a/pkg/config/config_test.go b/pkg/genericsyncconfig/config_test.go similarity index 96% rename from pkg/config/config_test.go rename to pkg/genericsyncconfig/config_test.go index 299ccf3c5..82d8eed53 100644 --- a/pkg/config/config_test.go +++ b/pkg/genericsyncconfig/config_test.go @@ -1,4 +1,4 @@ -package config +package genericsyncconfig import ( "strings" @@ -23,7 +23,7 @@ export: # Old from virtual cluster value: prometheus-instance: default-instance - op: copyFromObject - fromPath: .metadata.labels['prometheus-instance'] + fromPath: .metadata.labels['prometheus-instance'] path: .metadata.labels['prometheus-instance'] - op: replace path: .spec.namespaceSelector @@ -40,7 +40,7 @@ export: # Old from virtual cluster path: .spec.podTargetLabels[*] - op: rewriteLabelExpressionsSelector path: .spec.selector - reversePatches: + reversePatches: - op: copyFromObject # Sync status back by default fromPath: status path: status` diff --git a/pkg/genericsyncconfig/parse.go b/pkg/genericsyncconfig/parse.go new file mode 100644 index 000000000..92daff210 --- /dev/null +++ b/pkg/genericsyncconfig/parse.go @@ -0,0 +1,2 @@ +package genericsyncconfig + diff --git a/pkg/config/helmvalues/values_test.go b/pkg/helmvalues/values_test.go similarity index 100% rename from pkg/config/helmvalues/values_test.go rename to pkg/helmvalues/values_test.go diff --git a/pkg/leaderelection/leaderelection.go b/pkg/leaderelection/leaderelection.go index 97878c050..6b515ee93 100644 --- a/pkg/leaderelection/leaderelection.go +++ b/pkg/leaderelection/leaderelection.go @@ -6,7 +6,7 @@ import ( "os" "time" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" @@ -21,7 +21,7 @@ import ( "k8s.io/klog/v2" ) -func StartLeaderElection(ctx *options.ControllerContext, scheme *runtime.Scheme, run func() error) error { +func StartLeaderElection(ctx *config.ControllerContext, scheme *runtime.Scheme, run func() error) error { localConfig := ctx.LocalManager.GetConfig() // create the event recorder @@ -65,9 +65,9 @@ func StartLeaderElection(ctx *options.ControllerContext, scheme *runtime.Scheme, // try and become the leader and start controller manager loops leaderelection.RunOrDie(ctx.Context, leaderelection.LeaderElectionConfig{ Lock: rl, - LeaseDuration: time.Duration(ctx.Options.LeaseDuration) * time.Second, - RenewDeadline: time.Duration(ctx.Options.RenewDeadline) * time.Second, - RetryPeriod: time.Duration(ctx.Options.RetryPeriod) * time.Second, + LeaseDuration: time.Duration(ctx.Config.ControlPlane.StatefulSet.HighAvailability.LeaseDuration) * time.Second, + RenewDeadline: time.Duration(ctx.Config.ControlPlane.StatefulSet.HighAvailability.RenewDeadline) * time.Second, + RetryPeriod: time.Duration(ctx.Config.ControlPlane.StatefulSet.HighAvailability.RetryPeriod) * time.Second, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: func(_ context.Context) { klog.Info("Acquired leadership and run vcluster in leader mode") diff --git a/pkg/patches/conditions.go b/pkg/patches/conditions.go index dcc128dbb..0272fced6 100644 --- a/pkg/patches/conditions.go +++ b/pkg/patches/conditions.go @@ -1,13 +1,13 @@ package patches import ( - "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/genericsyncconfig" "github.com/pkg/errors" "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" yaml "gopkg.in/yaml.v3" ) -func ValidateAllConditions(obj *yaml.Node, match *yaml.Node, conditions []*config.PatchCondition) (bool, error) { +func ValidateAllConditions(obj *yaml.Node, match *yaml.Node, conditions []*genericsyncconfig.PatchCondition) (bool, error) { for _, condition := range conditions { matched, err := ValidateCondition(obj, match, condition) if err != nil { @@ -20,7 +20,7 @@ func ValidateAllConditions(obj *yaml.Node, match *yaml.Node, conditions []*confi return true, nil } -func ValidateCondition(obj *yaml.Node, match *yaml.Node, condition *config.PatchCondition) (bool, error) { +func ValidateCondition(obj *yaml.Node, match *yaml.Node, condition *genericsyncconfig.PatchCondition) (bool, error) { if condition == nil { return true, nil } diff --git a/pkg/patches/patch.go b/pkg/patches/patch.go index e679fdf75..8b20dbae2 100644 --- a/pkg/patches/patch.go +++ b/pkg/patches/patch.go @@ -8,7 +8,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" jsonyaml "github.com/ghodss/yaml" - "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/genericsyncconfig" "github.com/pkg/errors" yaml "gopkg.in/yaml.v3" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,7 +23,7 @@ type NameResolver interface { TranslateNamespaceRef(namespace string) (string, error) } -func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*config.Patch, reversePatchesConf []*config.Patch, nameResolver NameResolver) error { +func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*genericsyncconfig.Patch, reversePatchesConf []*genericsyncconfig.Patch, nameResolver NameResolver) error { node1, err := NewJSONNode(destObj) if err != nil { return errors.Wrap(err, "new json yaml node") @@ -50,8 +50,8 @@ func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*config.Patch, continue } - err := applyPatch(node1, node2, &config.Patch{ - Operation: config.PatchTypeRemove, + err := applyPatch(node1, node2, &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRemove, Path: p.Path, }, nameResolver) if err != nil { @@ -72,23 +72,23 @@ func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*config.Patch, return nil } -func applyPatch(obj1, obj2 *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func applyPatch(obj1, obj2 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { switch patch.Operation { - case config.PatchTypeRewriteName: + case genericsyncconfig.PatchTypeRewriteName: return RewriteName(obj1, patch, resolver) - case config.PatchTypeRewriteLabelKey: + case genericsyncconfig.PatchTypeRewriteLabelKey: return RewriteLabelKey(obj1, patch, resolver) - case config.PatchTypeRewriteLabelExpressionsSelector: + case genericsyncconfig.PatchTypeRewriteLabelExpressionsSelector: return RewriteLabelExpressionsSelector(obj1, patch, resolver) - case config.PatchTypeRewriteLabelSelector: + case genericsyncconfig.PatchTypeRewriteLabelSelector: return RewriteLabelSelector(obj1, patch, resolver) - case config.PatchTypeReplace: + case genericsyncconfig.PatchTypeReplace: return Replace(obj1, patch) - case config.PatchTypeRemove: + case genericsyncconfig.PatchTypeRemove: return Remove(obj1, patch) - case config.PatchTypeAdd: + case genericsyncconfig.PatchTypeAdd: return Add(obj1, patch) - case config.PatchTypeCopyFromObject: + case genericsyncconfig.PatchTypeCopyFromObject: return CopyFromObject(obj1, obj2, patch) } diff --git a/pkg/patches/patch_test.go b/pkg/patches/patch_test.go index cee53c637..6bf79e34e 100644 --- a/pkg/patches/patch_test.go +++ b/pkg/patches/patch_test.go @@ -10,7 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/genericsyncconfig" patchesregex "github.com/loft-sh/vcluster/pkg/patches/regex" "github.com/loft-sh/vcluster/pkg/util/translate" yaml "gopkg.in/yaml.v3" @@ -19,7 +19,7 @@ import ( type patchTestCase struct { name string - patch *config.Patch + patch *genericsyncconfig.Patch obj1 string obj2 string @@ -37,8 +37,8 @@ func TestPatch(t *testing.T) { testCases := []*patchTestCase{ { name: "copy merge", - patch: &config.Patch{ - Operation: config.PatchTypeCopyFromObject, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeCopyFromObject, FromPath: "status.test", Path: "test", }, @@ -52,8 +52,8 @@ test: test`, }, { name: "copy", - patch: &config.Patch{ - Operation: config.PatchTypeCopyFromObject, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeCopyFromObject, FromPath: "status", Path: "status", }, @@ -66,8 +66,8 @@ status: }, { name: "simple", - patch: &config.Patch{ - Operation: config.PatchTypeReplace, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeReplace, Path: "test.test2", Value: "abc", }, @@ -78,8 +78,8 @@ status: }, { name: "insert", - patch: &config.Patch{ - Operation: config.PatchTypeAdd, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeAdd, Path: "test.test2[0].test3", Value: "abc", }, @@ -94,8 +94,8 @@ test2: {}`, }, { name: "insert slice", - patch: &config.Patch{ - Operation: config.PatchTypeAdd, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeAdd, Path: "test.test2", Value: "abc", }, @@ -109,8 +109,8 @@ test2: {}`, }, { name: "insert slice", - patch: &config.Patch{ - Operation: config.PatchTypeReplace, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeReplace, Path: "test..abc", Value: "def", }, @@ -125,11 +125,11 @@ test2: {}`, }, { name: "condition", - patch: &config.Patch{ - Operation: config.PatchTypeReplace, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeReplace, Path: "test.abc", Value: "def", - Conditions: []*config.PatchCondition{ + Conditions: []*genericsyncconfig.PatchCondition{ { Path: "test.status", Empty: &True, @@ -143,11 +143,11 @@ test2: {}`, }, { name: "condition equal", - patch: &config.Patch{ - Operation: config.PatchTypeReplace, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeReplace, Path: "test.abc", Value: "def", - Conditions: []*config.PatchCondition{ + Conditions: []*genericsyncconfig.PatchCondition{ { Path: "test.status", Equal: map[string]interface{}{ @@ -167,11 +167,11 @@ test2: {}`, }, { name: "condition equal", - patch: &config.Patch{ - Operation: config.PatchTypeReplace, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeReplace, Path: "test.abc", Value: "def", - Conditions: []*config.PatchCondition{ + Conditions: []*genericsyncconfig.PatchCondition{ { Path: "test.status", Equal: map[string]interface{}{ @@ -191,8 +191,8 @@ test2: {}`, }, { name: "resolve label selector", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteLabelSelector, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteLabelSelector, Path: "test.abc", }, nameResolver: &fakeNameResolver{}, @@ -204,8 +204,8 @@ test2: {}`, }, { name: "resolve empty label selector", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteLabelSelector, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteLabelSelector, Path: "test.abc", }, nameResolver: &fakeNameResolver{}, @@ -216,8 +216,8 @@ test2: {}`, }, { name: "resolve filled label selector", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteLabelSelector, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteLabelSelector, Path: "test.abc", }, nameResolver: &fakeNameResolver{}, @@ -231,8 +231,8 @@ test2: {}`, }, { name: "rewrite name", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "name", }, nameResolver: &fakeVirtualToHostNameResolver{ @@ -244,8 +244,8 @@ test2: {}`, }, { name: "rewrite name - invalid object", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "name{]", }, obj1: `name: {}`, @@ -253,8 +253,8 @@ test2: {}`, }, { name: "rewrite name - namespace based", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "root.list", NamePath: "nm", NamespacePath: "ns", @@ -278,8 +278,8 @@ test2: {}`, }, { name: "rewrite name - multiple - no namespace", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "root.list", NamePath: "nm", }, @@ -302,8 +302,8 @@ test2: {}`, }, { name: "rewrite name - multiple name matches", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "root.includes", NamePath: "names..nm", NamespacePath: "namespace", @@ -327,8 +327,8 @@ test2: {}`, }, { name: "rewrite name - single name match - non array", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "root.includes", NamePath: "nm", NamespacePath: "namespace", @@ -348,8 +348,8 @@ test2: {}`, }, { name: "rewrite name - multiple name matches - multiple namespace references", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "root.includes", NamePath: "..names..nm", NamespacePath: "..namespaces..ns", @@ -398,8 +398,8 @@ test2: {}`, // }, { name: "rewrite name should not panic when match is not scalar", - patch: &config.Patch{ - Operation: config.PatchTypeRewriteName, + patch: &genericsyncconfig.Patch{ + Operation: genericsyncconfig.PatchTypeRewriteName, Path: "test.endpoints[*]", }, nameResolver: &fakeVirtualToHostNameResolver{}, diff --git a/pkg/patches/patch_types.go b/pkg/patches/patch_types.go index e3815d489..fca1f8b57 100644 --- a/pkg/patches/patch_types.go +++ b/pkg/patches/patch_types.go @@ -6,13 +6,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/genericsyncconfig" "github.com/pkg/errors" yaml "gopkg.in/yaml.v3" k8syaml "sigs.k8s.io/yaml" ) -func CopyFromObject(obj1, obj2 *yaml.Node, patch *config.Patch) error { +func CopyFromObject(obj1, obj2 *yaml.Node, patch *genericsyncconfig.Patch) error { if obj2 == nil { return nil } @@ -68,7 +68,7 @@ func CopyFromObject(obj1, obj2 *yaml.Node, patch *config.Patch) error { return nil } -func Remove(obj1 *yaml.Node, patch *config.Patch) error { +func Remove(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -95,7 +95,7 @@ func Remove(obj1 *yaml.Node, patch *config.Patch) error { return nil } -func Add(obj1 *yaml.Node, patch *config.Patch) error { +func Add(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -134,7 +134,7 @@ func Add(obj1 *yaml.Node, patch *config.Patch) error { return nil } -func Replace(obj1 *yaml.Node, patch *config.Patch) error { +func Replace(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -159,7 +159,7 @@ func Replace(obj1 *yaml.Node, patch *config.Patch) error { return nil } -func RewriteName(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func RewriteName(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -190,7 +190,7 @@ func RewriteName(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) er return nil } -func ProcessRewrite(obj *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func ProcessRewrite(obj *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { var namespace string var err error @@ -240,7 +240,7 @@ func ProcessRewrite(obj *yaml.Node, patch *config.Patch, resolver NameResolver) return nil } -func ValidateAndTranslateName(obj *yaml.Node, match *yaml.Node, patch *config.Patch, resolver NameResolver, namespace string) error { +func ValidateAndTranslateName(obj *yaml.Node, match *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver, namespace string) error { validated, err := ValidateAllConditions(obj, match, patch.Conditions) if err != nil { return errors.Wrap(err, "validate conditions") @@ -270,7 +270,7 @@ func ValidateAndTranslateName(obj *yaml.Node, match *yaml.Node, patch *config.Pa return nil } -func ValidateAndTranslateNamespace(obj *yaml.Node, match *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func ValidateAndTranslateNamespace(obj *yaml.Node, match *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { validated, err := ValidateAllConditions(obj, match, patch.Conditions) if err != nil { return errors.Wrap(err, "validate conditions") @@ -293,7 +293,7 @@ func ValidateAndTranslateNamespace(obj *yaml.Node, match *yaml.Node, patch *conf return nil } -func GetNamespace(obj *yaml.Node, patch *config.Patch) (string, error) { +func GetNamespace(obj *yaml.Node, patch *genericsyncconfig.Patch) (string, error) { var namespace string matches, err := FindMatches(obj, patch.NamespacePath) @@ -312,7 +312,7 @@ func GetNamespace(obj *yaml.Node, patch *config.Patch) (string, error) { return namespace, nil } -func RewriteLabelKey(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func RewriteLabelKey(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -350,7 +350,7 @@ func RewriteLabelKey(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver return nil } -func RewriteLabelSelector(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func RewriteLabelSelector(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -396,7 +396,7 @@ func RewriteLabelSelector(obj1 *yaml.Node, patch *config.Patch, resolver NameRes return nil } -func RewriteLabelExpressionsSelector(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { +func RewriteLabelExpressionsSelector(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index b47cd720f..ce23bd7be 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" pluginv1 "github.com/loft-sh/vcluster/pkg/plugin/v1" pluginv2 "github.com/loft-sh/vcluster/pkg/plugin/v2" @@ -34,14 +34,14 @@ func (m *manager) Start( virtualKubeConfig *rest.Config, physicalKubeConfig *rest.Config, syncerConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + vConfig *config.VirtualClusterConfig, ) error { - err := m.legacyManager.Start(ctx, currentNamespace, targetNamespace, virtualKubeConfig, physicalKubeConfig, syncerConfig, options) + err := m.legacyManager.Start(ctx, currentNamespace, targetNamespace, virtualKubeConfig, physicalKubeConfig, syncerConfig, vConfig.LegacyOptions) if err != nil { return fmt.Errorf("start legacy plugins: %w", err) } - err = m.pluginManager.Start(ctx, currentNamespace, physicalKubeConfig, syncerConfig, options) + err = m.pluginManager.Start(ctx, currentNamespace, physicalKubeConfig, syncerConfig, vConfig) if err != nil { return fmt.Errorf("start plugins: %w", err) } diff --git a/pkg/plugin/types/types.go b/pkg/plugin/types/types.go index a40f45a2c..8b1143d51 100644 --- a/pkg/plugin/types/types.go +++ b/pkg/plugin/types/types.go @@ -3,7 +3,7 @@ package types import ( "context" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -18,7 +18,7 @@ type Manager interface { virtualKubeConfig *rest.Config, physicalKubeConfig *rest.Config, syncerConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + config *config.VirtualClusterConfig, ) error // SetLeader sets the leader for the plugins diff --git a/pkg/plugin/v1/plugin.go b/pkg/plugin/v1/plugin.go index b3467be03..36f3904d4 100644 --- a/pkg/plugin/v1/plugin.go +++ b/pkg/plugin/v1/plugin.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "github.com/loft-sh/vcluster/pkg/util/loghelper" @@ -166,7 +166,7 @@ func (m *Manager) Start( virtualKubeConfig *rest.Config, physicalKubeConfig *rest.Config, syncerConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + options *config.LegacyVirtualClusterOptions, ) error { // set if we have plugins m.hasPlugins.Store(len(options.Plugins) > 0) @@ -239,7 +239,7 @@ func (m *Manager) Start( return m.waitForPlugins(ctx, options) } -func (m *Manager) waitForPlugins(ctx context.Context, options *options.VirtualClusterOptions) error { +func (m *Manager) waitForPlugins(ctx context.Context, options *config.LegacyVirtualClusterOptions) error { for _, plugin := range options.Plugins { klog.Infof("Waiting for plugin %s to register...", plugin) err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, time.Minute*10, true, func(context.Context) (done bool, err error) { diff --git a/pkg/plugin/v2/config.go b/pkg/plugin/v2/config.go index 0522b7bb7..b4415d1f7 100644 --- a/pkg/plugin/v2/config.go +++ b/pkg/plugin/v2/config.go @@ -8,8 +8,11 @@ type InitConfig struct { PhysicalClusterConfig []byte `json:"physicalClusterConfig,omitempty"` SyncerConfig []byte `json:"syncerConfig,omitempty"` CurrentNamespace string `json:"currentNamespace,omitempty"` - Options []byte `json:"options,omitempty"` - WorkingDir string `json:"workingDir,omitempty"` + + Config []byte `json:"config,omitempty"` + Options []byte `json:"options,omitempty"` + + WorkingDir string `json:"workingDir,omitempty"` } // InitConfigPro is used to signal the plugin if vCluster.Pro is enabled and what features are allowed diff --git a/pkg/plugin/v2/plugin.go b/pkg/plugin/v2/plugin.go index 2003624dd..160a0cb90 100644 --- a/pkg/plugin/v2/plugin.go +++ b/pkg/plugin/v2/plugin.go @@ -13,7 +13,7 @@ import ( "github.com/ghodss/yaml" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-plugin" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" "github.com/loft-sh/vcluster/pkg/plugin/v2/pluginv2" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" @@ -70,7 +70,7 @@ func (m *Manager) Start( currentNamespace string, physicalKubeConfig *rest.Config, syncerConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + options *config.VirtualClusterConfig, ) error { // try to search for plugins plugins, err := m.findPlugins(ctx) @@ -240,10 +240,16 @@ func (m *Manager) buildInitRequest( currentNamespace string, physicalKubeConfig *rest.Config, syncerConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + vConfig *config.VirtualClusterConfig, ) (*pluginv2.Initialize_Request, error) { - // Context options - encodedOptions, err := json.Marshal(options) + // encode config + encodedConfig, err := json.Marshal(vConfig) + if err != nil { + return nil, fmt.Errorf("encode config: %w", err) + } + + // We need this for downward compatibility + encodedLegacyOptions, err := json.Marshal(vConfig.LegacyOptions) if err != nil { return nil, fmt.Errorf("marshal options: %w", err) } @@ -277,7 +283,8 @@ func (m *Manager) buildInitRequest( PhysicalClusterConfig: phyisicalConfigBytes, SyncerConfig: syncerConfigBytes, CurrentNamespace: currentNamespace, - Options: encodedOptions, + Config: encodedConfig, + Options: encodedLegacyOptions, WorkingDir: workingDir, }) if err != nil { diff --git a/pkg/plugin/v2/types.go b/pkg/plugin/v2/types.go index 527d8bed0..6fe574e4f 100644 --- a/pkg/plugin/v2/types.go +++ b/pkg/plugin/v2/types.go @@ -3,7 +3,7 @@ package v2 import ( "context" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" @@ -19,7 +19,7 @@ type Plugin interface { virtualKubeConfig *rest.Config, physicalKubeConfig *rest.Config, syncerConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + vConfig *config.VirtualClusterConfig, ) error // SetLeader signals the plugin that the syncer acquired leadership and starts executing controllers diff --git a/pkg/pro/flags.go b/pkg/pro/flags.go deleted file mode 100644 index 28af7681a..000000000 --- a/pkg/pro/flags.go +++ /dev/null @@ -1,26 +0,0 @@ -package pro - -import ( - "github.com/loft-sh/vcluster/pkg/options" - "github.com/spf13/pflag" -) - -func AddProFlags(flags *pflag.FlagSet, options *options.VirtualClusterOptions) { - flags.StringVar(&options.ProOptions.ProLicenseSecret, "pro-license-secret", "", "If set, vCluster.Pro will try to find this secret to retrieve the vCluster.Pro license.") - - flags.StringVar(&options.ProOptions.RemoteKubeConfig, "remote-kube-config", "", "If set, will use the remote kube-config instead of the local in-cluster one. Expects a kube config to a headless vcluster installation") - flags.StringVar(&options.ProOptions.RemoteNamespace, "remote-namespace", "", "If set, will use this as the remote namespace") - flags.StringVar(&options.ProOptions.RemoteServiceName, "remote-service-name", "", "If set, will use this as the remote service name") - - flags.BoolVar(&options.ProOptions.IntegratedCoredns, "integrated-coredns", false, "If enabled vcluster will spin an in memory coreDNS inside the syncer container") - flags.BoolVar(&options.ProOptions.UseCoreDNSPlugin, "use-coredns-plugin", false, "If enabled, the vcluster plugin for coredns will be used") - flags.BoolVar(&options.ProOptions.NoopSyncer, "noop-syncer", false, "If enabled will setup a noop Syncer that filters and proxies requests to a specified remote cluster") - flags.BoolVar(&options.ProOptions.SyncKubernetesService, "sync-k8s-service", false, "If enabled will sync the kubernetes service endpoints in the remote cluster with the load balancer ip of this cluster") - - flags.BoolVar(&options.ProOptions.EtcdEmbedded, "etcd-embedded", false, "If true, will start an embedded etcd within vCluster") - flags.StringVar(&options.ProOptions.MigrateFrom, "migrate-from", "", "The url (including protocol) of the original database") - flags.IntVar(&options.ProOptions.EtcdReplicas, "etcd-replicas", 0, "The amount of replicas the etcd has") - - flags.StringArrayVar(&options.ProOptions.EnforceValidatingHooks, "enforce-validating-hook", nil, "A validating hook configuration in yaml format encoded with base64. Can be used multiple times") - flags.StringArrayVar(&options.ProOptions.EnforceMutatingHooks, "enforce-mutating-hook", nil, "A mutating hook configuration in yaml format encoded with base64. Can be used multiple times") -} diff --git a/pkg/pro/generic.go b/pkg/pro/generic.go index b5973cb31..af3f31d90 100644 --- a/pkg/pro/generic.go +++ b/pkg/pro/generic.go @@ -1,20 +1,20 @@ package pro import ( - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" "github.com/loft-sh/vcluster/pkg/util/pluginhookclient" "sigs.k8s.io/controller-runtime/pkg/client" ) -var InitProControllerContext = func(_ *options.ControllerContext) error { +var InitProControllerContext = func(_ *config.ControllerContext) error { return nil } -var NewPhysicalClient = func(_ *options.VirtualClusterOptions) client.NewClientFunc { +var NewPhysicalClient = func(_ *config.VirtualClusterConfig) client.NewClientFunc { return pluginhookclient.NewPhysicalPluginClientFactory(blockingcacheclient.NewCacheClient) } -var NewVirtualClient = func(_ *options.VirtualClusterOptions) client.NewClientFunc { +var NewVirtualClient = func(_ *config.VirtualClusterConfig) client.NewClientFunc { return pluginhookclient.NewVirtualPluginClientFactory(blockingcacheclient.NewCacheClient) } diff --git a/pkg/pro/integrated_coredns.go b/pkg/pro/integrated_coredns.go index 1e5590cb4..568b10053 100644 --- a/pkg/pro/integrated_coredns.go +++ b/pkg/pro/integrated_coredns.go @@ -1,14 +1,14 @@ package pro import ( - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/specialservices" ) -var StartIntegratedCoreDNS = func(_ *options.ControllerContext) error { +var StartIntegratedCoreDNS = func(_ *config.ControllerContext) error { return NewFeatureError("integrated core dns") } -var InitDNSServiceSyncing = func(_ *options.VirtualClusterOptions) specialservices.Interface { +var InitDNSServiceSyncing = func(_ *config.VirtualClusterConfig) specialservices.Interface { return specialservices.NewDefaultServiceSyncer() } diff --git a/pkg/pro/license.go b/pkg/pro/license.go index 86f7a3de3..67a9f0be1 100644 --- a/pkg/pro/license.go +++ b/pkg/pro/license.go @@ -7,7 +7,7 @@ import ( ) // LicenseInit is used to initialize the license reader -var LicenseInit = func(_ context.Context, _ *rest.Config, _, _ string) error { +var LicenseInit = func(_ context.Context, _ *rest.Config, _, _, _, _ string) error { return nil } diff --git a/pkg/pro/noop.go b/pkg/pro/noop.go index 6b4f93190..0c77675a4 100644 --- a/pkg/pro/noop.go +++ b/pkg/pro/noop.go @@ -1,11 +1,11 @@ package pro import ( - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) -var SyncNoopSyncerEndpoints = func(_ *options.ControllerContext, _ types.NamespacedName, _ client.Client, _ types.NamespacedName, _ string) error { +var SyncNoopSyncerEndpoints = func(_ *config.ControllerContext, _ types.NamespacedName, _ client.Client, _ types.NamespacedName, _ string) error { return NewFeatureError("noop syncer") } diff --git a/pkg/pro/remote.go b/pkg/pro/remote.go index 438a993b6..f00514d4f 100644 --- a/pkg/pro/remote.go +++ b/pkg/pro/remote.go @@ -4,7 +4,7 @@ import ( "context" "github.com/loft-sh/admin-apis/pkg/licenseapi" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/clienthelper" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -13,7 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var GetRemoteClient = func(options *options.VirtualClusterOptions) (*rest.Config, string, string, *rest.Config, string, string, error) { +var GetRemoteClient = func(vConfig *config.VirtualClusterConfig) (*rest.Config, string, string, *rest.Config, string, string, error) { inClusterConfig := ctrl.GetConfigOrDie() inClusterConfig.QPS = 40 inClusterConfig.Burst = 80 @@ -26,9 +26,8 @@ var GetRemoteClient = func(options *options.VirtualClusterOptions) (*rest.Config } // check if remote cluster - proOptions := options.ProOptions - if proOptions.RemoteKubeConfig == "" { - return inClusterConfig, currentNamespace, options.ServiceName, inClusterConfig, currentNamespace, options.ServiceName, nil + if vConfig.Experimental.IsolatedControlPlane.Enabled { + return inClusterConfig, currentNamespace, vConfig.ServiceName, inClusterConfig, currentNamespace, vConfig.ServiceName, nil } return nil, "", "", nil, "", "", NewFeatureError(string(licenseapi.VirtualClusterProDistroIsolatedControlPlane)) @@ -38,7 +37,7 @@ var AddRemoteNodePortSANs = func(_ context.Context, _, _ string, _ kubernetes.In return nil } -var ExchangeControlPlaneClient = func(controllerCtx *options.ControllerContext, _ string, _ *rest.Config) (client.Client, error) { +var ExchangeControlPlaneClient = func(controllerCtx *config.ControllerContext, _ string, _ *rest.Config) (client.Client, error) { return controllerCtx.CurrentNamespaceClient, nil } diff --git a/pkg/pro/validation.go b/pkg/pro/validation.go deleted file mode 100644 index 43324fb41..000000000 --- a/pkg/pro/validation.go +++ /dev/null @@ -1,97 +0,0 @@ -package pro - -import ( - "crypto/x509" - "encoding/base64" - "errors" - "fmt" - "net/url" - - "github.com/ghodss/yaml" - "github.com/loft-sh/vcluster/pkg/options" - - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" -) - -func ValidateProOptions(options *options.VirtualClusterOptions) error { - _, _, err := ParseExtraHooks(options.ProOptions.EnforceValidatingHooks, options.ProOptions.EnforceMutatingHooks) - return err -} - -func ParseExtraHooks(valHooks, mutHooks []string) ([]admissionregistrationv1.ValidatingWebhookConfiguration, []admissionregistrationv1.MutatingWebhookConfiguration, error) { - decodedVal := make([]string, 0, len(valHooks)) - for _, v := range valHooks { - bytes, err := base64.StdEncoding.DecodeString(v) - if err != nil { - return nil, nil, err - } - decodedVal = append(decodedVal, string(bytes)) - } - decodedMut := make([]string, 0, len(mutHooks)) - for _, v := range mutHooks { - bytes, err := base64.StdEncoding.DecodeString(v) - if err != nil { - return nil, nil, err - } - decodedMut = append(decodedMut, string(bytes)) - } - - validateConfs := make([]admissionregistrationv1.ValidatingWebhookConfiguration, 0, len(valHooks)) - mutateConfs := make([]admissionregistrationv1.MutatingWebhookConfiguration, 0, len(mutHooks)) - for _, v := range decodedVal { - var valHook admissionregistrationv1.ValidatingWebhookConfiguration - err := yaml.Unmarshal([]byte(v), &valHook) - if err != nil { - return nil, nil, err - } - for _, v := range valHook.Webhooks { - err := validateWebhookClientCfg(v.ClientConfig) - if err != nil { - return nil, nil, fmt.Errorf("webhook client config was not valid for ValidatingWebhookConfiguration %s: %w", v.Name, err) - } - } - validateConfs = append(validateConfs, valHook) - } - for _, v := range decodedMut { - var mutHook admissionregistrationv1.MutatingWebhookConfiguration - err := yaml.Unmarshal([]byte(v), &mutHook) - if err != nil { - return nil, nil, err - } - for _, v := range mutHook.Webhooks { - err := validateWebhookClientCfg(v.ClientConfig) - if err != nil { - return nil, nil, fmt.Errorf("webhook client config was not valid for MutatingWebhookConfiguration %s: %w", v.Name, err) - } - } - mutateConfs = append(mutateConfs, mutHook) - } - - return validateConfs, mutateConfs, nil -} - -func validateWebhookClientCfg(clientCfg admissionregistrationv1.WebhookClientConfig) error { - if len(clientCfg.CABundle) != 0 { - ok := x509.NewCertPool().AppendCertsFromPEM(clientCfg.CABundle) - if !ok { - return errors.New("could not parse the CABundle") - } - } - - if clientCfg.Service == nil && clientCfg.URL == nil { - return errors.New("there is no service config") - } - - if clientCfg.Service != nil && (clientCfg.Service.Name == "" || clientCfg.Service.Namespace == "") { - return errors.New("namespace or name of the service is missing") - } - - if clientCfg.URL != nil { - _, err := url.Parse(*clientCfg.URL) - if err != nil { - return errors.New("the url was not valid") - } - } - - return nil -} diff --git a/pkg/setup/controller_context.go b/pkg/setup/controller_context.go index 34368859a..62a441363 100644 --- a/pkg/setup/controller_context.go +++ b/pkg/setup/controller_context.go @@ -6,6 +6,7 @@ import ( "os" "time" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/plugin" @@ -43,7 +44,7 @@ func NewControllerContext( currentNamespace string, inClusterConfig *rest.Config, scheme *runtime.Scheme, -) (*options.ControllerContext, error) { +) (*config.ControllerContext, error) { // validate options err := ValidateOptions(options) if err != nil { diff --git a/pkg/setup/proxy.go b/pkg/setup/proxy.go index 2cb729362..5a03b6d32 100644 --- a/pkg/setup/proxy.go +++ b/pkg/setup/proxy.go @@ -1,14 +1,14 @@ package setup import ( - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/pro" "github.com/loft-sh/vcluster/pkg/server" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) -func StartProxy(ctx *options.ControllerContext, controlPlaneNamespace, controlPlaneService string, controlPlaneClient kubernetes.Interface) error { +func StartProxy(ctx *config.ControllerContext, controlPlaneNamespace, controlPlaneService string, controlPlaneClient kubernetes.Interface) error { // add remote node port sans err := pro.AddRemoteNodePortSANs(ctx.Context, controlPlaneNamespace, controlPlaneService, controlPlaneClient) if err != nil { @@ -23,7 +23,7 @@ func StartProxy(ctx *options.ControllerContext, controlPlaneNamespace, controlPl // start the proxy server in secure mode go func() { - err = proxyServer.ServeOnListenerTLS(ctx.Options.BindAddress, ctx.Options.Port, ctx.StopChan) + err = proxyServer.ServeOnListenerTLS(ctx.Config.ControlPlane.Proxy.BindAddress, ctx.Config.ControlPlane.Proxy.Port, ctx.StopChan) if err != nil { klog.Fatalf("Error serving: %v", err) } diff --git a/pkg/telemetry/collect.go b/pkg/telemetry/collect.go index 16251a72a..572e77234 100644 --- a/pkg/telemetry/collect.go +++ b/pkg/telemetry/collect.go @@ -10,7 +10,7 @@ import ( "github.com/loft-sh/analytics-client/client" managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" "github.com/loft-sh/log" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/upgrade" "github.com/loft-sh/vcluster/pkg/util/cliconfig" "github.com/loft-sh/vcluster/pkg/util/clihelper" @@ -45,7 +45,7 @@ type EventCollector interface { // Flush makes sure all events are sent to the backend Flush() - Init(currentNamespaceConfig *rest.Config, currentNamespace string, options *options.VirtualClusterOptions) + Init(currentNamespaceConfig *rest.Config, currentNamespace string, vConfig *config.VirtualClusterConfig) SetVirtualClient(virtualClient *kubernetes.Clientset) } @@ -110,7 +110,7 @@ type DefaultCollector struct { // everything below will be set during runtime virtualClient *kubernetes.Clientset - options *options.VirtualClusterOptions + options *config.VirtualClusterConfig hostClient *kubernetes.Clientset hostNamespace string } @@ -123,7 +123,7 @@ func (d *DefaultCollector) startReportStatus(ctx context.Context) { }, time.Minute*5, ctx.Done()) } -func (d *DefaultCollector) Init(currentNamespaceConfig *rest.Config, currentNamespace string, options *options.VirtualClusterOptions) { +func (d *DefaultCollector) Init(currentNamespaceConfig *rest.Config, currentNamespace string, vConfig *config.VirtualClusterConfig) { hostClient, err := kubernetes.NewForConfig(currentNamespaceConfig) if err != nil { klog.V(1).ErrorS(err, "create host client") @@ -131,7 +131,7 @@ func (d *DefaultCollector) Init(currentNamespaceConfig *rest.Config, currentName d.hostClient = hostClient d.hostNamespace = currentNamespace - d.options = options + d.options = vConfig } func (d *DefaultCollector) SetVirtualClient(virtualClient *kubernetes.Clientset) { diff --git a/pkg/telemetry/helpers.go b/pkg/telemetry/helpers.go index 89c6293e7..78cc00446 100644 --- a/pkg/telemetry/helpers.go +++ b/pkg/telemetry/helpers.go @@ -9,8 +9,8 @@ import ( "github.com/denisbrodbeck/machineid" managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" "github.com/loft-sh/log" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/helm" - "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/util/cliconfig" "github.com/loft-sh/vcluster/pkg/util/translate" homedir "github.com/mitchellh/go-homedir" @@ -58,7 +58,7 @@ func getChartInfo(ctx context.Context, hostClient *kubernetes.Clientset, vCluste } // getVClusterID provides instance ID based on the UID of the service -func getVClusterID(ctx context.Context, hostClient *kubernetes.Clientset, vClusterNamespace string, options *options.VirtualClusterOptions) (string, error) { +func getVClusterID(ctx context.Context, hostClient *kubernetes.Clientset, vClusterNamespace string, options *config.VirtualClusterConfig) (string, error) { if hostClient == nil || options == nil { return "", fmt.Errorf("kubernetes client or options are nil") } @@ -72,7 +72,7 @@ func getVClusterID(ctx context.Context, hostClient *kubernetes.Clientset, vClust } // returns a Kubernetes resource that can be used to uniquely identify this syncer instance - PVC or Service -func getUniqueSyncerObject(ctx context.Context, c *kubernetes.Clientset, vClusterNamespace string, options *options.VirtualClusterOptions) (client.Object, error) { +func getUniqueSyncerObject(ctx context.Context, c *kubernetes.Clientset, vClusterNamespace string, options *config.VirtualClusterConfig) (client.Object, error) { // If vCluster PVC doesn't exist we try to get UID from the vCluster Service if options.ServiceName == "" { return nil, fmt.Errorf("getUniqueSyncerObject failed - options.ServiceName is empty") diff --git a/pkg/telemetry/noop.go b/pkg/telemetry/noop.go index 84382b5be..eab903d1f 100644 --- a/pkg/telemetry/noop.go +++ b/pkg/telemetry/noop.go @@ -4,7 +4,7 @@ import ( "context" managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -15,7 +15,7 @@ func (n *noopCollector) RecordStart(_ context.Context) {} func (n *noopCollector) RecordError(_ context.Context, _ ErrorSeverityType, _ error) {} -func (n *noopCollector) Init(_ *rest.Config, _ string, _ *options.VirtualClusterOptions) {} +func (n *noopCollector) Init(_ *rest.Config, _ string, _ *config.VirtualClusterConfig) {} func (n *noopCollector) Flush() {} diff --git a/pkg/util/context/converter.go b/pkg/util/context/converter.go index 9f6922361..61ee5e0e7 100644 --- a/pkg/util/context/converter.go +++ b/pkg/util/context/converter.go @@ -1,16 +1,15 @@ package util import ( + "github.com/loft-sh/vcluster/pkg/config" synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" - "github.com/loft-sh/vcluster/pkg/options" ) -func ToRegisterContext(ctx *options.ControllerContext) *synccontext.RegisterContext { +func ToRegisterContext(ctx *config.ControllerContext) *synccontext.RegisterContext { return &synccontext.RegisterContext{ Context: ctx.Context, - Options: ctx.Options, - Controllers: ctx.Controllers, + Config: ctx.Config, CurrentNamespace: ctx.CurrentNamespace, CurrentNamespaceClient: ctx.CurrentNamespaceClient, From d4f4cc5bf249616b3d6c00306bb758bcf7545db3 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 11 Mar 2024 14:55:35 +0100 Subject: [PATCH 2/5] refactor: use config everywhere --- chart/values.yaml | 25 +- charts/eks/.helmignore | 25 - charts/eks/Chart.yaml | 26 - charts/eks/README.md | 64 --- charts/eks/templates/NOTES.txt | 10 - charts/eks/templates/_coredns.tpl | 52 -- charts/eks/templates/_helpers.tpl | 196 ------- charts/eks/templates/_kind.tpl | 64 --- charts/eks/templates/_plugin.tpl | 188 ------ charts/eks/templates/coredns.yaml | 215 ------- charts/eks/templates/etcd-service.yaml | 31 - .../templates/etcd-statefulset-service.yaml | 32 -- charts/eks/templates/etcd-statefulset.yaml | 170 ------ charts/eks/templates/ingress.yaml | 34 -- charts/eks/templates/init-configmap.yaml | 55 -- charts/eks/templates/integrated-coredns.yaml | 64 --- charts/eks/templates/limitrange.yaml | 22 - charts/eks/templates/networkpolicy.yaml | 78 --- charts/eks/templates/rbac/clusterrole.yaml | 103 ---- .../templates/rbac/clusterrolebinding.yaml | 27 - charts/eks/templates/rbac/role.yaml | 104 ---- charts/eks/templates/rbac/rolebinding.yaml | 41 -- charts/eks/templates/resourcequota.yaml | 28 - charts/eks/templates/service-monitor.yaml | 31 - charts/eks/templates/service.yaml | 93 --- charts/eks/templates/serviceaccount.yaml | 20 - charts/eks/templates/statefulset-service.yaml | 38 -- charts/eks/templates/syncer.yaml | 447 --------------- .../eks/templates/workloadserviceaccount.yaml | 19 - charts/eks/tests/README.md | 9 - charts/eks/tests/clusterrole_test.yaml | 41 -- charts/eks/tests/etcd_test.yaml | 26 - charts/eks/tests/legacy-plugins_test.yaml | 77 --- charts/eks/tests/plugins_test.yaml | 100 ---- charts/eks/tests/role_test.yaml | 32 -- charts/eks/tests/syncer_test.yaml | 52 -- charts/eks/values.yaml | 505 ---------------- charts/k0s/.helmignore | 24 - charts/k0s/Chart.yaml | 26 - charts/k0s/README.md | 64 --- charts/k0s/templates/NOTES.txt | 10 - charts/k0s/templates/_coredns.tpl | 52 -- charts/k0s/templates/_helpers.tpl | 196 ------- charts/k0s/templates/_migrate.tpl | 6 - charts/k0s/templates/_plugin.tpl | 188 ------ charts/k0s/templates/_storage.tpl | 20 - charts/k0s/templates/coredns.yaml | 231 -------- charts/k0s/templates/ingress.yaml | 34 -- charts/k0s/templates/init-configmap.yaml | 55 -- charts/k0s/templates/integrated-coredns.yaml | 64 --- charts/k0s/templates/limitrange.yaml | 22 - charts/k0s/templates/networkpolicy.yaml | 70 --- charts/k0s/templates/rbac/clusterrole.yaml | 103 ---- .../templates/rbac/clusterrolebinding.yaml | 27 - charts/k0s/templates/rbac/role.yaml | 104 ---- charts/k0s/templates/rbac/rolebinding.yaml | 41 -- charts/k0s/templates/resourcequota.yaml | 28 - charts/k0s/templates/secret.yaml | 65 --- charts/k0s/templates/service-monitor.yaml | 32 -- charts/k0s/templates/service.yaml | 93 --- charts/k0s/templates/serviceaccount.yaml | 20 - charts/k0s/templates/statefulset-service.yaml | 38 -- charts/k0s/templates/syncer.yaml | 348 ----------- .../k0s/templates/workloadserviceaccount.yaml | 19 - charts/k0s/tests/README.md | 9 - charts/k0s/tests/clusterrole_test.yaml | 41 -- charts/k0s/tests/legacy-plugins_test.yaml | 77 --- charts/k0s/tests/plugins_test.yaml | 100 ---- charts/k0s/tests/role_test.yaml | 32 -- charts/k0s/tests/syncer_test.yaml | 65 --- charts/k0s/values.yaml | 514 ----------------- charts/k3s/.helmignore | 24 - charts/k3s/Chart.yaml | 26 - charts/k3s/README.md | 64 --- charts/k3s/templates/NOTES.txt | 10 - charts/k3s/templates/_coredns.tpl | 52 -- charts/k3s/templates/_helpers.tpl | 196 ------- charts/k3s/templates/_migrate.tpl | 6 - charts/k3s/templates/_plugin.tpl | 188 ------ charts/k3s/templates/_storage.tpl | 20 - charts/k3s/templates/coredns.yaml | 231 -------- charts/k3s/templates/ingress.yaml | 34 -- charts/k3s/templates/init-configmap.yaml | 55 -- charts/k3s/templates/integrated-coredns.yaml | 64 --- charts/k3s/templates/limitrange.yaml | 22 - charts/k3s/templates/networkpolicy.yaml | 74 --- charts/k3s/templates/rbac/clusterrole.yaml | 103 ---- .../templates/rbac/clusterrolebinding.yaml | 27 - charts/k3s/templates/rbac/role.yaml | 104 ---- charts/k3s/templates/rbac/rolebinding.yaml | 41 -- charts/k3s/templates/resourcequota.yaml | 28 - charts/k3s/templates/service-monitor.yaml | 31 - charts/k3s/templates/service.yaml | 93 --- charts/k3s/templates/serviceaccount.yaml | 20 - charts/k3s/templates/statefulset-service.yaml | 38 -- charts/k3s/templates/syncer.yaml | 431 -------------- charts/k3s/templates/token-secret.yaml | 10 - .../k3s/templates/workloadserviceaccount.yaml | 19 - charts/k3s/tests/README.md | 9 - charts/k3s/tests/clusterrole_test.yaml | 41 -- charts/k3s/tests/legacy-plugins_test.yaml | 77 --- charts/k3s/tests/plugins_test.yaml | 100 ---- charts/k3s/tests/role_test.yaml | 32 -- charts/k3s/tests/syncer_test.yaml | 65 --- charts/k3s/values.yaml | 539 ------------------ charts/k8s/.helmignore | 24 - charts/k8s/Chart.yaml | 26 - charts/k8s/README.md | 64 --- charts/k8s/templates/NOTES.txt | 10 - charts/k8s/templates/_coredns.tpl | 52 -- charts/k8s/templates/_helpers.tpl | 196 ------- charts/k8s/templates/_kind.tpl | 62 -- charts/k8s/templates/_plugin.tpl | 188 ------ charts/k8s/templates/coredns.yaml | 231 -------- charts/k8s/templates/etcd-service.yaml | 31 - .../templates/etcd-statefulset-service.yaml | 32 -- charts/k8s/templates/etcd-statefulset.yaml | 205 ------- charts/k8s/templates/ingress.yaml | 34 -- charts/k8s/templates/init-configmap.yaml | 55 -- charts/k8s/templates/integrated-coredns.yaml | 64 --- charts/k8s/templates/limitrange.yaml | 22 - charts/k8s/templates/networkpolicy.yaml | 78 --- charts/k8s/templates/rbac/clusterrole.yaml | 103 ---- .../templates/rbac/clusterrolebinding.yaml | 27 - charts/k8s/templates/rbac/role.yaml | 104 ---- charts/k8s/templates/rbac/rolebinding.yaml | 41 -- charts/k8s/templates/resourcequota.yaml | 28 - charts/k8s/templates/service-monitor.yaml | 32 -- charts/k8s/templates/service.yaml | 93 --- charts/k8s/templates/serviceaccount.yaml | 20 - charts/k8s/templates/statefulset-service.yaml | 38 -- charts/k8s/templates/syncer.yaml | 493 ---------------- .../k8s/templates/workloadserviceaccount.yaml | 19 - charts/k8s/tests/README.md | 9 - charts/k8s/tests/clusterrole_test.yaml | 41 -- charts/k8s/tests/etcd_test.yaml | 26 - charts/k8s/tests/legacy-plugins_test.yaml | 77 --- charts/k8s/tests/plugins_test.yaml | 100 ---- charts/k8s/tests/role_test.yaml | 32 -- charts/k8s/tests/syncer_test.yaml | 52 -- charts/k8s/values.yaml | 524 ----------------- cmd/vcluster/cmd/start.go | 15 +- config/config.go | 99 ++-- pkg/config/config.go | 134 +++-- pkg/config/disable_missing_apis.go | 49 ++ pkg/config/flags.go | 7 +- pkg/config/options.go | 24 +- pkg/config/parse.go | 27 +- pkg/config/pro_options.go | 16 +- pkg/config/validation.go | 87 ++- pkg/constants/controllers.go | 50 -- pkg/constants/distro.go | 21 - pkg/controllers/generic/patcher.go | 6 +- pkg/controllers/register.go | 167 +++--- .../resources/configmaps/syncer.go | 2 +- .../resources/csistoragecapacities/syncer.go | 4 +- .../resources/namespaces/syncer.go | 33 +- .../resources/nodes/fake_syncer.go | 2 +- pkg/controllers/resources/nodes/register.go | 4 +- pkg/controllers/resources/nodes/syncer.go | 24 +- .../persistentvolumeclaims/syncer.go | 6 +- .../resources/persistentvolumes/register.go | 2 +- pkg/controllers/resources/pods/syncer.go | 21 +- .../resources/pods/translate/hostpath.go | 4 +- .../resources/pods/translate/translator.go | 57 +- pkg/controllers/resources/secrets/syncer.go | 6 +- .../resources/secrets/syncer_test.go | 4 +- pkg/controllers/resources/services/syncer.go | 2 +- pkg/controllers/syncer/testing/context.go | 51 +- .../syncer/translator/cluster_translator.go | 2 +- .../translator/namespaced_translator.go | 2 +- pkg/k0s/k0s.go | 130 ++++- pkg/k3s/k3s.go | 66 ++- pkg/k8s/k8s.go | 163 ++++-- pkg/metricsapiservice/register.go | 12 +- pkg/patches/conditions.go | 6 +- pkg/patches/patch.go | 26 +- pkg/patches/patch_types.go | 26 +- pkg/plugin/plugin.go | 7 +- pkg/plugin/v2/plugin.go | 8 +- pkg/server/cert/syncer.go | 15 +- pkg/server/server.go | 20 +- pkg/setup/controller_context.go | 107 +--- pkg/setup/controllers.go | 94 ++- pkg/setup/enable_controllers.go | 140 ----- pkg/setup/enable_controllers_test.go | 218 ------- pkg/setup/initialize.go | 102 ++-- pkg/setup/proxy.go | 2 +- pkg/util/servicecidr/servicecidr.go | 51 -- 189 files changed, 1004 insertions(+), 13186 deletions(-) delete mode 100644 charts/eks/.helmignore delete mode 100644 charts/eks/Chart.yaml delete mode 100644 charts/eks/README.md delete mode 100644 charts/eks/templates/NOTES.txt delete mode 100644 charts/eks/templates/_coredns.tpl delete mode 100644 charts/eks/templates/_helpers.tpl delete mode 100644 charts/eks/templates/_kind.tpl delete mode 100644 charts/eks/templates/_plugin.tpl delete mode 100644 charts/eks/templates/coredns.yaml delete mode 100644 charts/eks/templates/etcd-service.yaml delete mode 100644 charts/eks/templates/etcd-statefulset-service.yaml delete mode 100644 charts/eks/templates/etcd-statefulset.yaml delete mode 100644 charts/eks/templates/ingress.yaml delete mode 100644 charts/eks/templates/init-configmap.yaml delete mode 100644 charts/eks/templates/integrated-coredns.yaml delete mode 100644 charts/eks/templates/limitrange.yaml delete mode 100644 charts/eks/templates/networkpolicy.yaml delete mode 100644 charts/eks/templates/rbac/clusterrole.yaml delete mode 100644 charts/eks/templates/rbac/clusterrolebinding.yaml delete mode 100644 charts/eks/templates/rbac/role.yaml delete mode 100644 charts/eks/templates/rbac/rolebinding.yaml delete mode 100644 charts/eks/templates/resourcequota.yaml delete mode 100644 charts/eks/templates/service-monitor.yaml delete mode 100644 charts/eks/templates/service.yaml delete mode 100644 charts/eks/templates/serviceaccount.yaml delete mode 100644 charts/eks/templates/statefulset-service.yaml delete mode 100644 charts/eks/templates/syncer.yaml delete mode 100644 charts/eks/templates/workloadserviceaccount.yaml delete mode 100644 charts/eks/tests/README.md delete mode 100644 charts/eks/tests/clusterrole_test.yaml delete mode 100644 charts/eks/tests/etcd_test.yaml delete mode 100644 charts/eks/tests/legacy-plugins_test.yaml delete mode 100644 charts/eks/tests/plugins_test.yaml delete mode 100644 charts/eks/tests/role_test.yaml delete mode 100644 charts/eks/tests/syncer_test.yaml delete mode 100644 charts/eks/values.yaml delete mode 100644 charts/k0s/.helmignore delete mode 100644 charts/k0s/Chart.yaml delete mode 100644 charts/k0s/README.md delete mode 100644 charts/k0s/templates/NOTES.txt delete mode 100644 charts/k0s/templates/_coredns.tpl delete mode 100644 charts/k0s/templates/_helpers.tpl delete mode 100644 charts/k0s/templates/_migrate.tpl delete mode 100644 charts/k0s/templates/_plugin.tpl delete mode 100644 charts/k0s/templates/_storage.tpl delete mode 100644 charts/k0s/templates/coredns.yaml delete mode 100644 charts/k0s/templates/ingress.yaml delete mode 100644 charts/k0s/templates/init-configmap.yaml delete mode 100644 charts/k0s/templates/integrated-coredns.yaml delete mode 100644 charts/k0s/templates/limitrange.yaml delete mode 100644 charts/k0s/templates/networkpolicy.yaml delete mode 100644 charts/k0s/templates/rbac/clusterrole.yaml delete mode 100644 charts/k0s/templates/rbac/clusterrolebinding.yaml delete mode 100644 charts/k0s/templates/rbac/role.yaml delete mode 100644 charts/k0s/templates/rbac/rolebinding.yaml delete mode 100644 charts/k0s/templates/resourcequota.yaml delete mode 100644 charts/k0s/templates/secret.yaml delete mode 100644 charts/k0s/templates/service-monitor.yaml delete mode 100644 charts/k0s/templates/service.yaml delete mode 100644 charts/k0s/templates/serviceaccount.yaml delete mode 100644 charts/k0s/templates/statefulset-service.yaml delete mode 100644 charts/k0s/templates/syncer.yaml delete mode 100644 charts/k0s/templates/workloadserviceaccount.yaml delete mode 100644 charts/k0s/tests/README.md delete mode 100644 charts/k0s/tests/clusterrole_test.yaml delete mode 100644 charts/k0s/tests/legacy-plugins_test.yaml delete mode 100644 charts/k0s/tests/plugins_test.yaml delete mode 100644 charts/k0s/tests/role_test.yaml delete mode 100644 charts/k0s/tests/syncer_test.yaml delete mode 100644 charts/k0s/values.yaml delete mode 100644 charts/k3s/.helmignore delete mode 100644 charts/k3s/Chart.yaml delete mode 100644 charts/k3s/README.md delete mode 100644 charts/k3s/templates/NOTES.txt delete mode 100644 charts/k3s/templates/_coredns.tpl delete mode 100644 charts/k3s/templates/_helpers.tpl delete mode 100644 charts/k3s/templates/_migrate.tpl delete mode 100644 charts/k3s/templates/_plugin.tpl delete mode 100644 charts/k3s/templates/_storage.tpl delete mode 100644 charts/k3s/templates/coredns.yaml delete mode 100644 charts/k3s/templates/ingress.yaml delete mode 100644 charts/k3s/templates/init-configmap.yaml delete mode 100644 charts/k3s/templates/integrated-coredns.yaml delete mode 100644 charts/k3s/templates/limitrange.yaml delete mode 100644 charts/k3s/templates/networkpolicy.yaml delete mode 100644 charts/k3s/templates/rbac/clusterrole.yaml delete mode 100644 charts/k3s/templates/rbac/clusterrolebinding.yaml delete mode 100644 charts/k3s/templates/rbac/role.yaml delete mode 100644 charts/k3s/templates/rbac/rolebinding.yaml delete mode 100644 charts/k3s/templates/resourcequota.yaml delete mode 100644 charts/k3s/templates/service-monitor.yaml delete mode 100644 charts/k3s/templates/service.yaml delete mode 100644 charts/k3s/templates/serviceaccount.yaml delete mode 100644 charts/k3s/templates/statefulset-service.yaml delete mode 100644 charts/k3s/templates/syncer.yaml delete mode 100644 charts/k3s/templates/token-secret.yaml delete mode 100644 charts/k3s/templates/workloadserviceaccount.yaml delete mode 100644 charts/k3s/tests/README.md delete mode 100644 charts/k3s/tests/clusterrole_test.yaml delete mode 100644 charts/k3s/tests/legacy-plugins_test.yaml delete mode 100644 charts/k3s/tests/plugins_test.yaml delete mode 100644 charts/k3s/tests/role_test.yaml delete mode 100644 charts/k3s/tests/syncer_test.yaml delete mode 100644 charts/k3s/values.yaml delete mode 100644 charts/k8s/.helmignore delete mode 100644 charts/k8s/Chart.yaml delete mode 100644 charts/k8s/README.md delete mode 100644 charts/k8s/templates/NOTES.txt delete mode 100644 charts/k8s/templates/_coredns.tpl delete mode 100644 charts/k8s/templates/_helpers.tpl delete mode 100644 charts/k8s/templates/_kind.tpl delete mode 100644 charts/k8s/templates/_plugin.tpl delete mode 100644 charts/k8s/templates/coredns.yaml delete mode 100644 charts/k8s/templates/etcd-service.yaml delete mode 100644 charts/k8s/templates/etcd-statefulset-service.yaml delete mode 100644 charts/k8s/templates/etcd-statefulset.yaml delete mode 100644 charts/k8s/templates/ingress.yaml delete mode 100644 charts/k8s/templates/init-configmap.yaml delete mode 100644 charts/k8s/templates/integrated-coredns.yaml delete mode 100644 charts/k8s/templates/limitrange.yaml delete mode 100644 charts/k8s/templates/networkpolicy.yaml delete mode 100644 charts/k8s/templates/rbac/clusterrole.yaml delete mode 100644 charts/k8s/templates/rbac/clusterrolebinding.yaml delete mode 100644 charts/k8s/templates/rbac/role.yaml delete mode 100644 charts/k8s/templates/rbac/rolebinding.yaml delete mode 100644 charts/k8s/templates/resourcequota.yaml delete mode 100644 charts/k8s/templates/service-monitor.yaml delete mode 100644 charts/k8s/templates/service.yaml delete mode 100644 charts/k8s/templates/serviceaccount.yaml delete mode 100644 charts/k8s/templates/statefulset-service.yaml delete mode 100644 charts/k8s/templates/syncer.yaml delete mode 100644 charts/k8s/templates/workloadserviceaccount.yaml delete mode 100644 charts/k8s/tests/README.md delete mode 100644 charts/k8s/tests/clusterrole_test.yaml delete mode 100644 charts/k8s/tests/etcd_test.yaml delete mode 100644 charts/k8s/tests/legacy-plugins_test.yaml delete mode 100644 charts/k8s/tests/plugins_test.yaml delete mode 100644 charts/k8s/tests/role_test.yaml delete mode 100644 charts/k8s/tests/syncer_test.yaml delete mode 100644 charts/k8s/values.yaml create mode 100644 pkg/config/disable_missing_apis.go delete mode 100644 pkg/constants/controllers.go delete mode 100644 pkg/constants/distro.go delete mode 100644 pkg/setup/enable_controllers.go delete mode 100644 pkg/setup/enable_controllers_test.go diff --git a/chart/values.yaml b/chart/values.yaml index 7b3482158..5de8abd51 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -68,6 +68,8 @@ controlPlane: distro: k3s: enabled: false + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "rancher/k3s" @@ -83,6 +85,9 @@ controlPlane: k0s: enabled: false + config: "" + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "k0sproject/k0s" @@ -100,17 +105,23 @@ controlPlane: enabled: false apiServer: disabled: false + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "registry.k8s.io/kube-apiserver" tag: "v1.29.0" controllerManager: disabled: false + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "registry.k8s.io/kube-controller-manager" tag: "v1.29.0" scheduler: + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "registry.k8s.io/kube-scheduler" @@ -129,17 +140,23 @@ controlPlane: enabled: false apiServer: disabled: false + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "public.ecr.aws/eks-distro/kubernetes/kube-apiserver" tag: "v1.28.2-eks-1-28-6" controllerManager: disabled: false + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "public.ecr.aws/eks-distro/kubernetes/kube-controller-manager" tag: "v1.28.2-eks-1-28-6" scheduler: + command: [] + extraArgs: [] imagePullPolicy: "" image: repository: "public.ecr.aws/eks-distro/kubernetes/kube-scheduler" @@ -365,6 +382,9 @@ networking: advanced: clusterDomain: "cluster.local" fallbackHostCluster: true + proxyKubelets: + byHostname: true + byIP: true policies: resourceQuota: @@ -389,7 +409,7 @@ policies: count/persistentvolumeclaims: 20 scopeSelector: matchExpressions: [] - scopes: {} + scopes: [] limitRange: enabled: false @@ -436,7 +456,10 @@ experimental: enabled: false syncSettings: + disableSync: false + rewriteKubernetesService: false targetNamespace: "" + setOwner: true isolatedControlPlane: headless: false diff --git a/charts/eks/.helmignore b/charts/eks/.helmignore deleted file mode 100644 index 3b2afc666..000000000 --- a/charts/eks/.helmignore +++ /dev/null @@ -1,25 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. - -.DS_Store - -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ - -# Common backup files -*.swp -*.bak -*.tmp -*~ - -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/eks/Chart.yaml b/charts/eks/Chart.yaml deleted file mode 100644 index 53f33240f..000000000 --- a/charts/eks/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v2 -name: vcluster-eks -description: vcluster - Virtual Kubernetes Clusters (eks) -home: https://vcluster.com -icon: https://static.loft.sh/branding/logos/vcluster/vertical/vcluster_vertical.svg -keywords: - - developer - - development - - sharing - - share - - multi-tenancy - - tenancy - - cluster - - space - - namespace - - vcluster - - vclusters -maintainers: - - name: Loft Labs, Inc. - email: info@loft.sh - url: https://twitter.com/loft_sh -sources: - - https://github.com/loft-sh/vcluster -type: application - -version: 0.0.1 # version is auto-generated by release pipeline diff --git a/charts/eks/README.md b/charts/eks/README.md deleted file mode 100644 index 6541ee835..000000000 --- a/charts/eks/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -# vcluster (eks) - -## **[GitHub](https://github.com/loft-sh/vcluster)** • **[Website](https://www.vcluster.com)** • **[Quickstart](https://www.vcluster.com/docs/getting-started/setup)** • **[Documentation](https://www.vcluster.com/docs/what-are-virtual-clusters)** • **[Blog](https://loft.sh/blog)** • **[Twitter](https://twitter.com/loft_sh)** • **[Slack](https://slack.loft.sh/)** - -Create fully functional virtual Kubernetes clusters - Each vcluster runs inside a namespace of the underlying k8s cluster. It's cheaper than creating separate full-blown clusters and it offers better multi-tenancy and isolation than regular namespaces. - -## Prerequisites - -- Kubernetes 1.18+ -- Helm 3+ - -## Get Helm Repository Info - -```bash -helm repo add loft-sh https://charts.loft.sh -helm repo update -``` - -See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation. - -## Install Helm Chart - -```bash -helm upgrade [RELEASE_NAME] loft-sh/vcluster-eks -n [RELEASE_NAMESPACE] --create-namespace --install -``` - -See [vcluster docs](https://vcluster.com/docs) for configuration options. - -See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation. - -## Connect to the vcluster - -In order to connect to the installed vcluster, please install [vcluster cli](https://www.vcluster.com/docs/getting-started/setup) and run: - -```bash -vcluster connect [RELEASE_NAME] -n [RELEASE_NAMESPACE] -``` - -## Uninstall Helm Chart - -```bash -helm uninstall [RELEASE_NAME] -``` - -This removes all the Kubernetes components associated with the chart and deletes the release. - -See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation. - -### Why Virtual Kubernetes Clusters? - -- **Cluster Scoped Resources**: much more powerful than simple namespaces (virtual clusters allow users to use CRDs, namespaces, cluster roles etc.) -- **Ease of Use**: usable in any Kubernetes cluster and created in seconds either via a single command or [cluster-api](https://github.com/loft-sh/cluster-api-provider-vcluster) -- **Cost Efficient**: much cheaper and efficient than "real" clusters (single pod and shared resources just like for namespaces) -- **Lightweight**: built upon the ultra-fast k3s distribution with minimal overhead per virtual cluster (other distributions work as well) -- **Strict isolation**: complete separate Kubernetes control plane and access point for each vcluster while still being able to share certain services of the underlying host cluster -- **Cluster Wide Permissions**: allow users to install apps which require cluster-wide permissions while being limited to actually just one namespace within the host cluster -- **Great for Testing**: allow you to test different Kubernetes versions inside a single host cluster which may have a different version than the virtual clusters - -Learn more on [www.vcluster.com](https://vcluster.com). - -![vcluster Intro](https://github.com/loft-sh/vcluster/raw/main/docs/static/media/vcluster-comparison.png) - -Learn more in the [documentation](https://vcluster.com/docs/what-are-virtual-clusters). diff --git a/charts/eks/templates/NOTES.txt b/charts/eks/templates/NOTES.txt deleted file mode 100644 index 6cf960d62..000000000 --- a/charts/eks/templates/NOTES.txt +++ /dev/null @@ -1,10 +0,0 @@ -Thank you for installing vcluster. - -Your vcluster is named {{ .Release.Name }} in namespace {{ .Release.Namespace }}. - -To connect to the vcluster, use vcluster CLI (https://www.vcluster.com/docs/getting-started/setup): - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} -- kubectl get ns - - -For more information, please take a look at the vcluster docs at https://www.vcluster.com/docs diff --git a/charts/eks/templates/_coredns.tpl b/charts/eks/templates/_coredns.tpl deleted file mode 100644 index f5b1aff93..000000000 --- a/charts/eks/templates/_coredns.tpl +++ /dev/null @@ -1,52 +0,0 @@ -{{/* - Define a common coredns config -*/}} -{{- define "vcluster.corefile" -}} -Corefile: |- - {{- if .Values.coredns.config }} -{{ .Values.coredns.config | indent 8 }} - {{- else }} - .:1053 { - errors - health - ready - rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local - kubernetes cluster.local in-addr.arpa ip6.arpa { - {{- if .Values.pro }} - {{- if .Values.coredns.integrated }} - kubeconfig /pki/admin.conf - {{- end }} - {{- end }} - pods insecure - {{- if .Values.fallbackHostDns }} - fallthrough cluster.local in-addr.arpa ip6.arpa - {{- else }} - fallthrough in-addr.arpa ip6.arpa - {{- end }} - } - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - vcluster {{ toYaml .Values.coredns.plugin.config | b64enc }} - {{- end }} - hosts /etc/NodeHosts { - ttl 60 - reload 15s - fallthrough - } - prometheus :9153 - {{- if .Values.fallbackHostDns }} - forward . {{`{{.HOST_CLUSTER_DNS}}`}} - {{- else if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} - forward . /etc/resolv.conf {{ .Values.coredns.fallback }} { - policy sequential - } - {{- else }} - forward . /etc/resolv.conf - {{- end }} - cache 30 - loop - loadbalance - } - - import /etc/coredns/custom/*.server - {{- end }} -{{- end -}} diff --git a/charts/eks/templates/_helpers.tpl b/charts/eks/templates/_helpers.tpl deleted file mode 100644 index 50f08d6f9..000000000 --- a/charts/eks/templates/_helpers.tpl +++ /dev/null @@ -1,196 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "vcluster.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Whether the ingressclasses syncer should be enabled -*/}} -{{- define "vcluster.syncIngressclassesEnabled" -}} -{{- if or - (.Values.sync.ingressclasses).enabled - (and - .Values.sync.ingresses.enabled - (not .Values.sync.ingressclasses)) -}} - {{- true -}} -{{- end -}} -{{- end -}} - -{{/* -Whether to create a cluster role or not -*/}} -{{- define "vcluster.createClusterRole" -}} -{{- if or - (not (empty (include "vcluster.serviceMapping.fromHost" . ))) - (not (empty (include "vcluster.plugin.clusterRoleExtraRules" . ))) - (not (empty (include "vcluster.generic.clusterRoleExtraRules" . ))) - .Values.rbac.clusterRole.create - .Values.sync.hoststorageclasses.enabled - (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") - (include "vcluster.syncIngressclassesEnabled" . ) - .Values.pro - .Values.sync.nodes.enabled - .Values.sync.persistentvolumes.enabled - .Values.sync.storageclasses.enabled - .Values.sync.priorityclasses.enabled - .Values.sync.volumesnapshots.enabled - .Values.proxy.metricsServer.nodes.enabled - .Values.multiNamespaceMode.enabled - .Values.coredns.plugin.enabled -}} -{{- true -}} -{{- end -}} -{{- end -}} - -{{- define "vcluster.clusterRoleName" -}} -{{- printf "vc-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "vcluster.clusterRoleNameMultinamespace" -}} -{{- printf "vc-mn-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Syncer flags for enabling/disabling controllers -Prints only the flags that modify the defaults: -- when default controller has enabled: false => `- "--sync=-controller` -- when non-default controller has enabled: true => `- "--sync=controller` -*/}} -{{- define "vcluster.syncer.syncArgs" -}} -{{- $defaultEnabled := list "services" "configmaps" "secrets" "endpoints" "pods" "events" "persistentvolumeclaims" "fake-nodes" "fake-persistentvolumes" -}} -{{- if and (hasKey .Values.sync.nodes "enableScheduler") .Values.sync.nodes.enableScheduler -}} - {{- $defaultEnabled = concat $defaultEnabled (list "csinodes" "csidrivers" "csistoragecapacities" ) -}} -{{- end -}} -{{- range $key, $val := .Values.sync }} -{{- if and (has $key $defaultEnabled) (not $val.enabled) }} -- --sync=-{{ $key }} -{{- else if and (not (has $key $defaultEnabled)) ($val.enabled)}} -{{- if eq $key "legacy-storageclasses" }} -- --sync=hoststorageclasses -{{- else }} -- --sync={{ $key }} -{{- end -}} -{{- end -}} -{{- end -}} -{{- if not (include "vcluster.syncIngressclassesEnabled" . ) }} -- --sync=-ingressclasses -{{- end -}} -{{- end -}} - -{{/* - Cluster role rules defined by plugins -*/}} -{{- define "vcluster.plugin.clusterRoleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.clusterRole }} -{{- if $container.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.clusterRoleExtraRules" -}} -{{- if .Values.sync.generic.clusterRole }} -{{- if .Values.sync.generic.clusterRole.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined on global level -*/}} -{{- define "vcluster.rbac.clusterRoleExtraRules" -}} -{{- if .Values.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - - -{{/* - Role rules defined on global level -*/}} -{{- define "vcluster.rbac.roleExtraRules" -}} -{{- if .Values.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined by plugins -*/}} -{{- define "vcluster.plugin.roleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.role }} -{{- if $container.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.roleExtraRules" -}} -{{- if .Values.sync.generic.role }} -{{- if .Values.sync.generic.role.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Virtual cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromVirtual" -}} -{{- range $key, $value := .Values.mapServices.fromVirtual }} -- '--map-virtual-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} - -{{/* - Host cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromHost" -}} -{{- range $key, $value := .Values.mapServices.fromHost }} -- '--map-host-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} diff --git a/charts/eks/templates/_kind.tpl b/charts/eks/templates/_kind.tpl deleted file mode 100644 index c560376e7..000000000 --- a/charts/eks/templates/_kind.tpl +++ /dev/null @@ -1,64 +0,0 @@ - -{{/* - deployment kind -*/}} -{{- define "vcluster.kind" -}} -{{ if and .Values.embeddedEtcd.enabled .Values.pro }}StatefulSet{{ else }}Deployment{{ end }} -{{- end -}} - -{{/* - service name for statefulset -*/}} -{{- define "vcluster.statefulset.serviceName" }} -{{- if .Values.embeddedEtcd.enabled }} -serviceName: {{ .Release.Name }}-headless -{{- end }} -{{- end -}} - -{{/* - volumeClaimTemplate -*/}} -{{- define "vcluster.statefulset.volumeClaimTemplate" }} -{{- if .Values.embeddedEtcd.enabled }} -{{- if .Values.autoDeletePersistentVolumeClaims }} -{{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} -persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete -{{- end }} -{{- end }} -{{- if (hasKey .Values "volumeClaimTemplates") }} -volumeClaimTemplates: -{{ toYaml .Values.volumeClaimTemplates | indent 4 }} -{{- else if .Values.syncer.storage.persistence }} -volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if .Values.syncer.storage.className }} - storageClassName: {{ .Values.syncer.storage.className }} - {{- end }} - resources: - requests: - storage: {{ .Values.syncer.storage.size }} -{{- end }} -{{- end }} -{{- end -}} - - -{{/* - deployment strategy -*/}} -{{- define "vcluster.deployment.strategy" }} -{{- if not .Values.embeddedEtcd.enabled }} -strategy: - rollingUpdate: - maxSurge: 1 - {{- if (eq (int .Values.syncer.replicas) 1) }} - maxUnavailable: 0 - {{- else }} - maxUnavailable: 1 - {{- end }} - type: RollingUpdate -{{- end }} -{{- end -}} diff --git a/charts/eks/templates/_plugin.tpl b/charts/eks/templates/_plugin.tpl deleted file mode 100644 index b1ecd2fa2..000000000 --- a/charts/eks/templates/_plugin.tpl +++ /dev/null @@ -1,188 +0,0 @@ -{{/* - Plugin config definition -*/}} -{{- define "vcluster.plugins.config" -}} -{{- $pluginFound := false -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} -{{- $pluginFound = true -}} -{{- end }} -{{- if $pluginFound }} -- name: PLUGIN_CONFIG - value: |- -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} - {{ $key }}: {{ toYaml $container.config | nindent 6 }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume mount definition -*/}} -{{- define "vcluster.plugins.volumeMounts" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- mountPath: /plugins - name: plugins -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume definition -*/}} -{{- define "vcluster.plugins.volumes" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- name: plugins - emptyDir: {} -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin init container definition -*/}} -{{- define "vcluster.plugins.initContainers" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if or $container.command $container.args }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- else }} - command: ["sh"] - args: ["-c", "cp -r /plugin /plugins/{{ $key }}"] - {{- end }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - {{- if $container.volumeMounts }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- else }} - volumeMounts: - - mountPath: /plugins - name: plugins - {{- end }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} -{{- end }} -{{- end -}} - -{{/* - Extra Syncer Args for the legacy Plugins -*/}} -{{- define "vcluster.legacyPlugins.args" -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{- continue }} -{{- end }} -{{- if not $container.optional }} -- --plugins={{ $key }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Sidecar container definition for the legacy syncer parts -*/}} -{{- define "vcluster.legacyPlugins.containers" -}} -{{- $counter := -1 -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{ continue }} -{{- end }} -{{- $counter = add1 $counter }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if $container.workingDir }} - workingDir: {{ $container.workingDir }} - {{- end }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- if $container.terminationMessagePath }} - terminationMessagePath: {{ $container.terminationMessagePath }} - {{- end }} - {{- if $container.terminationMessagePolicy }} - terminationMessagePolicy: {{ $container.terminationMessagePolicy }} - {{- end }} - env: - - name: VCLUSTER_PLUGIN_ADDRESS - value: "localhost:{{ add 14000 $counter }}" - - name: VCLUSTER_PLUGIN_NAME - value: "{{ $key }}" - {{- if $container.env }} -{{ toYaml $container.env | indent 4 }} - {{- end }} - envFrom: -{{ toYaml $container.envFrom | indent 4 }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - lifecycle: -{{ toYaml $container.lifecycle | indent 4 }} - livenessProbe: -{{ toYaml $container.livenessProbe | indent 4 }} - readinessProbe: -{{ toYaml $container.readinessProbe | indent 4 }} - startupProbe: -{{ toYaml $container.startupProbe | indent 4 }} - volumeDevices: -{{ toYaml $container.volumeDevices | indent 4 }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} - {{- end }} -{{- end }} - - diff --git a/charts/eks/templates/coredns.yaml b/charts/eks/templates/coredns.yaml deleted file mode 100644 index df520c17c..000000000 --- a/charts/eks/templates/coredns.yaml +++ /dev/null @@ -1,215 +0,0 @@ -{{- if not .Values.headless }} -{{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-coredns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations}} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4}} - {{- end}} -data: -{{- if .Values.coredns.manifests }} - coredns.yaml: |- -{{ .Values.coredns.manifests | indent 4 }} -{{- else }} - coredns.yaml: |- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: coredns - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns - subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: -{{ include "vcluster.corefile" . | indent 6 }} - NodeHosts: "" - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/name: "CoreDNS" - spec: - replicas: {{ .Values.coredns.replicas }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - {{- if .Values.coredns.podAnnotations }} - annotations: -{{ toYaml .Values.coredns.podAnnotations | indent 12 }} - {{- end }} - labels: - k8s-app: kube-dns - {{- range $k, $v := .Values.coredns.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - priorityClassName: "system-cluster-critical" - serviceAccountName: coredns - nodeSelector: - kubernetes.io/os: linux - {{- if .Values.coredns.nodeSelector }} -{{ toYaml .Values.coredns.nodeSelector | indent 12 }} - {{- end }} - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - k8s-app: kube-dns - {{- if .Values.isolation.enabled }} - securityContext: - seccompProfile: - type: RuntimeDefault - {{- end }} - containers: - - name: coredns - image: "{{ .Values.defaultImageRegistry }}{{ .Values.coredns.image }}" - imagePullPolicy: IfNotPresent - resources: -{{ toYaml .Values.coredns.resources | indent 14}} - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - - name: custom-config-volume - mountPath: /etc/coredns/custom - readOnly: true - securityContext: - runAsNonRoot: true - runAsUser: {{`{{.RUN_AS_USER}}`}} - runAsGroup: {{`{{.RUN_AS_GROUP}}`}} - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 2 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile - - key: NodeHosts - path: NodeHosts - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - selector: - k8s-app: kube-dns - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/eks/templates/etcd-service.yaml b/charts/eks/templates/etcd-service.yaml deleted file mode 100644 index 57c60a145..000000000 --- a/charts/eks/templates/etcd-service.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if or (and (not .Values.embeddedEtcd.enabled) (not .Values.headless ) (not .Values.etcd.disabled )) .Values.embeddedEtcd.migrateFromEtcd }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-etcd - namespace: {{ .Release.Namespace }} - labels: - app: vcluster-etcd - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.etcd.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: ClusterIP - ports: - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - selector: - app: vcluster-etcd - release: {{ .Release.Name }} -{{- end }} diff --git a/charts/eks/templates/etcd-statefulset-service.yaml b/charts/eks/templates/etcd-statefulset-service.yaml deleted file mode 100644 index f037365bf..000000000 --- a/charts/eks/templates/etcd-statefulset-service.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if or (and (not .Values.embeddedEtcd.enabled) (not .Values.headless ) (not .Values.etcd.disabled )) .Values.embeddedEtcd.migrateFromEtcd }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-etcd-headless - namespace: {{ .Release.Namespace }} - labels: - app: vcluster-etcd - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.etcd.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - publishNotReadyAddresses: true - ports: - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - clusterIP: None - selector: - app: vcluster-etcd - release: "{{ .Release.Name }}" -{{- end }} diff --git a/charts/eks/templates/etcd-statefulset.yaml b/charts/eks/templates/etcd-statefulset.yaml deleted file mode 100644 index 2e3de57c0..000000000 --- a/charts/eks/templates/etcd-statefulset.yaml +++ /dev/null @@ -1,170 +0,0 @@ -{{- if or (and (not .Values.embeddedEtcd.enabled) (not .Values.headless ) (not .Values.etcd.disabled )) .Values.embeddedEtcd.migrateFromEtcd }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Release.Name }}-etcd - namespace: {{ .Release.Namespace }} - labels: - app: vcluster-etcd - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.etcd.labels }} -{{ toYaml .Values.etcd.labels | indent 4 }} -{{- end }} - {{- $annotations := merge .Values.globalAnnotations .Values.etcd.annotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - serviceName: {{ .Release.Name }}-etcd-headless - {{- if .Values.autoDeletePersistentVolumeClaims }} - {{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} - persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete - {{- end }} - {{- end }} - replicas: {{ .Values.etcd.replicas }} - podManagementPolicy: Parallel - selector: - matchLabels: - app: vcluster-etcd - release: {{ .Release.Name }} - {{- if (hasKey .Values.etcd "volumeClaimTemplates") }} - volumeClaimTemplates: -{{ toYaml .Values.etcd.volumeClaimTemplates | indent 4 }} - {{- else if .Values.etcd.storage.persistence }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if .Values.etcd.storage.className }} - storageClassName: {{ .Values.etcd.storage.className }} - {{- end }} - resources: - requests: - storage: {{ .Values.etcd.storage.size }} - {{- end }} - template: - metadata: - {{- if .Values.etcd.podAnnotations }} - annotations: -{{ toYaml .Values.etcd.podAnnotations | indent 8 }} - {{- end }} - labels: - app: vcluster-etcd - release: {{ .Release.Name }} - {{- range $k, $v := .Values.etcd.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - terminationGracePeriodSeconds: 10 - {{- if .Values.etcd.affinity }} - affinity: -{{ toYaml .Values.etcd.affinity | indent 8 }} - {{- end }} - {{- if .Values.etcd.topologySpreadConstraints }} - topologySpreadConstraints: -{{ toYaml .Values.etcd.topologySpreadConstraints | indent 8 }} - {{- end }} - nodeSelector: -{{ toYaml .Values.etcd.nodeSelector | indent 8 }} - affinity: -{{ toYaml .Values.etcd.affinity | indent 8 }} - tolerations: -{{ toYaml .Values.etcd.tolerations | indent 8 }} - automountServiceAccountToken: false - volumes: - - name: certs - secret: - secretName: {{ .Release.Name }}-certs - {{- if .Values.etcd.volumes }} -{{ toYaml .Values.etcd.volumes | indent 8 }} - {{- end }} - {{- if not .Values.etcd.storage.persistence }} - - name: data - emptyDir: {} - {{- end }} - {{- if .Values.etcd.priorityClassName }} - priorityClassName: {{ .Values.etcd.priorityClassName }} - {{- end }} - {{- if .Values.etcd.fsGroup }} - securityContext: - fsGroup: {{ .Values.etcd.fsGroup }} - {{- end }} - containers: - - name: etcd - image: "{{ .Values.defaultImageRegistry }}{{ .Values.etcd.image }}" - command: - - etcd - - '--cert-file=/run/config/pki/etcd-server.crt' - - '--client-cert-auth=true' - - '--data-dir=/var/lib/etcd' - - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2379' - - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380' - {{- $releaseName := .Release.Name -}} - {{- $releaseNamespace := .Release.Namespace }} - - '--initial-cluster={{ range $index := untilStep 0 (int .Values.etcd.replicas) 1 }}{{ if (ne (int $index) 0) }},{{ end }}{{ $releaseName }}-etcd-{{ $index }}=https://{{ $releaseName }}-etcd-{{ $index }}.{{ $releaseName }}-etcd-headless.{{ $releaseNamespace }}:2380{{ end }}' - - '--initial-cluster-token={{ .Release.Name }}' - - '--initial-cluster-state=new' - - '--listen-client-urls=https://0.0.0.0:2379' - - '--listen-metrics-urls=http://0.0.0.0:2381' - - '--listen-peer-urls=https://0.0.0.0:2380' - - '--key-file=/run/config/pki/etcd-server.key' - - '--name=$(NAME)' - - '--peer-cert-file=/run/config/pki/etcd-peer.crt' - - '--peer-client-cert-auth=true' - - '--peer-key-file=/run/config/pki/etcd-peer.key' - - '--peer-trusted-ca-file=/run/config/pki/etcd-ca.crt' - - '--snapshot-count=10000' - - '--trusted-ca-file=/run/config/pki/etcd-ca.crt' - {{- range $f := .Values.etcd.extraArgs }} - - {{ $f | quote }} - {{- end }} - securityContext: -{{ toYaml .Values.etcd.securityContext | indent 10 }} - {{- if .Values.etcd.imagePullPolicy }} - imagePullPolicy: {{ .Values.etcd.imagePullPolicy }} - {{- end }} - env: - - name: NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if .Values.etcd.env }} -{{ toYaml .Values.etcd.env | indent 10 }} - {{- end }} - volumeMounts: - - name: data - mountPath: /var/lib/etcd - - mountPath: /run/config/pki - name: certs - readOnly: true - {{- if .Values.etcd.volumeMounts }} -{{ toYaml .Values.etcd.volumeMounts | indent 10 }} - {{- end }} - resources: -{{ toYaml .Values.etcd.resources | indent 10 }} - livenessProbe: - httpGet: - path: /health - port: 2381 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 15 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 8 - startupProbe: - httpGet: - path: /health - port: 2381 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 15 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 24 -{{- end }} diff --git a/charts/eks/templates/ingress.yaml b/charts/eks/templates/ingress.yaml deleted file mode 100644 index 26191ffb6..000000000 --- a/charts/eks/templates/ingress.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - {{- $annotations := merge .Values.ingress.annotations .Values.globalAnnotations }} - {{- if .Values.ingress.tls }} - {{- $annotations = omit $annotations "nginx.ingress.kubernetes.io/ssl-passthrough" }} - {{- end }} - {{- if $annotations }} - annotations: - {{- toYaml $annotations | nindent 4 }} - {{- end }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - {{- if .Values.ingress.ingressClassName }} - ingressClassName: {{ .Values.ingress.ingressClassName | quote }} - {{- end }} - rules: - - host: {{ .Values.ingress.host | quote }} - http: - paths: - - backend: - service: - name: {{ .Release.Name }} - port: - name: https - path: / - pathType: {{ .Values.ingress.pathType }} - {{- with .Values.ingress.tls }} - tls: - {{- toYaml . | nindent 4 }} - {{- end -}} -{{- end }} diff --git a/charts/eks/templates/init-configmap.yaml b/charts/eks/templates/init-configmap.yaml deleted file mode 100644 index cc461ae6c..000000000 --- a/charts/eks/templates/init-configmap.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- if .Values.init.manifests }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-init-manifests - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations}} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: - manifests: |- - {{ .Values.init.manifests | nindent 4 | trim }} - {{ tpl .Values.init.manifestsTemplate $ | nindent 4 | trim }} - {{- if .Values.init.helm }} - charts: |- - {{- range .Values.init.helm }} - {{- /* only render this chart entry if either of chart or bundle is defined */}} - {{- if .chart }} - - name: {{ .chart.name }} - repo: {{ .chart.repo }} - version: {{ .chart.version }} - {{- if .chart.username }} - username: {{ .chart.username }} - {{- end }} - {{- if .chart.password }} - password: {{ .chart.password }} - {{- end }} - {{- if .insecure }} - insecure: true - {{- end}} - {{- end }} - {{- if .bundle }} - bundle: {{ .bundle }} - {{- end }} - {{- if or .chart .bundle }} - {{- if or .values .valuesTemplate }} - values: |- - {{ (.values | default "") | nindent 8 | trim }} - {{ tpl (.valuesTemplate | default "") $ | nindent 8 | trim }} - {{- end}} - {{- if .release }} - timeout: {{ .timeout | default "120s" | quote }} - releaseName: {{ .release.name }} - releaseNamespace: {{ .release.namespace }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/eks/templates/integrated-coredns.yaml b/charts/eks/templates/integrated-coredns.yaml deleted file mode 100644 index 16907cba0..000000000 --- a/charts/eks/templates/integrated-coredns.yaml +++ /dev/null @@ -1,64 +0,0 @@ -{{- if .Values.pro }} -{{- if .Values.coredns.integrated }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-dns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{ include "vcluster.corefile" . | indent 2 }} - coredns.yaml: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: - NodeHosts: "" - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} diff --git a/charts/eks/templates/limitrange.yaml b/charts/eks/templates/limitrange.yaml deleted file mode 100644 index ed219d063..000000000 --- a/charts/eks/templates/limitrange.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.limitRange.enabled }} -apiVersion: v1 -kind: LimitRange -metadata: - name: {{ .Release.Name }}-limit-range - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - limits: - - default: - {{- range $key, $val := .Values.isolation.limitRange.default }} - {{ $key }}: {{ $val | quote }} - {{- end }} - defaultRequest: - {{- range $key, $val := .Values.isolation.limitRange.defaultRequest }} - {{ $key }}: {{ $val | quote }} - {{- end }} - type: Container -{{- end }} diff --git a/charts/eks/templates/networkpolicy.yaml b/charts/eks/templates/networkpolicy.yaml deleted file mode 100644 index 81886898b..000000000 --- a/charts/eks/templates/networkpolicy.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-workloads - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - egress: - # Allows outgoing connections to the vcluster control plane - - ports: - - port: 443 - - port: 8443 - to: - - podSelector: - matchLabels: - release: {{ .Release.Name }} - # Allows outgoing connections to DNS server - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Allows outgoing connections to the internet or - # other vcluster workloads - - to: - - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - - ipBlock: - cidr: {{ .Values.isolation.networkPolicy.outgoingConnections.ipBlock.cidr }} - except: - {{- range .Values.isolation.networkPolicy.outgoingConnections.ipBlock.except }} - - {{ . }} - {{- end }} - policyTypes: - - Egress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-control-plane - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - podSelector: - matchLabels: - release: {{ .Release.Name }} - egress: - # Allows outgoing connections to all pods with - # port 443, 8443 or 6443. This is needed for host Kubernetes - # access - - ports: - - port: 443 - - port: 8443 - - port: 6443 - # Allows outgoing connections to all vcluster workloads - # or kube system dns server - - to: - - podSelector: {} - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: 'kube-system' - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Egress -{{- end }} \ No newline at end of file diff --git a/charts/eks/templates/rbac/clusterrole.yaml b/charts/eks/templates/rbac/clusterrole.yaml deleted file mode 100644 index 984267cae..000000000 --- a/charts/eks/templates/rbac/clusterrole.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: -{{- if .Values.pro }} - - apiGroups: ["cluster.loft.sh", "storage.loft.sh"] - resources: ["features", "virtualclusters"] - verbs: ["get", "list", "watch"] -{{- end }} - {{- if or .Values.pro .Values.sync.nodes.enabled }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: [ "pods", "nodes/metrics", "nodes/stats"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.coredns.plugin.enabled }} - - apiGroups: [""] - resources: [ "pods"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled (or (not .Values.isolation.enabled) (and .Values.isolation.nodeProxyPermission.enabled .Values.isolation.enabled)) }} - - apiGroups: [""] - resources: ["nodes/proxy"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled .Values.sync.nodes.syncNodeChanges }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["update", "patch"] - {{- end }} - {{- if .Values.sync.persistentvolumes.enabled }} - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if .Values.sync.nodes.enableScheduler }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses","csinodes","csidrivers","csistoragecapacities"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if (include "vcluster.syncIngressclassesEnabled" . ) }} - - apiGroups: ["networking.k8s.io"] - resources: ["ingressclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.storageclasses.enabled }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if or .Values.sync.hoststorageclasses.enabled (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") .Values.rbac.clusterRole.create }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.priorityclasses.enabled }} - - apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if (not (empty (include "vcluster.serviceMapping.fromHost" . ))) }} - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.proxy.metricsServer.nodes.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["nodes"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.clusterRoleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/eks/templates/rbac/clusterrolebinding.yaml b/charts/eks/templates/rbac/clusterrolebinding.yaml deleted file mode 100644 index 0a5645d28..000000000 --- a/charts/eks/templates/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ template "vcluster.clusterRoleName" . }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/eks/templates/rbac/role.yaml b/charts/eks/templates/rbac/role.yaml deleted file mode 100644 index de05bbb62..000000000 --- a/charts/eks/templates/rbac/role.yaml +++ /dev/null @@ -1,104 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRole -{{- else -}} -kind: Role -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} -{{- end }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: - {{- if .Values.pro }} - - apiGroups: [""] - {{- $resources := list "configmaps" "secrets" "services" "pods" "pods/attach" "pods/portforward" "pods/exec" "persistentvolumeclaims" }} - {{- range $excluded := .Values.rbac.role.excludedApiResources }} - {{- $resources = without $resources $excluded }} - {{- end}} - resources: {{ $resources | toJson }} - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- else }} - - apiGroups: [""] - resources: ["configmaps", "secrets", "services", "pods", "pods/attach", "pods/portforward", "pods/exec", "persistentvolumeclaims"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.status }} - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.ephemeralContainers }} - - apiGroups: [""] - resources: ["pods/ephemeralcontainers"] - verbs: ["patch", "update"] - {{- end }} - {{- if or .Values.sync.endpoints.enabled .Values.headless }} - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["create", "delete", "patch", "update"] - {{- end }} - {{- if gt (int .Values.syncer.replicas) 1 }} - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: [""] - resources: ["endpoints", "events", "pods/log"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.ingresses.enabled}} - - apiGroups: ["networking.k8s.io"] - resources: ["ingresses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "deployments"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.networkpolicies.enabled }} - - apiGroups: ["networking.k8s.io"] - resources: ["networkpolicies"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.serviceaccounts.enabled }} - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.poddisruptionbudgets.enabled }} - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.openshift.enable }} - {{- if .Values.sync.endpoints.enabled }} - - apiGroups: [""] - resources: ["endpoints/restricted"] - verbs: ["create"] - {{- end }} - {{- end }} - {{- if .Values.proxy.metricsServer.pods.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["pods"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.roleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.roleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.roleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/eks/templates/rbac/rolebinding.yaml b/charts/eks/templates/rbac/rolebinding.yaml deleted file mode 100644 index 676e5002a..000000000 --- a/charts/eks/templates/rbac/rolebinding.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRoleBinding -{{- else -}} -kind: RoleBinding -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -{{- end }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: -{{- if .Values.multiNamespaceMode.enabled }} - kind: ClusterRole - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - kind: Role - name: {{ .Release.Name }} -{{- end }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/eks/templates/resourcequota.yaml b/charts/eks/templates/resourcequota.yaml deleted file mode 100644 index b19d89fff..000000000 --- a/charts/eks/templates/resourcequota.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.resourceQuota.enabled }} -apiVersion: v1 -kind: ResourceQuota -metadata: - name: {{ .Release.Name }}-quota - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - hard: - {{- range $key, $val := .Values.isolation.resourceQuota.quota }} - {{ $key }}: {{ $val | quote }} - {{- end }} - - {{- if .Values.isolation.resourceQuota.scopeSelector.matchExpressions }} - scopeSelector: - matchExpressions: - {{- toYaml .Values.isolation.resourceQuota.scopeSelector.matchExpressions | nindent 4 }} - {{- end}} - - {{- if .Values.isolation.resourceQuota.scopes }} - scopes: - {{- toYaml .Values.isolation.resourceQuota.scopes | nindent 4 }} - {{- end}} - -{{- end }} diff --git a/charts/eks/templates/service-monitor.yaml b/charts/eks/templates/service-monitor.yaml deleted file mode 100644 index 683506ab5..000000000 --- a/charts/eks/templates/service-monitor.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if .Values.monitoring.serviceMonitor.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: vcluster - release: "{{ .Release.Name }}" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: "{{ .Release.Service }}" - endpoints: - - interval: 30s - port: https - path: /metrics - scheme: https - tlsConfig: - ca: - secret: - name: vc-{{ .Release.Name }} - key: certificate-authority - cert: - secret: - name: vc-{{ .Release.Name }} - key: client-certificate - keySecret: - name: vc-{{ .Release.Name }} - key: client-key -{{- end }} diff --git a/charts/eks/templates/service.yaml b/charts/eks/templates/service.yaml deleted file mode 100644 index b575a9fe1..000000000 --- a/charts/eks/templates/service.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: {{ if eq .Values.service.type "LoadBalancer" -}}ClusterIP{{- else }} {{- .Values.service.type }} {{- end }} - ports: - - name: https - port: 443 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.httpsNodePort }} - protocol: TCP - - name: kubelet - port: 10250 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.kubeletNodePort }} - protocol: TCP - {{- if .Values.service.externalIPs }} - externalIPs: - {{- range $f := .Values.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- if eq .Values.service.type "NodePort" }} - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} ---- -{{ if eq .Values.service.type "LoadBalancer" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-lb - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.service.loadBalancerAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: LoadBalancer - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.service.loadBalancerClass }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range $f := .Values.service.loadBalancerSourceRanges }} - - "{{ $f }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/eks/templates/serviceaccount.yaml b/charts/eks/templates/serviceaccount.yaml deleted file mode 100644 index 504998037..000000000 --- a/charts/eks/templates/serviceaccount.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} -{{- end }} diff --git a/charts/eks/templates/statefulset-service.yaml b/charts/eks/templates/statefulset-service.yaml deleted file mode 100644 index e7b5d3d91..000000000 --- a/charts/eks/templates/statefulset-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if eq ( include "vcluster.kind" . ) "StatefulSet" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-headless - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "vcluster.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - publishNotReadyAddresses: true - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.embeddedEtcd.enabled }} - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - {{- end }} - clusterIP: None - selector: - app: vcluster - release: "{{ .Release.Name }}" -{{- end }} diff --git a/charts/eks/templates/syncer.yaml b/charts/eks/templates/syncer.yaml deleted file mode 100644 index b82ac99eb..000000000 --- a/charts/eks/templates/syncer.yaml +++ /dev/null @@ -1,447 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: apps/v1 -kind: {{ include "vcluster.kind" . }} -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.syncer.labels }} -{{ toYaml .Values.syncer.labels | indent 4 }} -{{- end }} - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.annotations }} - {{- if $annotations}} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - {{- include "vcluster.deployment.strategy" . | indent 2 }} - {{- include "vcluster.statefulset.serviceName" . | indent 2 }} - {{- include "vcluster.statefulset.volumeClaimTemplate" . | indent 2 }} - replicas: {{ .Values.syncer.replicas }} - selector: - matchLabels: - app: vcluster - release: {{ .Release.Name }} - template: - metadata: - {{- if .Values.syncer.podAnnotations }} - annotations: -{{ toYaml .Values.syncer.podAnnotations | indent 8 }} - {{- end }} - labels: - app: vcluster - release: {{ .Release.Name }} - {{- range $k, $v := .Values.syncer.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - terminationGracePeriodSeconds: 10 - {{- if .Values.syncer.affinity }} - affinity: -{{ toYaml .Values.syncer.affinity | indent 8 }} - {{- else if (gt (int .Values.syncer.replicas) 1) }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - # if possible avoid scheduling more than one pod on one node - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: "kubernetes.io/hostname" - # if possible avoid scheduling pod onto node that is in the same zone as one or more vcluster pods are running - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: topology.kubernetes.io/zone - {{- end }} - {{- if .Values.syncer.topologySpreadConstraints }} - topologySpreadConstraints: -{{ toYaml .Values.syncer.topologySpreadConstraints | indent 8 }} - {{- end }} - nodeSelector: -{{ toYaml .Values.syncer.nodeSelector | indent 8 }} - affinity: -{{ toYaml .Values.syncer.affinity | indent 8 }} - tolerations: -{{ toYaml .Values.syncer.tolerations | indent 8 }} - {{- if .Values.serviceAccount.name }} - serviceAccountName: {{ .Values.serviceAccount.name }} - {{- else }} - serviceAccountName: vc-{{ .Release.Name }} - {{- end }} - {{- if .Values.syncer.podSecurityContext }} - securityContext: -{{ toYaml .Values.syncer.podSecurityContext | indent 8 }} - {{- end }} - volumes: - {{- include "vcluster.plugins.volumes" . | indent 8 }} - - name: helm-cache - emptyDir: {} - - name: tmp - emptyDir: {} - - name: certs - emptyDir: {} - {{- if .Values.volumes }} -{{ toYaml .Values.volumes | indent 8 }} - {{- end }} - - name: binaries - emptyDir: {} - {{- if .Values.syncer.volumes }} -{{ toYaml .Values.syncer.volumes | indent 8 }} - {{- end }} - {{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} - - name: coredns - configMap: - name: {{ .Release.Name }}-coredns - {{- else if .Values.coredns.integrated }} - - name: coredns - configMap: - name: {{ .Release.Name }}-dns - {{- end }} - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - {{- if .Values.syncer.priorityClassName }} - priorityClassName: {{ .Values.syncer.priorityClassName }} - {{- end }} - initContainers: - {{- include "vcluster.plugins.initContainers" . | indent 6 }} - # this is needed because the k8s containers are distroless and thus we don't have any - # way of copying the binaries otherwise - - name: vcluster-copy - {{- if .Values.syncer.image }} - image: "{{ .Values.defaultImageRegistry }}{{ .Values.syncer.image }}" - {{- else }} - {{- if .Values.pro }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster-pro:{{ .Chart.Version }}" - {{- else }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster:{{ .Chart.Version }}" - {{- end }} - {{- end }} - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /bin/sh - args: - - -c - - "cp /vcluster /binaries/vcluster" - {{- if .Values.syncer.imagePullPolicy }} - imagePullPolicy: {{ .Values.syncer.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- if not .Values.controller.disabled }} - - name: kube-controller-manager - image: "{{ .Values.defaultImageRegistry }}{{ .Values.controller.image }}" - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /binaries/vcluster - args: - - cp - - /usr/local/bin/kube-controller-manager - - /binaries/kube-controller-manager - {{- if .Values.controller.imagePullPolicy }} - imagePullPolicy: {{ .Values.controller.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} - {{- if not .Values.api.disabled }} - - name: kube-apiserver - image: "{{ .Values.defaultImageRegistry }}{{ .Values.api.image }}" - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /binaries/vcluster - args: - - cp - - /usr/local/bin/kube-apiserver - - /binaries/kube-apiserver - {{- if .Values.api.imagePullPolicy }} - imagePullPolicy: {{ .Values.api.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} - containers: - - name: syncer - {{- if .Values.syncer.image }} - image: "{{ .Values.defaultImageRegistry }}{{ .Values.syncer.image }}" - {{- else }} - {{- if .Values.pro }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster-pro:{{ .Chart.Version }}" - {{- else }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster:{{ .Chart.Version }}" - {{- end }} - {{- end }} - {{- if .Values.syncer.workingDir }} - workingDir: {{ .Values.syncer.workingDir }} - {{- end }} - {{- if .Values.syncer.command }} - command: - {{- range $f := .Values.syncer.command }} - - {{ $f | quote }} - {{- end }} - {{- end }} - args: - - --name={{ .Release.Name }} - - --request-header-ca-cert=/pki/front-proxy-ca.crt - - --client-ca-cert=/pki/ca.crt - - --server-ca-cert=/pki/ca.crt - - --server-ca-key=/pki/ca.key - - --kube-config=/pki/admin.conf - - --service-account=vc-workload-{{ .Release.Name }} - {{- if .Values.embeddedEtcd.migrateFromEtcd }} - - --migrate-from=https://{{ .Release.Name }}-etcd:2379 - {{- end }} - {{- include "vcluster.legacyPlugins.args" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromHost" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromVirtual" . | indent 10 }} - {{- if .Values.defaultImageRegistry }} - - --default-image-registry={{ .Values.defaultImageRegistry }} - {{- end }} - {{- if .Values.syncer.kubeConfigContextName }} - - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }} - {{- end }} - {{- if .Values.pro }} - {{- if .Values.proLicenseSecret }} - - --pro-license-secret={{ .Values.proLicenseSecret }} - {{- end }} - {{- if .Values.embeddedEtcd.enabled }} - - --etcd-embedded - - --etcd-replicas={{ .Values.syncer.replicas }} - {{- end }} - {{- end }} - {{- if (gt (int .Values.syncer.replicas) 1) }} - - --leader-elect=true - {{- else }} - - --leader-elect=false - {{- end }} - {{- if .Values.ingress.enabled }} - - --tls-san={{ .Values.ingress.host }} - {{- end }} - {{- include "vcluster.syncer.syncArgs" . | indent 10 -}} - {{- if .Values.sync.nodes.syncAllNodes }} - - --sync-all-nodes - {{- end }} - {{- if .Values.isolation.enabled }} - - --enforce-pod-security-standard={{ .Values.isolation.podSecurityStandard }} - {{- end}} - {{- if .Values.sync.nodes.nodeSelector }} - - --node-selector={{ .Values.sync.nodes.nodeSelector }} - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - --multi-namespace-mode=true - {{- end }} - {{- if .Values.sync.configmaps.all }} - - --sync-all-configmaps=true - {{- end }} - {{- if .Values.sync.secrets.all }} - - --sync-all-secrets=true - {{- end }} - {{- if not .Values.sync.nodes.fakeKubeletIPs }} - - --fake-kubelet-ips=false - {{- end }} - {{- if or .Values.proxy.metricsServer.nodes.enabled .Values.proxy.metricsServer.pods.enabled }} - - --proxy-metrics-server=true - {{- end }} - {{- if .Values.coredns.integrated }} - - --integrated-coredns=true - {{- end }} - {{- if .Values.centralAdmission.validatingWebhooks }} - {{- range .Values.centralAdmission.validatingWebhooks }} - - --enforce-validating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- if .Values.centralAdmission.mutatingWebhooks }} - {{- range .Values.centralAdmission.mutatingWebhooks }} - - --enforce-mutating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - - --use-coredns-plugin=true - {{- end }} - {{- range $f := .Values.syncer.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- if .Values.syncer.livenessProbe }} - {{- if .Values.syncer.livenessProbe.enabled }} - livenessProbe: - httpGet: - path: /healthz - port: 8443 - scheme: HTTPS - failureThreshold: 10 - initialDelaySeconds: 60 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.readinessProbe }} - {{- if .Values.syncer.readinessProbe.enabled }} - startupProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 300 - periodSeconds: 6 - readinessProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 30 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.imagePullPolicy }} - imagePullPolicy: {{ .Values.syncer.imagePullPolicy }} - {{- end }} - securityContext: -{{ toYaml .Values.syncer.securityContext | indent 10 }} - env: - {{- include "vcluster.plugins.config" . | indent 10 }} - - name: VCLUSTER_DISTRO - value: eks - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if eq (.Values.syncer.replicas | toString | atoi) 1 }} - - name: VCLUSTER_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- end }} - {{- if .Values.syncer.env }} -{{ toYaml .Values.syncer.env | indent 10 }} - {{- end }} - {{- if .Values.sync.generic.config }} - - name: CONFIG - value: |- - {{- .Values.sync.generic.config | nindent 14 }} - {{- end }} - - name: VCLUSTER_TELEMETRY_CONFIG - value: {{ .Values.telemetry | toJson | quote }} - {{- if not .Values.api.disabled }} - - name: APISERVER_COMMAND - value: |- - command: - - /binaries/kube-apiserver - - '--advertise-address=127.0.0.1' - - '--bind-address=127.0.0.1' - - '--allow-privileged=true' - - '--authorization-mode=RBAC' - - '--client-ca-file=/pki/ca.crt' - - '--enable-bootstrap-token-auth=true' - - '--etcd-cafile=/pki/etcd/ca.crt' - - '--etcd-certfile=/pki/apiserver-etcd-client.crt' - - '--etcd-keyfile=/pki/apiserver-etcd-client.key' - {{- if .Values.embeddedEtcd.enabled }} - - '--etcd-servers=https://127.0.0.1:2379' - {{- else }} - - '--etcd-servers=https://{{ .Release.Name }}-etcd:2379' - {{- end }} - - '--proxy-client-cert-file=/pki/front-proxy-client.crt' - - '--proxy-client-key-file=/pki/front-proxy-client.key' - - '--requestheader-allowed-names=front-proxy-client' - - '--requestheader-client-ca-file=/pki/front-proxy-ca.crt' - - '--requestheader-extra-headers-prefix=X-Remote-Extra-' - - '--requestheader-group-headers=X-Remote-Group' - - '--requestheader-username-headers=X-Remote-User' - - '--secure-port=6443' - - '--service-account-issuer=https://kubernetes.default.svc.cluster.local' - - '--service-account-key-file=/pki/sa.pub' - - '--service-account-signing-key-file=/pki/sa.key' - - '--tls-cert-file=/pki/apiserver.crt' - - '--tls-private-key-file=/pki/apiserver.key' - - '--watch-cache=false' - - '--endpoint-reconciler-type=none' - {{- range $f := .Values.api.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- end }} - {{- if not .Values.controller.disabled }} - - name: CONTROLLER_COMMAND - value: |- - command: - - /binaries/kube-controller-manager - - '--authentication-kubeconfig=/pki/controller-manager.conf' - - '--authorization-kubeconfig=/pki/controller-manager.conf' - - '--bind-address=127.0.0.1' - - '--client-ca-file=/pki/ca.crt' - - '--cluster-name=kubernetes' - - '--cluster-signing-cert-file=/pki/ca.crt' - - '--cluster-signing-key-file=/pki/ca.key' - - '--controllers=*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' - - '--node-monitor-grace-period=1h' - - '--node-monitor-period=1h' - - '--horizontal-pod-autoscaler-sync-period=60s' - - '--kubeconfig=/pki/controller-manager.conf' - {{- if (gt (int .Values.syncer.replicas) 1) }} - - '--leader-elect=true' - {{- else }} - - '--leader-elect=false' - {{- end }} - - '--node-monitor-grace-period=180s' - - '--node-monitor-period=30s' - - '--pvclaimbinder-sync-period=60s' - - '--requestheader-client-ca-file=/pki/front-proxy-ca.crt' - - '--root-ca-file=/pki/ca.crt' - - '--service-account-private-key-file=/pki/sa.key' - - '--use-service-account-credentials=true' - {{- range $f := .Values.controller.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- end }} - volumeMounts: - {{- include "vcluster.plugins.volumeMounts" . | indent 10 }} - {{- if eq ( include "vcluster.kind" . ) "StatefulSet" }} - - name: data - mountPath: /data - {{- end }} - - name: helm-cache - mountPath: /.cache/helm - - name: tmp - mountPath: /tmp - - mountPath: /pki - name: certs - - mountPath: /binaries - name: binaries - {{- if .Values.syncer.volumeMounts }} -{{ toYaml .Values.syncer.volumeMounts | indent 10 }} - {{- end }} - {{- if .Values.syncer.extraVolumeMounts }} -{{ toYaml .Values.syncer.extraVolumeMounts | indent 10 }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} -{{- include "vcluster.legacyPlugins.containers" . | indent 6 }} -{{- end }} diff --git a/charts/eks/templates/workloadserviceaccount.yaml b/charts/eks/templates/workloadserviceaccount.yaml deleted file mode 100644 index 8d05fc1e8..000000000 --- a/charts/eks/templates/workloadserviceaccount.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-workload-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.workloadServiceAccount.annotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} diff --git a/charts/eks/tests/README.md b/charts/eks/tests/README.md deleted file mode 100644 index 228e10d6e..000000000 --- a/charts/eks/tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Add [unittest plugin](https://github.com/helm-unittest/helm-unittest) via: -``` -helm plugin install https://github.com/helm-unittest/helm-unittest.git -``` - -Run tests via: -``` -helm unittest charts/eks -d -``` diff --git a/charts/eks/tests/clusterrole_test.yaml b/charts/eks/tests/clusterrole_test.yaml deleted file mode 100644 index 3385c786c..000000000 --- a/charts/eks/tests/clusterrole_test.yaml +++ /dev/null @@ -1,41 +0,0 @@ -suite: ClusterRole -templates: - - rbac/clusterrole.yaml - -tests: - - it: should create clusterrole - set: - rbac: - clusterRole: - create: true - asserts: - - hasDocuments: - count: 1 - - it: should not create clusterrole - set: - rbac: - clusterRole: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - clusterRole: - create: true - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/eks/tests/etcd_test.yaml b/charts/eks/tests/etcd_test.yaml deleted file mode 100644 index 931ce5d34..000000000 --- a/charts/eks/tests/etcd_test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -suite: Etcd -templates: - - etcd-statefulset.yaml - - etcd-service.yaml - - etcd-statefulset-service.yaml - -tests: - - it: should have etcd when migratefrometcd is true - set: - pro: true - embeddedEtcd: - enabled: true - migrateFromEtcd: true - asserts: - - hasDocuments: - count: 1 - - it: shouldn't have etcd when migratefrometcd is false - set: - pro: true - embeddedEtcd: - enabled: true - migrateFromEtcd: false - asserts: - - hasDocuments: - count: 0 - diff --git a/charts/eks/tests/legacy-plugins_test.yaml b/charts/eks/tests/legacy-plugins_test.yaml deleted file mode 100644 index 6977c7562..000000000 --- a/charts/eks/tests/legacy-plugins_test.yaml +++ /dev/null @@ -1,77 +0,0 @@ -suite: Legacy Plugins -templates: - - syncer.yaml - -tests: - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - bootstrap-with-deployment2: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment2" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[2].name - value: bootstrap-with-deployment2 - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[2].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - equal: - path: spec.template.spec.containers[2].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14001 - - - it: should check no legacy plugin rendering - asserts: - - hasDocuments: - count: 1 - - lengthEqual: - path: spec.template.spec.containers - count: 1 diff --git a/charts/eks/tests/plugins_test.yaml b/charts/eks/tests/plugins_test.yaml deleted file mode 100644 index 2ce3149dc..000000000 --- a/charts/eks/tests/plugins_test.yaml +++ /dev/null @@ -1,100 +0,0 @@ -suite: Plugins -templates: - - syncer.yaml - -tests: - - it: should check plugin config rendering - set: - plugin: - plugin1: - version: v2 - config: - myConfig: true - plugin2: - version: v2 - image: test - plugin3: - version: v2 - image: test123 - config: - myOtherConfig: - - test123 - - test456 - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.containers[0].env[0].name - value: PLUGIN_CONFIG - - equal: - path: spec.template.spec.containers[0].env[0].value - value: |- - plugin1: - myConfig: true - plugin3: - myOtherConfig: - - test123 - - test456 - - - it: should check plugin rendering - set: - plugin: - bootstrap-with-deployment: - version: v2 - image: test - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.volumes[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins diff --git a/charts/eks/tests/role_test.yaml b/charts/eks/tests/role_test.yaml deleted file mode 100644 index 5eeedba74..000000000 --- a/charts/eks/tests/role_test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -suite: Role -templates: - - rbac/role.yaml - -tests: - - it: should not create role - set: - rbac: - role: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - role: - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/eks/tests/syncer_test.yaml b/charts/eks/tests/syncer_test.yaml deleted file mode 100644 index 7713b2e10..000000000 --- a/charts/eks/tests/syncer_test.yaml +++ /dev/null @@ -1,52 +0,0 @@ -suite: Syncer -templates: - - syncer.yaml - -tests: - - it: should pass pro license secret as a flag - set: - pro: true - proLicenseSecret: "my-test-secret" - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--pro-license-secret=my-test-secret" - count: 1 - - it: should be a statefulset when embeddedEtcd is enabled - set: - pro: true - embeddedEtcd: - enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: StatefulSet - - exists: - path: spec.serviceName - - exists: - path: spec.volumeClaimTemplates - - notExists: - path: spec.strategy - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /data - - - it: should be a deployment when embeddedEtcd is disabled - set: - pro: true - embeddedEtcd: - enabled: false - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Deployment - - notExists: - path: spec.serviceName - - notExists: - path: spec.volumeClaimTemplates - - exists: - path: spec.strategy diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml deleted file mode 100644 index dd20e9f47..000000000 --- a/charts/eks/values.yaml +++ /dev/null @@ -1,505 +0,0 @@ -# DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed -# images within the vcluster will not be rewritten. -defaultImageRegistry: "" - -# Global annotations to add to all objects -globalAnnotations: {} - -# If vCluster.Pro is enabled -pro: false - -# Defines where vCluster should search for a license secret. If you are using the vCluster.Pro control-plane, -# this is optional. vCluster by default will lookout for a secret called vc-VCLUSTER_NAME-license and if found use that. -# You can also use a secret within another namespace by using the format NAMESPACE/NAME. If the secret is in another -# namespace, please enable clusterRole.create and define an extra clusterRole.extraRules to allow vCluster to retrieve -# secrets within the cluster. -proLicenseSecret: "" - -# Embedded etcd settings -embeddedEtcd: - # If embedded etcd should be enabled, this is a PRO only feature - enabled: false - # To use if embeddedEtcd is enabled and you want to migrate the data - # from the external etcd - migrateFromEtcd: false - -# Extra Annotations for the stateful set -annotations: {} -podAnnotations: {} - -headless: false - -monitoring: - serviceMonitor: - enabled: false - -# Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml' -plugin: {} -# Manually configure a plugin called test -# test: -# image: ... -# env: ... -# rbac: -# clusterRole: -# extraRules: ... -# role: -# extraRules: ... - -# Resource syncers that should be enabled/disabled. -# Enabling syncers will impact RBAC Role and ClusterRole permissions. -# To disable a syncer set "enabled: false". -# See docs for details - https://www.vcluster.com/docs/architecture/synced-resources -sync: - services: - enabled: true - configmaps: - enabled: true - all: false - secrets: - enabled: true - all: false - endpoints: - enabled: true - pods: - enabled: true - ephemeralContainers: false - status: false - events: - enabled: true - persistentvolumeclaims: - enabled: true - ingresses: - enabled: false - ingressclasses: {} - # By default IngressClasses sync is enabled when the Ingress sync is enabled - # but it can be explicitly disabled by setting: - # enabled: false - fake-nodes: - enabled: true # will be ignored if nodes.enabled = true - fake-persistentvolumes: - enabled: true # will be ignored if persistentvolumes.enabled = true - nodes: - fakeKubeletIPs: true - enabled: false - # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster - # will sync all nodes instead of only the ones where some pods are running. - syncAllNodes: false - # nodeSelector is used to limit which nodes get synced to the vcluster, - # and which nodes are used to run vcluster pods. - # A valid string representation of a label selector must be used. - nodeSelector: "" - # syncNodeChanges allows vcluster user edits of the nodes to be synced down to the host nodes. - # Write permissions on node resource will be given to the vcluster. - syncNodeChanges: false - persistentvolumes: - enabled: false - storageclasses: - enabled: false - # formerly named - "legacy-storageclasses" - hoststorageclasses: - enabled: false - priorityclasses: - enabled: false - networkpolicies: - enabled: false - volumesnapshots: - enabled: false - poddisruptionbudgets: - enabled: false - serviceaccounts: - enabled: false - # generic CRD configuration - generic: - config: |- - --- - -# If enabled, will fallback to host dns for resolving domains. This -# is useful if using istio or dapr in the host cluster and sidecar -# containers cannot connect to the central instance. Its also useful -# if you want to access host cluster services from within the vcluster. -fallbackHostDns: false - -# Map Services between host and virtual cluster -mapServices: - # Services that should get mapped from the - # virtual cluster to the host cluster. - # vcluster will make sure to sync the service - # ip to the host cluster automatically as soon - # as the service exists. - # For example: - # fromVirtual: - # - from: my-namespace/name - # to: host-service - fromVirtual: [] - # Same as from virtual, but instead sync services - # from the host cluster into the virtual cluster. - # If the namespace does not exist, vcluster will - # also create the namespace for the service. - fromHost: [] - -proxy: - metricsServer: - nodes: - enabled: false - pods: - enabled: false - - -# Syncer configuration -syncer: - # Image to use for the syncer - # image: ghcr.io/loft-sh/vcluster - imagePullPolicy: "" - extraArgs: [] - volumeMounts: - - mountPath: /manifests/coredns - name: coredns - readOnly: true - extraVolumeMounts: [] - env: [] - livenessProbe: - enabled: true - readinessProbe: - enabled: true - resources: - limits: - ephemeral-storage: 8Gi - memory: 2Gi - requests: - ephemeral-storage: 200Mi - cpu: 10m - memory: 256Mi - # Extra volumes - volumes: [] - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used to schedule the syncer - nodeSelector: {} - # Affinity to apply to the syncer deployment - affinity: {} - # Tolerations to apply to the syncer deployment - tolerations: [] - # Extra Labels for the syncer deployment - labels: {} - # Extra Annotations for the syncer deployment - annotations: {} - podAnnotations: {} - podLabels: {} - priorityClassName: "" - kubeConfigContextName: "my-vcluster" - # Security context configuration - securityContext: {} - podSecurityContext: - allowPrivilegeEscalation: false - runAsUser: 0 - runAsGroup: 0 - serviceAnnotations: {} - # Storage settings for the vcluster - storage: - # If this is disabled, vcluster will use an emptyDir instead - # of a PersistentVolumeClaim - persistence: true - # Size of the persistent volume claim - size: 5Gi - # Optional StorageClass used for the pvc - # if empty default StorageClass defined in your host cluster will be used - #className: - -# Etcd settings -etcd: - image: public.ecr.aws/eks-distro/etcd-io/etcd:v3.5.9-eks-1-28-6 - imagePullPolicy: "" - # The amount of replicas to run - replicas: 1 - # NodeSelector used - nodeSelector: {} - # Affinity to apply - affinity: {} - # Tolerations to apply - tolerations: [] - # Extra Labels - labels: {} - # Extra Annotations - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 20m - memory: 150Mi - # Storage settings for the etcd - storage: - # If this is disabled, vcluster will use an emptyDir instead - # of a PersistentVolumeClaim - persistence: true - # Size of the persistent volume claim - size: 5Gi - # Optional StorageClass used for the pvc - # if empty default StorageClass defined in your host cluster will be used - #className: - priorityClassName: "" - securityContext: {} - serviceAnnotations: {} - autoDeletePersistentVolumeClaims: false - -# Kubernetes Controller Manager settings -controller: - image: public.ecr.aws/eks-distro/kubernetes/kube-controller-manager:v1.28.2-eks-1-28-6 - imagePullPolicy: "" - -# Kubernetes API Server settings -api: - image: public.ecr.aws/eks-distro/kubernetes/kube-apiserver:v1.28.2-eks-1-28-6 - imagePullPolicy: "" - extraArgs: [] - -# Core DNS settings -coredns: - # If CoreDns is enabled - enabled: true - # Pro only feature - integrated: false - # only used if isolation is enabled - fallback: 8.8.8.8 - plugin: - enabled: false - config: [] - # example configuration for plugin syntax, will be documented in detail - # - record: - # fqdn: google.com - # target: - # mode: url - # url: google.co.in - # - record: - # service: my-namespace/my-svc # dns-test/nginx-svc - # target: - # mode: host - # service: dns-test/nginx-svc - # - record: - # service: my-namespace-lb/my-svc-lb - # target: - # mode: host - # service: dns-test-exposed-lb/nginx-svc-exposed-lb - # - record: - # service: my-ns-external-name/my-svc-external-name - # target: - # mode: host - # service: dns-test-external-name/nginx-svc-external-name - # - record: - # service: my-ns-in-vcluster/my-svc-vcluster - # target: - # mode: vcluster # can be tested only manually for now - # vcluster: test-vcluster-ns/test-vcluster - # service: dns-test-in-vcluster-ns/test-in-vcluster-service - # - record: - # service: my-ns-in-vcluster-mns/my-svc-mns - # target: - # mode: vcluster # can be tested only manually for now - # service: dns-test-in-vcluster-mns/test-in-vcluster-svc-mns - # vcluster: test-vcluster-ns-mns/test-vcluster-mns - # - record: - # service: my-self-vc-ns/my-self-vc-svc - # target: - # mode: self - # service: dns-test/nginx-svc - image: public.ecr.aws/eks-distro/coredns/coredns:v1.10.1-eks-1-28-6 - replicas: 1 - # The nodeSelector example below specifices that coredns should only be scheduled to nodes with the arm64 label - # nodeSelector: - # kubernetes.io/arch: arm64 - # CoreDNS service configurations - service: - # Extra Annotations - annotations: {} - resources: - limits: - cpu: 1000m - memory: 512Mi - requests: - # ensure that cpu/memory requests are high enough. - # for example gke wants minimum 10m/32Mi here! - cpu: 20m - memory: 64Mi -# if below option is configured, it will override the coredns manifests with the following string -# manifests: |- -# apiVersion: ... -# ... - podAnnotations: {} - podLabels: {} - -# Service account that should be used by the vcluster -serviceAccount: - create: true - # Optional name of the service account to use - # name: default - # Optional pull secrets - # imagePullSecrets: - # - name: my-pull-secret - -# Service account that should be used by the pods synced by vcluster -workloadServiceAccount: - # This is not supported in multi-namespace mode - annotations: {} - -# Roles & ClusterRoles for the vcluster -rbac: - clusterRole: - # You don't need to toggle this as necessary cluster roles are created based on the enabled syncers (.sync.*.enabled). - # This only makes sense to enable if you want to use the extraRules below. - create: false - # Extra Rules for the cluster role - extraRules: [] - role: - # Disable this only if you don't want vCluster to create a role. This will break most functionality if disabled. - create: true - # Extra Rules for the role - extraRules: [] - # all entries in excludedApiResources will be excluded from the Role created for vcluster - excludedApiResources: - # - pods/exec - -# Syncer service configurations -service: - type: ClusterIP - - # Optional configuration - # A list of IP addresses for which nodes in the cluster will also accept traffic for this service. - # These IPs are not managed by Kubernetes; e.g., an external load balancer. - externalIPs: [] - - # Optional configuration for LoadBalancer & NodePort service types - # Route external traffic to node-local or cluster-wide endpoints [ Local | Cluster ] - externalTrafficPolicy: "" - - # Optional configuration for LoadBalancer service type - # Specify IP of load balancer to be created - loadBalancerIP: "" - # CIDR block(s) for the service allowlist - loadBalancerSourceRanges: [] - # Set the loadBalancerClass if using an external load balancer controller - loadBalancerClass: "" - # Set loadBalancer specific annotations on the Kubernetes service - loadBalancerAnnotations: {} - -# Configure the ingress resource that allows you to access the vcluster -ingress: - # Enable ingress record generation - enabled: false - # Ingress path type - pathType: ImplementationSpecific - ingressClassName: "" - host: vcluster.local - annotations: - nginx.ingress.kubernetes.io/backend-protocol: HTTPS - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - # Ingress TLS configuration - tls: [] - # - secretName: tls-vcluster.local - # hosts: - # - vcluster.local - -# Set "enable" to true when running vcluster in an OpenShift host -# This will add an extra rule to the deployed role binding in order -# to manage service endpoints -openshift: - enable: false - -# manifests to setup when initializing a vcluster -init: - manifests: |- - --- - # The contents of manifests-template will be templated using helm - # this allows you to use helm values inside, e.g.: {{ .Release.Name }} - manifestsTemplate: '' - helm: [] - # - bundle: - base64-encoded .tar.gz file content (optional - overrides chart.repo) - # chart: - # name: REQUIRED - # version: REQUIRED - # repo: (optional when bundle is used) - # username: (if required for repo) - # password: (if required for repo) - # insecure: boolean (if required for repo) - # release: - # name: REQUIRED - # namespace: REQUIRED - # timeout: number - # values: |- string YAML object - # foo: bar - # valuesTemplate: |- string YAML object - # foo: {{ .Release.Name }} - -# If enabled will deploy vcluster in an isolated mode with pod security -# standards, limit ranges and resource quotas -isolation: - enabled: false - namespace: null - - podSecurityStandard: baseline - - # If enabled will add node/proxy permission to the cluster role - # in isolation mode - nodeProxyPermission: - enabled: false - - resourceQuota: - enabled: true - quota: - requests.cpu: 10 - requests.memory: 20Gi - requests.storage: "100Gi" - requests.ephemeral-storage: 60Gi - limits.cpu: 20 - limits.memory: 40Gi - limits.ephemeral-storage: 160Gi - services.nodeports: 0 - services.loadbalancers: 1 - count/endpoints: 40 - count/pods: 20 - count/services: 20 - count/secrets: 100 - count/configmaps: 100 - count/persistentvolumeclaims: 20 - scopeSelector: - matchExpressions: - scopes: - - limitRange: - enabled: true - default: - ephemeral-storage: 8Gi - memory: 512Mi - cpu: "1" - defaultRequest: - ephemeral-storage: 3Gi - memory: 128Mi - cpu: 100m - - networkPolicy: - enabled: true - outgoingConnections: - ipBlock: - cidr: 0.0.0.0/0 - except: - - 100.64.0.0/10 - - 127.0.0.0/8 - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - -multiNamespaceMode: - enabled: false - -# list of {validating/mutating}webhooks that the syncer should proxy. -# This is a PRO only feature. -centralAdmission: - validatingWebhooks: [] - mutatingWebhooks: [] - -telemetry: - disabled: false - instanceCreator: "helm" - platformUserID: "" - platformInstanceID: "" - machineID: "" diff --git a/charts/k0s/.helmignore b/charts/k0s/.helmignore deleted file mode 100644 index b6a3eb5b3..000000000 --- a/charts/k0s/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store - -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ - -# Common backup files -*.swp -*.bak -*.tmp -*~ - -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/k0s/Chart.yaml b/charts/k0s/Chart.yaml deleted file mode 100644 index 228652bb5..000000000 --- a/charts/k0s/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v2 -name: vcluster-k0s -description: vcluster - Virtual Kubernetes Clusters (k0s) -home: https://vcluster.com -icon: https://static.loft.sh/branding/logos/vcluster/vertical/vcluster_vertical.svg -keywords: - - developer - - development - - sharing - - share - - multi-tenancy - - tenancy - - cluster - - space - - namespace - - vcluster - - vclusters -maintainers: - - name: Loft Labs, Inc. - email: info@loft.sh - url: https://twitter.com/loft_sh -sources: - - https://github.com/loft-sh/vcluster -type: application - -version: 0.0.1 # version is auto-generated by release pipeline diff --git a/charts/k0s/README.md b/charts/k0s/README.md deleted file mode 100644 index ae0e7b0c0..000000000 --- a/charts/k0s/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -# vcluster (k0s) - -## **[GitHub](https://github.com/loft-sh/vcluster)** • **[Website](https://www.vcluster.com)** • **[Quickstart](https://www.vcluster.com/docs/getting-started/setup)** • **[Documentation](https://www.vcluster.com/docs/what-are-virtual-clusters)** • **[Blog](https://loft.sh/blog)** • **[Twitter](https://twitter.com/loft_sh)** • **[Slack](https://slack.loft.sh/)** - -Create fully functional virtual Kubernetes clusters - Each vcluster runs inside a namespace of the underlying k8s cluster. It's cheaper than creating separate full-blown clusters and it offers better multi-tenancy and isolation than regular namespaces. - -## Prerequisites - -- Kubernetes 1.18+ -- Helm 3+ - -## Get Helm Repository Info - -```bash -helm repo add loft-sh https://charts.loft.sh -helm repo update -``` - -See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation. - -## Install Helm Chart - -```bash -helm upgrade [RELEASE_NAME] loft-sh/vcluster-k0s -n [RELEASE_NAMESPACE] --create-namespace --install -``` - -See [vcluster docs](https://vcluster.com/docs) for configuration options. - -See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation. - -## Connect to the vcluster - -In order to connect to the installed vcluster, please install [vcluster cli](https://www.vcluster.com/docs/getting-started/setup) and run: - -```bash -vcluster connect [RELEASE_NAME] -n [RELEASE_NAMESPACE] -``` - -## Uninstall Helm Chart - -```bash -helm uninstall [RELEASE_NAME] -``` - -This removes all the Kubernetes components associated with the chart and deletes the release. - -See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation. - -### Why Virtual Kubernetes Clusters? - -- **Cluster Scoped Resources**: much more powerful than simple namespaces (virtual clusters allow users to use CRDs, namespaces, cluster roles etc.) -- **Ease of Use**: usable in any Kubernetes cluster and created in seconds either via a single command or [cluster-api](https://github.com/loft-sh/cluster-api-provider-vcluster) -- **Cost Efficient**: much cheaper and efficient than "real" clusters (single pod and shared resources just like for namespaces) -- **Lightweight**: built upon the ultra-fast k3s distribution with minimal overhead per virtual cluster (other distributions work as well) -- **Strict isolation**: complete separate Kubernetes control plane and access point for each vcluster while still being able to share certain services of the underlying host cluster -- **Cluster Wide Permissions**: allow users to install apps which require cluster-wide permissions while being limited to actually just one namespace within the host cluster -- **Great for Testing**: allow you to test different Kubernetes versions inside a single host cluster which may have a different version than the virtual clusters - -Learn more on [www.vcluster.com](https://vcluster.com). - -![vcluster Intro](https://github.com/loft-sh/vcluster/raw/main/docs/static/media/vcluster-comparison.png) - -Learn more in the [documentation](https://vcluster.com/docs/what-are-virtual-clusters). diff --git a/charts/k0s/templates/NOTES.txt b/charts/k0s/templates/NOTES.txt deleted file mode 100644 index 6cf960d62..000000000 --- a/charts/k0s/templates/NOTES.txt +++ /dev/null @@ -1,10 +0,0 @@ -Thank you for installing vcluster. - -Your vcluster is named {{ .Release.Name }} in namespace {{ .Release.Namespace }}. - -To connect to the vcluster, use vcluster CLI (https://www.vcluster.com/docs/getting-started/setup): - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} -- kubectl get ns - - -For more information, please take a look at the vcluster docs at https://www.vcluster.com/docs diff --git a/charts/k0s/templates/_coredns.tpl b/charts/k0s/templates/_coredns.tpl deleted file mode 100644 index c41f016f5..000000000 --- a/charts/k0s/templates/_coredns.tpl +++ /dev/null @@ -1,52 +0,0 @@ -{{/* - Define a common coredns config -*/}} -{{- define "vcluster.corefile" -}} -Corefile: |- - {{- if .Values.coredns.config }} -{{ .Values.coredns.config | indent 8 }} - {{- else }} - .:1053 { - errors - health - ready - rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local - kubernetes cluster.local in-addr.arpa ip6.arpa { - {{- if .Values.pro }} - {{- if .Values.coredns.integrated }} - kubeconfig /data/k0s/pki/admin.conf - {{- end }} - {{- end }} - pods insecure - {{- if .Values.fallbackHostDns }} - fallthrough cluster.local in-addr.arpa ip6.arpa - {{- else }} - fallthrough in-addr.arpa ip6.arpa - {{- end }} - } - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - vcluster {{ toYaml .Values.coredns.plugin.config | b64enc }} - {{- end }} - hosts /etc/NodeHosts { - ttl 60 - reload 15s - fallthrough - } - prometheus :9153 - {{- if .Values.fallbackHostDns }} - forward . {{`{{.HOST_CLUSTER_DNS}}`}} - {{- else if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} - forward . /etc/resolv.conf {{ .Values.coredns.fallback }} { - policy sequential - } - {{- else }} - forward . /etc/resolv.conf - {{- end }} - cache 30 - loop - loadbalance - } - - import /etc/coredns/custom/*.server - {{- end }} -{{- end -}} diff --git a/charts/k0s/templates/_helpers.tpl b/charts/k0s/templates/_helpers.tpl deleted file mode 100644 index 50f08d6f9..000000000 --- a/charts/k0s/templates/_helpers.tpl +++ /dev/null @@ -1,196 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "vcluster.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Whether the ingressclasses syncer should be enabled -*/}} -{{- define "vcluster.syncIngressclassesEnabled" -}} -{{- if or - (.Values.sync.ingressclasses).enabled - (and - .Values.sync.ingresses.enabled - (not .Values.sync.ingressclasses)) -}} - {{- true -}} -{{- end -}} -{{- end -}} - -{{/* -Whether to create a cluster role or not -*/}} -{{- define "vcluster.createClusterRole" -}} -{{- if or - (not (empty (include "vcluster.serviceMapping.fromHost" . ))) - (not (empty (include "vcluster.plugin.clusterRoleExtraRules" . ))) - (not (empty (include "vcluster.generic.clusterRoleExtraRules" . ))) - .Values.rbac.clusterRole.create - .Values.sync.hoststorageclasses.enabled - (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") - (include "vcluster.syncIngressclassesEnabled" . ) - .Values.pro - .Values.sync.nodes.enabled - .Values.sync.persistentvolumes.enabled - .Values.sync.storageclasses.enabled - .Values.sync.priorityclasses.enabled - .Values.sync.volumesnapshots.enabled - .Values.proxy.metricsServer.nodes.enabled - .Values.multiNamespaceMode.enabled - .Values.coredns.plugin.enabled -}} -{{- true -}} -{{- end -}} -{{- end -}} - -{{- define "vcluster.clusterRoleName" -}} -{{- printf "vc-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "vcluster.clusterRoleNameMultinamespace" -}} -{{- printf "vc-mn-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Syncer flags for enabling/disabling controllers -Prints only the flags that modify the defaults: -- when default controller has enabled: false => `- "--sync=-controller` -- when non-default controller has enabled: true => `- "--sync=controller` -*/}} -{{- define "vcluster.syncer.syncArgs" -}} -{{- $defaultEnabled := list "services" "configmaps" "secrets" "endpoints" "pods" "events" "persistentvolumeclaims" "fake-nodes" "fake-persistentvolumes" -}} -{{- if and (hasKey .Values.sync.nodes "enableScheduler") .Values.sync.nodes.enableScheduler -}} - {{- $defaultEnabled = concat $defaultEnabled (list "csinodes" "csidrivers" "csistoragecapacities" ) -}} -{{- end -}} -{{- range $key, $val := .Values.sync }} -{{- if and (has $key $defaultEnabled) (not $val.enabled) }} -- --sync=-{{ $key }} -{{- else if and (not (has $key $defaultEnabled)) ($val.enabled)}} -{{- if eq $key "legacy-storageclasses" }} -- --sync=hoststorageclasses -{{- else }} -- --sync={{ $key }} -{{- end -}} -{{- end -}} -{{- end -}} -{{- if not (include "vcluster.syncIngressclassesEnabled" . ) }} -- --sync=-ingressclasses -{{- end -}} -{{- end -}} - -{{/* - Cluster role rules defined by plugins -*/}} -{{- define "vcluster.plugin.clusterRoleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.clusterRole }} -{{- if $container.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.clusterRoleExtraRules" -}} -{{- if .Values.sync.generic.clusterRole }} -{{- if .Values.sync.generic.clusterRole.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined on global level -*/}} -{{- define "vcluster.rbac.clusterRoleExtraRules" -}} -{{- if .Values.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - - -{{/* - Role rules defined on global level -*/}} -{{- define "vcluster.rbac.roleExtraRules" -}} -{{- if .Values.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined by plugins -*/}} -{{- define "vcluster.plugin.roleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.role }} -{{- if $container.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.roleExtraRules" -}} -{{- if .Values.sync.generic.role }} -{{- if .Values.sync.generic.role.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Virtual cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromVirtual" -}} -{{- range $key, $value := .Values.mapServices.fromVirtual }} -- '--map-virtual-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} - -{{/* - Host cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromHost" -}} -{{- range $key, $value := .Values.mapServices.fromHost }} -- '--map-host-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} diff --git a/charts/k0s/templates/_migrate.tpl b/charts/k0s/templates/_migrate.tpl deleted file mode 100644 index 34bef9a19..000000000 --- a/charts/k0s/templates/_migrate.tpl +++ /dev/null @@ -1,6 +0,0 @@ -{{/* - handles both replicas and syncer.replicas -*/}} -{{- define "vcluster.replicas" -}} -{{ if .Values.replicas }}{{ .Values.replicas }}{{ else }}{{ .Values.syncer.replicas }}{{ end }} -{{- end }} diff --git a/charts/k0s/templates/_plugin.tpl b/charts/k0s/templates/_plugin.tpl deleted file mode 100644 index b1ecd2fa2..000000000 --- a/charts/k0s/templates/_plugin.tpl +++ /dev/null @@ -1,188 +0,0 @@ -{{/* - Plugin config definition -*/}} -{{- define "vcluster.plugins.config" -}} -{{- $pluginFound := false -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} -{{- $pluginFound = true -}} -{{- end }} -{{- if $pluginFound }} -- name: PLUGIN_CONFIG - value: |- -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} - {{ $key }}: {{ toYaml $container.config | nindent 6 }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume mount definition -*/}} -{{- define "vcluster.plugins.volumeMounts" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- mountPath: /plugins - name: plugins -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume definition -*/}} -{{- define "vcluster.plugins.volumes" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- name: plugins - emptyDir: {} -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin init container definition -*/}} -{{- define "vcluster.plugins.initContainers" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if or $container.command $container.args }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- else }} - command: ["sh"] - args: ["-c", "cp -r /plugin /plugins/{{ $key }}"] - {{- end }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - {{- if $container.volumeMounts }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- else }} - volumeMounts: - - mountPath: /plugins - name: plugins - {{- end }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} -{{- end }} -{{- end -}} - -{{/* - Extra Syncer Args for the legacy Plugins -*/}} -{{- define "vcluster.legacyPlugins.args" -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{- continue }} -{{- end }} -{{- if not $container.optional }} -- --plugins={{ $key }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Sidecar container definition for the legacy syncer parts -*/}} -{{- define "vcluster.legacyPlugins.containers" -}} -{{- $counter := -1 -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{ continue }} -{{- end }} -{{- $counter = add1 $counter }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if $container.workingDir }} - workingDir: {{ $container.workingDir }} - {{- end }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- if $container.terminationMessagePath }} - terminationMessagePath: {{ $container.terminationMessagePath }} - {{- end }} - {{- if $container.terminationMessagePolicy }} - terminationMessagePolicy: {{ $container.terminationMessagePolicy }} - {{- end }} - env: - - name: VCLUSTER_PLUGIN_ADDRESS - value: "localhost:{{ add 14000 $counter }}" - - name: VCLUSTER_PLUGIN_NAME - value: "{{ $key }}" - {{- if $container.env }} -{{ toYaml $container.env | indent 4 }} - {{- end }} - envFrom: -{{ toYaml $container.envFrom | indent 4 }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - lifecycle: -{{ toYaml $container.lifecycle | indent 4 }} - livenessProbe: -{{ toYaml $container.livenessProbe | indent 4 }} - readinessProbe: -{{ toYaml $container.readinessProbe | indent 4 }} - startupProbe: -{{ toYaml $container.startupProbe | indent 4 }} - volumeDevices: -{{ toYaml $container.volumeDevices | indent 4 }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} - {{- end }} -{{- end }} - - diff --git a/charts/k0s/templates/_storage.tpl b/charts/k0s/templates/_storage.tpl deleted file mode 100644 index b10675ab6..000000000 --- a/charts/k0s/templates/_storage.tpl +++ /dev/null @@ -1,20 +0,0 @@ -{{/* - storage size -*/}} -{{- define "vcluster.storage.size" -}} -{{if .Values.storage }}{{ .Values.storage.size }}{{ else }}{{ .Values.syncer.storage.size }}{{ end }} -{{- end -}} - -{{/* - storage persistence -*/}} -{{- define "vcluster.storage.persistence" -}} -{{if .Values.storage }}{{ .Values.storage.persistence }}{{ else }}{{ .Values.syncer.storage.persistence }}{{ end }} -{{- end -}} - -{{/* - storage classname -*/}} -{{- define "vcluster.storage.className" -}} -{{if .Values.storage }}{{ .Values.storage.className }}{{ else }}{{ .Values.syncer.storage.className }}{{ end }} -{{- end -}} diff --git a/charts/k0s/templates/coredns.yaml b/charts/k0s/templates/coredns.yaml deleted file mode 100644 index cc25dd17f..000000000 --- a/charts/k0s/templates/coredns.yaml +++ /dev/null @@ -1,231 +0,0 @@ -{{- if not .Values.headless }} -{{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-coredns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{- if .Values.coredns.manifests }} - coredns.yaml: |- -{{ .Values.coredns.manifests | indent 4 }} -{{- else }} - coredns.yaml: |- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: coredns - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns - subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: -{{ include "vcluster.corefile" . | indent 6 }} - NodeHosts: "" - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/name: "CoreDNS" - spec: - replicas: {{ .Values.coredns.replicas }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - {{- if .Values.coredns.podAnnotations }} - annotations: -{{ toYaml .Values.coredns.podAnnotations | indent 12 }} - {{- end }} - labels: - k8s-app: kube-dns - {{- range $k, $v := .Values.coredns.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - priorityClassName: "system-cluster-critical" - serviceAccountName: coredns - nodeSelector: - kubernetes.io/os: linux - {{- if .Values.coredns.nodeSelector }} -{{ toYaml .Values.coredns.nodeSelector | indent 12 }} - {{- end }} - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - k8s-app: kube-dns - {{- if .Values.isolation.enabled }} - securityContext: - seccompProfile: - type: RuntimeDefault - {{- end }} - containers: - - name: coredns - {{- if .Values.coredns.image }} - image: {{ .Values.defaultImageRegistry }}{{ .Values.coredns.image }} - {{- else }} - image: {{`{{.IMAGE}}`}} - {{- end }} - imagePullPolicy: IfNotPresent - resources: -{{ toYaml .Values.coredns.resources | indent 16}} - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - - name: custom-config-volume - mountPath: /etc/coredns/custom - readOnly: true - securityContext: - runAsNonRoot: true - runAsUser: {{`{{.RUN_AS_USER}}`}} - runAsGroup: {{`{{.RUN_AS_GROUP}}`}} - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 2 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile - - key: NodeHosts - path: NodeHosts - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - selector: - k8s-app: kube-dns - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/k0s/templates/ingress.yaml b/charts/k0s/templates/ingress.yaml deleted file mode 100644 index d629f096f..000000000 --- a/charts/k0s/templates/ingress.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - {{- $annotations := merge .Values.ingress.annotations .Values.globalAnnotations }} - {{- if .Values.ingress.tls }} - {{- $annotations = omit $annotations "nginx.ingress.kubernetes.io/ssl-passthrough" }} - {{- end }} - {{- if $annotations }} - annotations: - {{- toYaml $annotations | nindent 4 }} - {{- end }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - {{- if .Values.ingress.ingressClassName }} - ingressClassName: {{ .Values.ingress.ingressClassName | quote }} - {{- end }} - rules: - - host: {{ .Values.ingress.host | quote }} - http: - paths: - - backend: - service: - name: {{ .Release.Name }} - port: - name: https - path: / - pathType: {{ .Values.ingress.pathType }} - {{- with .Values.ingress.tls }} - tls: - {{- toYaml . | nindent 4 }} - {{- end -}} -{{- end }} diff --git a/charts/k0s/templates/init-configmap.yaml b/charts/k0s/templates/init-configmap.yaml deleted file mode 100644 index 34498fd14..000000000 --- a/charts/k0s/templates/init-configmap.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- if .Values.init.manifests }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-init-manifests - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: - manifests: |- - {{ .Values.init.manifests | nindent 4 | trim }} - {{ tpl .Values.init.manifestsTemplate $ | nindent 4 | trim }} - {{- if .Values.init.helm }} - charts: |- - {{- range .Values.init.helm }} - {{- /* only render this chart entry if either of chart or bundle is defined */}} - {{- if .chart }} - - name: {{ .chart.name }} - repo: {{ .chart.repo }} - version: {{ .chart.version }} - {{- if .chart.username }} - username: {{ .chart.username }} - {{- end }} - {{- if .chart.password }} - password: {{ .chart.password }} - {{- end }} - {{- if .insecure }} - insecure: true - {{- end}} - {{- end }} - {{- if .bundle }} - bundle: {{ .bundle }} - {{- end }} - {{- if or .chart .bundle }} - {{- if or .values .valuesTemplate }} - values: |- - {{ (.values | default "") | nindent 8 | trim }} - {{ tpl (.valuesTemplate | default "") $ | nindent 8 | trim }} - {{- end}} - {{- if .release }} - timeout: {{ .timeout | default "120s" | quote }} - releaseName: {{ .release.name }} - releaseNamespace: {{ .release.namespace }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/k0s/templates/integrated-coredns.yaml b/charts/k0s/templates/integrated-coredns.yaml deleted file mode 100644 index 16907cba0..000000000 --- a/charts/k0s/templates/integrated-coredns.yaml +++ /dev/null @@ -1,64 +0,0 @@ -{{- if .Values.pro }} -{{- if .Values.coredns.integrated }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-dns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{ include "vcluster.corefile" . | indent 2 }} - coredns.yaml: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: - NodeHosts: "" - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} diff --git a/charts/k0s/templates/limitrange.yaml b/charts/k0s/templates/limitrange.yaml deleted file mode 100644 index ed219d063..000000000 --- a/charts/k0s/templates/limitrange.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.limitRange.enabled }} -apiVersion: v1 -kind: LimitRange -metadata: - name: {{ .Release.Name }}-limit-range - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - limits: - - default: - {{- range $key, $val := .Values.isolation.limitRange.default }} - {{ $key }}: {{ $val | quote }} - {{- end }} - defaultRequest: - {{- range $key, $val := .Values.isolation.limitRange.defaultRequest }} - {{ $key }}: {{ $val | quote }} - {{- end }} - type: Container -{{- end }} diff --git a/charts/k0s/templates/networkpolicy.yaml b/charts/k0s/templates/networkpolicy.yaml deleted file mode 100644 index c932065e0..000000000 --- a/charts/k0s/templates/networkpolicy.yaml +++ /dev/null @@ -1,70 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-workloads - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} -spec: - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - egress: - # Allows outgoing connections to the vcluster control plane - - ports: - - port: 443 - - port: 8443 - to: - - podSelector: - matchLabels: - release: {{ .Release.Name }} - # Allows outgoing connections to DNS server - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Allows outgoing connections to the internet or - # other vcluster workloads - - to: - - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - - ipBlock: - cidr: {{ .Values.isolation.networkPolicy.outgoingConnections.ipBlock.cidr }} - except: - {{- range .Values.isolation.networkPolicy.outgoingConnections.ipBlock.except }} - - {{ . }} - {{- end }} - policyTypes: - - Egress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-control-plane - namespace: {{ .Release.Namespace }} -spec: - podSelector: - matchLabels: - release: {{ .Release.Name }} - egress: - # Allows outgoing connections to all pods with - # port 443, 8443 or 6443. This is needed for host Kubernetes - # access - - ports: - - port: 443 - - port: 8443 - - port: 6443 - # Allows outgoing connections to all vcluster workloads - # or kube system dns server - - to: - - podSelector: {} - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: 'kube-system' - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Egress -{{- end }} \ No newline at end of file diff --git a/charts/k0s/templates/rbac/clusterrole.yaml b/charts/k0s/templates/rbac/clusterrole.yaml deleted file mode 100644 index 984267cae..000000000 --- a/charts/k0s/templates/rbac/clusterrole.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: -{{- if .Values.pro }} - - apiGroups: ["cluster.loft.sh", "storage.loft.sh"] - resources: ["features", "virtualclusters"] - verbs: ["get", "list", "watch"] -{{- end }} - {{- if or .Values.pro .Values.sync.nodes.enabled }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: [ "pods", "nodes/metrics", "nodes/stats"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.coredns.plugin.enabled }} - - apiGroups: [""] - resources: [ "pods"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled (or (not .Values.isolation.enabled) (and .Values.isolation.nodeProxyPermission.enabled .Values.isolation.enabled)) }} - - apiGroups: [""] - resources: ["nodes/proxy"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled .Values.sync.nodes.syncNodeChanges }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["update", "patch"] - {{- end }} - {{- if .Values.sync.persistentvolumes.enabled }} - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if .Values.sync.nodes.enableScheduler }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses","csinodes","csidrivers","csistoragecapacities"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if (include "vcluster.syncIngressclassesEnabled" . ) }} - - apiGroups: ["networking.k8s.io"] - resources: ["ingressclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.storageclasses.enabled }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if or .Values.sync.hoststorageclasses.enabled (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") .Values.rbac.clusterRole.create }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.priorityclasses.enabled }} - - apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if (not (empty (include "vcluster.serviceMapping.fromHost" . ))) }} - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.proxy.metricsServer.nodes.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["nodes"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.clusterRoleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/k0s/templates/rbac/clusterrolebinding.yaml b/charts/k0s/templates/rbac/clusterrolebinding.yaml deleted file mode 100644 index 0a5645d28..000000000 --- a/charts/k0s/templates/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ template "vcluster.clusterRoleName" . }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/k0s/templates/rbac/role.yaml b/charts/k0s/templates/rbac/role.yaml deleted file mode 100644 index de05bbb62..000000000 --- a/charts/k0s/templates/rbac/role.yaml +++ /dev/null @@ -1,104 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRole -{{- else -}} -kind: Role -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} -{{- end }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: - {{- if .Values.pro }} - - apiGroups: [""] - {{- $resources := list "configmaps" "secrets" "services" "pods" "pods/attach" "pods/portforward" "pods/exec" "persistentvolumeclaims" }} - {{- range $excluded := .Values.rbac.role.excludedApiResources }} - {{- $resources = without $resources $excluded }} - {{- end}} - resources: {{ $resources | toJson }} - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- else }} - - apiGroups: [""] - resources: ["configmaps", "secrets", "services", "pods", "pods/attach", "pods/portforward", "pods/exec", "persistentvolumeclaims"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.status }} - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.ephemeralContainers }} - - apiGroups: [""] - resources: ["pods/ephemeralcontainers"] - verbs: ["patch", "update"] - {{- end }} - {{- if or .Values.sync.endpoints.enabled .Values.headless }} - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["create", "delete", "patch", "update"] - {{- end }} - {{- if gt (int .Values.syncer.replicas) 1 }} - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: [""] - resources: ["endpoints", "events", "pods/log"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.ingresses.enabled}} - - apiGroups: ["networking.k8s.io"] - resources: ["ingresses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "deployments"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.networkpolicies.enabled }} - - apiGroups: ["networking.k8s.io"] - resources: ["networkpolicies"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.serviceaccounts.enabled }} - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.poddisruptionbudgets.enabled }} - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.openshift.enable }} - {{- if .Values.sync.endpoints.enabled }} - - apiGroups: [""] - resources: ["endpoints/restricted"] - verbs: ["create"] - {{- end }} - {{- end }} - {{- if .Values.proxy.metricsServer.pods.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["pods"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.roleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.roleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.roleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/k0s/templates/rbac/rolebinding.yaml b/charts/k0s/templates/rbac/rolebinding.yaml deleted file mode 100644 index 676e5002a..000000000 --- a/charts/k0s/templates/rbac/rolebinding.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRoleBinding -{{- else -}} -kind: RoleBinding -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -{{- end }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: -{{- if .Values.multiNamespaceMode.enabled }} - kind: ClusterRole - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - kind: Role - name: {{ .Release.Name }} -{{- end }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/k0s/templates/resourcequota.yaml b/charts/k0s/templates/resourcequota.yaml deleted file mode 100644 index b19d89fff..000000000 --- a/charts/k0s/templates/resourcequota.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.resourceQuota.enabled }} -apiVersion: v1 -kind: ResourceQuota -metadata: - name: {{ .Release.Name }}-quota - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - hard: - {{- range $key, $val := .Values.isolation.resourceQuota.quota }} - {{ $key }}: {{ $val | quote }} - {{- end }} - - {{- if .Values.isolation.resourceQuota.scopeSelector.matchExpressions }} - scopeSelector: - matchExpressions: - {{- toYaml .Values.isolation.resourceQuota.scopeSelector.matchExpressions | nindent 4 }} - {{- end}} - - {{- if .Values.isolation.resourceQuota.scopes }} - scopes: - {{- toYaml .Values.isolation.resourceQuota.scopes | nindent 4 }} - {{- end}} - -{{- end }} diff --git a/charts/k0s/templates/secret.yaml b/charts/k0s/templates/secret.yaml deleted file mode 100644 index 2b55f5caa..000000000 --- a/charts/k0s/templates/secret.yaml +++ /dev/null @@ -1,65 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: v1 -kind: Secret -metadata: - name: vc-{{ .Release.Name }}-config - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations}} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -type: Opaque -stringData: - {{- if .Values.config }} - config.yaml: {{ toJson .Values.config }} - {{- else }} - config.yaml: |- - apiVersion: k0s.k0sproject.io/v1beta1 - kind: Cluster - metadata: - name: k0s - spec: - api: - port: 6443 - k0sApiPort: 9443 - extraArgs: - bind-address: 127.0.0.1 - enable-admission-plugins: NodeRestriction - endpoint-reconciler-type: none - network: - {{- if .Values.serviceCIDR }} - serviceCIDR: {{ .Values.serviceCIDR }} - {{- else }} - # Will be replaced automatically by the syncer container on first startup - serviceCIDR: CIDR_PLACEHOLDER - {{- end }} - provider: custom - {{- if .Values.vcluster.clusterDomain }} - clusterDomain: {{ .Values.vcluster.clusterDomain }} - {{- end}} - controllerManager: - extraArgs: - {{- if not .Values.sync.nodes.enableScheduler }} - controllers: '*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' - {{- else }} - controllers: '*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' - node-monitor-grace-period: 1h - node-monitor-period: 1h - {{- end }} - {{- if .Values.embeddedEtcd.enabled }} - storage: - etcd: - externalCluster: - endpoints: ["127.0.0.1:2379"] - caFile: /data/k0s/pki/etcd/ca.crt - etcdPrefix: "/registry" - clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt - clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key - {{- end }} - {{- end }} - {{- end }} diff --git a/charts/k0s/templates/service-monitor.yaml b/charts/k0s/templates/service-monitor.yaml deleted file mode 100644 index b6917ae27..000000000 --- a/charts/k0s/templates/service-monitor.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.monitoring.serviceMonitor.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: vcluster - release: "{{ .Release.Name }}" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: "{{ .Release.Service }}" - endpoints: - - interval: 30s - port: https - path: /metrics - scheme: https - tlsConfig: - ca: - secret: - name: vc-{{ .Release.Name }} - key: certificate-authority - cert: - secret: - name: vc-{{ .Release.Name }} - key: client-certificate - keySecret: - name: vc-{{ .Release.Name }} - key: client-key - serverName: 127.0.0.1 -{{- end }} diff --git a/charts/k0s/templates/service.yaml b/charts/k0s/templates/service.yaml deleted file mode 100644 index b575a9fe1..000000000 --- a/charts/k0s/templates/service.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: {{ if eq .Values.service.type "LoadBalancer" -}}ClusterIP{{- else }} {{- .Values.service.type }} {{- end }} - ports: - - name: https - port: 443 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.httpsNodePort }} - protocol: TCP - - name: kubelet - port: 10250 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.kubeletNodePort }} - protocol: TCP - {{- if .Values.service.externalIPs }} - externalIPs: - {{- range $f := .Values.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- if eq .Values.service.type "NodePort" }} - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} ---- -{{ if eq .Values.service.type "LoadBalancer" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-lb - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.service.loadBalancerAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: LoadBalancer - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.service.loadBalancerClass }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range $f := .Values.service.loadBalancerSourceRanges }} - - "{{ $f }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/k0s/templates/serviceaccount.yaml b/charts/k0s/templates/serviceaccount.yaml deleted file mode 100644 index 504998037..000000000 --- a/charts/k0s/templates/serviceaccount.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} -{{- end }} diff --git a/charts/k0s/templates/statefulset-service.yaml b/charts/k0s/templates/statefulset-service.yaml deleted file mode 100644 index 7523bf36e..000000000 --- a/charts/k0s/templates/statefulset-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-headless - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "vcluster.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - publishNotReadyAddresses: true - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.embeddedEtcd.enabled }} - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - {{- end }} - clusterIP: None - selector: - app: vcluster - release: "{{ .Release.Name }}" -{{- end }} diff --git a/charts/k0s/templates/syncer.yaml b/charts/k0s/templates/syncer.yaml deleted file mode 100644 index 965c975c9..000000000 --- a/charts/k0s/templates/syncer.yaml +++ /dev/null @@ -1,348 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.labels }} -{{ toYaml .Values.labels | indent 4 }} -{{- end }} -{{- $annotations := merge .Values.annotations .Values.globalAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - serviceName: {{ .Release.Name }}-headless - {{- if .Values.autoDeletePersistentVolumeClaims }} - {{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} - persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete - {{- end }} - {{- end }} - replicas: {{ include "vcluster.replicas" . }} - selector: - matchLabels: - app: vcluster - release: {{ .Release.Name }} - {{- if (hasKey .Values "volumeClaimTemplates") }} - volumeClaimTemplates: -{{ toYaml .Values.volumeClaimTemplates | indent 4 }} - {{- else if ( eq ( include "vcluster.storage.persistence" . ) "true" ) }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if (include "vcluster.storage.className" . ) }} - storageClassName: {{ include "vcluster.storage.className" . }} - {{- end }} - resources: - requests: - storage: {{ include "vcluster.storage.size" . }} - {{- end }} - template: - metadata: - {{- if .Values.podAnnotations }} - annotations: -{{ toYaml .Values.podAnnotations | indent 8 }} - {{- end }} - labels: - app: vcluster - release: {{ .Release.Name }} - {{- range $k, $v := .Values.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - terminationGracePeriodSeconds: 10 - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} - affinity: -{{ toYaml .Values.affinity | indent 8 }} - tolerations: -{{ toYaml .Values.tolerations | indent 8 }} - {{- if .Values.serviceAccount.name }} - serviceAccountName: {{ .Values.serviceAccount.name }} - {{- else }} - serviceAccountName: vc-{{ .Release.Name }} - {{- end }} - volumes: - {{- include "vcluster.plugins.volumes" . | indent 8 }} - - name: helm-cache - emptyDir: {} - - name: tmp - emptyDir: {} - - name: run-k0s - emptyDir: {} - - name: binaries - emptyDir: {} - {{- if .Values.syncer.volumes }} -{{ toYaml .Values.syncer.volumes | indent 8 }} - {{- end }} - {{- if .Values.volumes }} -{{ toYaml .Values.volumes | indent 8 }} - {{- end }} - {{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} - - name: coredns - configMap: - name: {{ .Release.Name }}-coredns - {{- else if .Values.coredns.integrated }} - - name: coredns - configMap: - name: {{ .Release.Name }}-dns - {{- end }} - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - {{- if not (eq ( include "vcluster.storage.persistence" . ) "true" ) }} - - name: data - emptyDir: {} - {{- end }} - {{- if .Values.priorityClassName }} - priorityClassName: {{ .Values.priorityClassName }} - {{- end }} - {{- if or .Values.podSecurityContext .Values.fsGroup }} - {{- $fsContext := dict "fsGroup" .Values.fsGroup }} - {{- with merge .Values.podSecurityContext $fsContext }} - securityContext: -{{ toYaml . | indent 8 }} - {{- end }} - {{- end }} - initContainers: - {{- include "vcluster.plugins.initContainers" . | indent 6 }} - - image: {{ .Values.defaultImageRegistry }}{{ .Values.vcluster.image }} - name: vcluster - command: - - /bin/sh - args: - - -c - - "cp /usr/local/bin/k0s /binaries/k0s" - {{- if .Values.vcluster.imagePullPolicy }} - imagePullPolicy: {{ .Values.vcluster.imagePullPolicy }} - {{- end }} - securityContext: -{{ toYaml .Values.securityContext | indent 10 }} - volumeMounts: - - name: binaries - mountPath: /binaries - resources: -{{ toYaml .Values.vcluster.resources | indent 10 }} - containers: - {{- if not .Values.syncer.disabled }} - - name: syncer - {{- if .Values.syncer.image }} - image: "{{ .Values.defaultImageRegistry }}{{ .Values.syncer.image }}" - {{- else }} - {{- if .Values.pro }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster-pro:{{ .Chart.Version }}" - {{- else }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster:{{ .Chart.Version }}" - {{- end }} - {{- end }} - {{- if .Values.syncer.workingDir }} - workingDir: {{ .Values.syncer.workingDir }} - {{- end }} - {{- if .Values.syncer.command }} - command: - {{- range $f := .Values.syncer.command }} - - {{ $f | quote }} - {{- end }} - {{- end }} - args: - - --name={{ .Release.Name }} - - --service-account=vc-workload-{{ .Release.Name }} - - --request-header-ca-cert=/data/k0s/pki/front-proxy-ca.crt - - --client-ca-cert=/data/k0s/pki/ca.crt - - --server-ca-cert=/data/k0s/pki/ca.crt - - --server-ca-key=/data/k0s/pki/ca.key - - --kube-config=/data/k0s/pki/admin.conf - {{- if and .Values.embeddedEtcd.enabled .Values.pro }} - - --etcd-embedded - - --etcd-replicas={{ .Values.syncer.replicas }} - {{- end }} - {{- if (gt (int .Values.syncer.replicas ) 1) }} - - --leader-elect=true - {{- else }} - - --leader-elect=false - {{- end }} - {{- include "vcluster.legacyPlugins.args" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromHost" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromVirtual" . | indent 10 }} - {{- if .Values.sync.nodes.enableScheduler }} - - --enable-scheduler - {{- end }} - {{- if .Values.pro }} - {{- if .Values.proLicenseSecret }} - - --pro-license-secret={{ .Values.proLicenseSecret }} - {{- end }} - {{- end }} - {{- if .Values.defaultImageRegistry }} - - --default-image-registry={{ .Values.defaultImageRegistry }} - {{- end }} - {{- if .Values.syncer.kubeConfigContextName }} - - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }} - {{- end }} - {{- if .Values.ingress.enabled }} - - --tls-san={{ .Values.ingress.host }} - {{- end }} - {{- if .Values.isolation.enabled }} - - --enforce-pod-security-standard={{ .Values.isolation.podSecurityStandard }} - {{- end}} - {{- include "vcluster.syncer.syncArgs" . | indent 10 -}} - {{- if .Values.sync.nodes.syncAllNodes }} - - --sync-all-nodes - {{- end }} - {{- if .Values.sync.nodes.nodeSelector }} - - --node-selector={{ .Values.sync.nodes.nodeSelector }} - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - --multi-namespace-mode=true - {{- end }} - {{- if .Values.sync.configmaps.all }} - - --sync-all-configmaps=true - {{- end }} - {{- if .Values.sync.secrets.all }} - - --sync-all-secrets=true - {{- end }} - {{- if not .Values.sync.nodes.fakeKubeletIPs }} - - --fake-kubelet-ips=false - {{- end }} - {{- if or .Values.proxy.metricsServer.nodes.enabled .Values.proxy.metricsServer.pods.enabled }} - - --proxy-metrics-server=true - {{- end }} - {{- if .Values.coredns.integrated }} - - --integrated-coredns=true - {{- end }} - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - - --use-coredns-plugin=true - {{- end }} - {{- if .Values.centralAdmission.validatingWebhooks }} - {{- range .Values.centralAdmission.validatingWebhooks }} - - --enforce-validating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- if .Values.centralAdmission.mutatingWebhooks }} - {{- range .Values.centralAdmission.mutatingWebhooks }} - - --enforce-mutating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- range $f := .Values.syncer.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- if .Values.syncer.livenessProbe }} - {{- if .Values.syncer.livenessProbe.enabled }} - livenessProbe: - httpGet: - path: /healthz - port: 8443 - scheme: HTTPS - failureThreshold: 10 - initialDelaySeconds: 60 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.readinessProbe }} - {{- if .Values.syncer.readinessProbe.enabled }} - startupProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 300 - periodSeconds: 6 - readinessProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 30 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.imagePullPolicy }} - imagePullPolicy: {{ .Values.syncer.imagePullPolicy }} - {{- end }} - securityContext: -{{ toYaml .Values.securityContext | indent 10 }} - env: - {{- include "vcluster.plugins.config" . | indent 10 }} - - name: VCLUSTER_DISTRO - value: k0s - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if eq ( ( include "vcluster.replicas" . ) | toString | atoi) 1 }} - - name: VCLUSTER_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- end }} - {{- if .Values.vcluster.env }} -{{ toYaml .Values.vcluster.env | indent 10 }} - {{- end }} - - name: ETCD_UNSUPPORTED_ARCH - value: arm64 - - name: VCLUSTER_COMMAND - value: |- - command: - {{- range $f := .Values.vcluster.command }} - - {{ $f | quote }} - {{- end }} - args: - {{- range $f := .Values.vcluster.baseArgs }} - - {{ $f | quote }} - {{- end }} - - --status-socket=/run/k0s/status.sock - {{- if not .Values.sync.nodes.enableScheduler }} - - --disable-components=konnectivity-server,kube-scheduler,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,worker-config - {{- else }} - - --disable-components=konnectivity-server,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,worker-config - {{- end }} - {{- range $f := .Values.vcluster.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- if .Values.syncer.env }} -{{ toYaml .Values.syncer.env | indent 10 }} - {{- end }} - {{- if .Values.sync.generic.config }} - - name: CONFIG - value: |- - {{- .Values.sync.generic.config | nindent 14 }} - {{- end }} - - name: VCLUSTER_TELEMETRY_CONFIG - value: {{ .Values.telemetry | toJson | quote }} - volumeMounts: - {{- include "vcluster.plugins.volumeMounts" . | indent 10 }} - - name: helm-cache - mountPath: /.cache/helm - - name: binaries - mountPath: /binaries - - mountPath: /data - name: data - - name: run-k0s - mountPath: /run/k0s - - name: tmp - mountPath: /tmp - {{- if .Values.coredns.enabled }} - - name: coredns - mountPath: /manifests/coredns - readOnly: true - {{- end }} - {{- if .Values.vcluster.volumeMounts }} -{{ toYaml .Values.vcluster.volumeMounts | indent 10 }} - {{- end }} - {{- if .Values.syncer.extraVolumeMounts }} -{{ toYaml .Values.syncer.extraVolumeMounts | indent 10 }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} -{{- include "vcluster.legacyPlugins.containers" . | indent 6 }} -{{- end }} diff --git a/charts/k0s/templates/workloadserviceaccount.yaml b/charts/k0s/templates/workloadserviceaccount.yaml deleted file mode 100644 index 8d05fc1e8..000000000 --- a/charts/k0s/templates/workloadserviceaccount.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-workload-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.workloadServiceAccount.annotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} diff --git a/charts/k0s/tests/README.md b/charts/k0s/tests/README.md deleted file mode 100644 index e6d4fa1c8..000000000 --- a/charts/k0s/tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Add [unittest plugin](https://github.com/helm-unittest/helm-unittest) via: -``` -helm plugin install https://github.com/helm-unittest/helm-unittest.git -``` - -Run tests via: -``` -helm unittest charts/k0s -d -``` diff --git a/charts/k0s/tests/clusterrole_test.yaml b/charts/k0s/tests/clusterrole_test.yaml deleted file mode 100644 index 3385c786c..000000000 --- a/charts/k0s/tests/clusterrole_test.yaml +++ /dev/null @@ -1,41 +0,0 @@ -suite: ClusterRole -templates: - - rbac/clusterrole.yaml - -tests: - - it: should create clusterrole - set: - rbac: - clusterRole: - create: true - asserts: - - hasDocuments: - count: 1 - - it: should not create clusterrole - set: - rbac: - clusterRole: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - clusterRole: - create: true - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/k0s/tests/legacy-plugins_test.yaml b/charts/k0s/tests/legacy-plugins_test.yaml deleted file mode 100644 index 6977c7562..000000000 --- a/charts/k0s/tests/legacy-plugins_test.yaml +++ /dev/null @@ -1,77 +0,0 @@ -suite: Legacy Plugins -templates: - - syncer.yaml - -tests: - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - bootstrap-with-deployment2: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment2" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[2].name - value: bootstrap-with-deployment2 - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[2].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - equal: - path: spec.template.spec.containers[2].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14001 - - - it: should check no legacy plugin rendering - asserts: - - hasDocuments: - count: 1 - - lengthEqual: - path: spec.template.spec.containers - count: 1 diff --git a/charts/k0s/tests/plugins_test.yaml b/charts/k0s/tests/plugins_test.yaml deleted file mode 100644 index 2ce3149dc..000000000 --- a/charts/k0s/tests/plugins_test.yaml +++ /dev/null @@ -1,100 +0,0 @@ -suite: Plugins -templates: - - syncer.yaml - -tests: - - it: should check plugin config rendering - set: - plugin: - plugin1: - version: v2 - config: - myConfig: true - plugin2: - version: v2 - image: test - plugin3: - version: v2 - image: test123 - config: - myOtherConfig: - - test123 - - test456 - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.containers[0].env[0].name - value: PLUGIN_CONFIG - - equal: - path: spec.template.spec.containers[0].env[0].value - value: |- - plugin1: - myConfig: true - plugin3: - myOtherConfig: - - test123 - - test456 - - - it: should check plugin rendering - set: - plugin: - bootstrap-with-deployment: - version: v2 - image: test - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.volumes[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins diff --git a/charts/k0s/tests/role_test.yaml b/charts/k0s/tests/role_test.yaml deleted file mode 100644 index 5eeedba74..000000000 --- a/charts/k0s/tests/role_test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -suite: Role -templates: - - rbac/role.yaml - -tests: - - it: should not create role - set: - rbac: - role: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - role: - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/k0s/tests/syncer_test.yaml b/charts/k0s/tests/syncer_test.yaml deleted file mode 100644 index 685792854..000000000 --- a/charts/k0s/tests/syncer_test.yaml +++ /dev/null @@ -1,65 +0,0 @@ -suite: Syncer -templates: - - syncer.yaml - -tests: - - it: should pass pro license secret as a flag - set: - pro: true - proLicenseSecret: "my-test-secret" - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--pro-license-secret=my-test-secret" - count: 1 - - it: should have pvc if persistence is on - set: - storage: - persistence: true - - asserts: - - hasDocuments: - count: 1 - - isNotEmpty: - path: spec.volumeClaimTemplates - - - it: should not have pvc if persistence is off - set: - storage: - persistence: false - - asserts: - - hasDocuments: - count: 1 - - isNull: - path: spec.volumeClaimTemplates - - - it: should not have emptyDir data volume if persistence is on - set: - storage: - persistence: true - - asserts: - - hasDocuments: - count: 1 - - notContains: - path: .spec.template.spec.volumes - content: - name: data - emptyDir: {} - - - it: should have emptyDir data volume if persistence is off - set: - storage: - persistence: false - - asserts: - - hasDocuments: - count: 1 - - contains: - path: .spec.template.spec.volumes - content: - name: data - emptyDir: {} diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml deleted file mode 100644 index 04f4ae4f9..000000000 --- a/charts/k0s/values.yaml +++ /dev/null @@ -1,514 +0,0 @@ -# DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed -# images within the vcluster will not be rewritten. -defaultImageRegistry: "" - -# Global annotations to add to all objects -globalAnnotations: {} - -# If vCluster.Pro is enabled -pro: false - -# Defines where vCluster should search for a license secret. If you are using the vCluster.Pro control-plane, -# this is optional. vCluster by default will lookout for a secret called vc-VCLUSTER_NAME-license and if found use that. -# You can also use a secret within another namespace by using the format NAMESPACE/NAME. If the secret is in another -# namespace, please enable clusterRole.create and define an extra clusterRole.extraRules to allow vCluster to retrieve -# secrets within the cluster. -proLicenseSecret: "" - -# Embedded etcd settings -embeddedEtcd: - # If embedded etcd should be enabled, this is a PRO only feature - enabled: false - -# If true, will deploy vcluster in headless mode, which means no deployment -# or statefulset is created. -headless: false - -monitoring: - serviceMonitor: - enabled: false - -# Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml' -plugin: {} -# Manually configure a plugin called test -# test: -# image: ... -# env: ... -# rbac: -# clusterRole: -# extraRules: ... -# role: -# extraRules: ... - -# Resource syncers that should be enabled/disabled. -# Enabling syncers will impact RBAC Role and ClusterRole permissions. -# To disable a syncer set "enabled: false". -# See docs for details - https://www.vcluster.com/docs/architecture/synced-resources -sync: - services: - enabled: true - configmaps: - enabled: true - all: false - secrets: - enabled: true - all: false - endpoints: - enabled: true - pods: - enabled: true - ephemeralContainers: false - status: false - events: - enabled: true - persistentvolumeclaims: - enabled: true - ingresses: - enabled: false - ingressclasses: {} - # By default IngressClasses sync is enabled when the Ingress sync is enabled - # but it can be explicitly disabled by setting: - # enabled: false - fake-nodes: - enabled: true # will be ignored if nodes.enabled = true - fake-persistentvolumes: - enabled: true # will be ignored if persistentvolumes.enabled = true - nodes: - fakeKubeletIPs: true - enabled: false - # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster - # will sync all nodes instead of only the ones where some pods are running. - syncAllNodes: false - # nodeSelector is used to limit which nodes get synced to the vcluster, - # and which nodes are used to run vcluster pods. - # A valid string representation of a label selector must be used. - nodeSelector: "" - # if true, vcluster will run with a scheduler and node changes are possible - # from within the virtual cluster. This is useful if you would like to - # taint, drain and label nodes from within the virtual cluster - enableScheduler: false - # DEPRECATED: use enable scheduler instead - # syncNodeChanges allows vcluster user edits of the nodes to be synced down to the host nodes. - # Write permissions on node resource will be given to the vcluster. - syncNodeChanges: false - persistentvolumes: - enabled: false - storageclasses: - enabled: false - # formerly named - "legacy-storageclasses" - hoststorageclasses: - enabled: false - priorityclasses: - enabled: false - networkpolicies: - enabled: false - volumesnapshots: - enabled: false - poddisruptionbudgets: - enabled: false - serviceaccounts: - enabled: false - # generic CRD configuration - generic: - config: |- - --- - -# If enabled, will fallback to host dns for resolving domains. This -# is useful if using istio or dapr in the host cluster and sidecar -# containers cannot connect to the central instance. Its also useful -# if you want to access host cluster services from within the vcluster. -fallbackHostDns: false - -# Map Services between host and virtual cluster -mapServices: - # Services that should get mapped from the - # virtual cluster to the host cluster. - # vcluster will make sure to sync the service - # ip to the host cluster automatically as soon - # as the service exists. - # For example: - # fromVirtual: - # - from: my-namespace/name - # to: host-service - fromVirtual: [] - # Same as from virtual, but instead sync services - # from the host cluster into the virtual cluster. - # If the namespace does not exist, vcluster will - # also create the namespace for the service. - fromHost: [] - -proxy: - metricsServer: - nodes: - enabled: false - pods: - enabled: false - -# Syncer configuration -syncer: - replicas: 1 - # Image to use for the syncer - # image: ghcr.io/loft-sh/vcluster - imagePullPolicy: "" - extraArgs: [] - env: [] - livenessProbe: - enabled: true - readinessProbe: - enabled: true - extraVolumeMounts: [] - resources: - limits: - ephemeral-storage: 8Gi - memory: 2Gi - requests: - ephemeral-storage: 200Mi - cpu: 10m - memory: 64Mi - kubeConfigContextName: "my-vcluster" - serviceAnnotations: {} - # Storage settings for the vcluster - storage: - # If this is disabled, vcluster will use an emptyDir instead - # of a PersistentVolumeClaim - persistence: true - # Size of the persistent volume claim - size: 5Gi - # Optional StorageClass used for the pvc - # if empty default StorageClass defined in your host cluster will be used - #className: - -# Virtual Cluster (k0s) configuration -vcluster: - # Image to use for the virtual cluster - image: k0sproject/k0s:v1.29.1-k0s.0 - imagePullPolicy: "" - command: - - /binaries/k0s - baseArgs: - - controller - - --config=/tmp/k0s-config.yaml - - --data-dir=/data/k0s - # Extra arguments for k0s. - extraArgs: [] - env: [] - resources: - limits: - cpu: 100m - memory: 256Mi - requests: - cpu: 40m - memory: 64Mi - priorityClassName: "" - # clusterDomain: cluster.local - - -# Extra volumes that should be created for the StatefulSet -volumes: [] - -# Service account that should be used by the vcluster -serviceAccount: - create: true - # Optional name of the service account to use - # name: default - # Optional pull secrets - # imagePullSecrets: - # - name: my-pull-secret - -# Service account that should be used by the pods synced by vcluster -workloadServiceAccount: - # This is not supported in multi-namespace mode - annotations: {} - -# Roles & ClusterRoles for the vcluster -rbac: - clusterRole: - # You don't need to toggle this as necessary cluster roles are created based on the enabled syncers (.sync.*.enabled). - # This only makes sense to enable if you want to use the extraRules below. - create: false - # Extra Rules for the cluster role - extraRules: [] - role: - # Disable this only if you don't want vCluster to create a role. This will break most functionality if disabled. - create: true - # Extra Rules for the role - extraRules: [] - # all entries in excludedApiResources will be excluded from the Role created for vcluster - excludedApiResources: - # - pods/exec - -# If vCluster persistent volume claims should get deleted automatically. -autoDeletePersistentVolumeClaims: false - -# NodeSelector used to schedule the vcluster -nodeSelector: {} - -# Affinity to apply to the vcluster statefulset -affinity: {} - -# PriorityClassName to apply to the vcluster statefulset -priorityClassName: "" - -# Tolerations to apply to the vcluster statefulset -tolerations: [] - -# Extra Labels for the stateful set -labels: {} -podLabels: {} - -# Extra Annotations for the stateful set -annotations: {} -podAnnotations: {} - -# Service configurations -service: - type: ClusterIP - - # Optional configuration - # A list of IP addresses for which nodes in the cluster will also accept traffic for this service. - # These IPs are not managed by Kubernetes; e.g., an external load balancer. - externalIPs: [] - - # Optional configuration for LoadBalancer & NodePort service types - # Route external traffic to node-local or cluster-wide endpoints [ Local | Cluster ] - externalTrafficPolicy: "" - - # Optional configuration for LoadBalancer service type - # Specify IP of load balancer to be created - loadBalancerIP: "" - # CIDR block(s) for the service allowlist - loadBalancerSourceRanges: [] - # Set the loadBalancerClass if using an external load balancer controller - loadBalancerClass: "" - # Set loadBalancer specific annotations on the Kubernetes service - loadBalancerAnnotations: {} - -# Configure the ingress resource that allows you to access the vcluster -ingress: - # Enable ingress record generation - enabled: false - # Ingress path type - pathType: ImplementationSpecific - ingressClassName: "" - host: vcluster.local - annotations: - nginx.ingress.kubernetes.io/backend-protocol: HTTPS - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - # Ingress TLS configuration - tls: [] - # - secretName: tls-vcluster.local - # hosts: - # - vcluster.local - -# Configure SecurityContext of the containers in the VCluster pod -securityContext: - allowPrivilegeEscalation: false - # capabilities: - # drop: - # - all - # readOnlyRootFilesystem will be set to true by default at a later release - # currently leaving it undefined for backwards compatibility with older vcluster cli versions - # readOnlyRootFilesystem: true - - # To run vcluster pod as non-root uncomment runAsUser and runAsNonRoot values. - # Update the runAsUser value if your cluster has limitations on user UIDs. - # For installation on OpenShift leave the runAsUser undefined (commented out). - # runAsUser: 12345 - # runAsNonRoot: true - runAsUser: 0 - runAsGroup: 0 - -# PodSecurityContext holds pod-level security attributes and common container settings for the vCluster pod -podSecurityContext: {} - -# Custom k0s to deploy -#config: |- -# apiVersion: k0s.k0sproject.io/v1beta1 -# ... - -# Set "enable" to true when running vcluster in an OpenShift host -# This will add an extra rule to the deployed role binding in order -# to manage service endpoints -openshift: - enable: false - -# If enabled will deploy the coredns configmap -coredns: - # If CoreDns is enabled - enabled: true - # Pro only feature - integrated: false - # only used if isolation is enabled - fallback: 8.8.8.8 - plugin: - enabled: false - config: [] - # example configuration for plugin syntax, will be documented in detail - # - record: - # fqdn: google.com - # target: - # mode: url - # url: google.co.in - # - record: - # service: my-namespace/my-svc # dns-test/nginx-svc - # target: - # mode: host - # service: dns-test/nginx-svc - # - record: - # service: my-namespace-lb/my-svc-lb - # target: - # mode: host - # service: dns-test-exposed-lb/nginx-svc-exposed-lb - # - record: - # service: my-ns-external-name/my-svc-external-name - # target: - # mode: host - # service: dns-test-external-name/nginx-svc-external-name - # - record: - # service: my-ns-in-vcluster/my-svc-vcluster - # target: - # mode: vcluster # can be tested only manually for now - # vcluster: test-vcluster-ns/test-vcluster - # service: dns-test-in-vcluster-ns/test-in-vcluster-service - # - record: - # service: my-ns-in-vcluster-mns/my-svc-mns - # target: - # mode: vcluster # can be tested only manually for now - # service: dns-test-in-vcluster-mns/test-in-vcluster-svc-mns - # vcluster: test-vcluster-ns-mns/test-vcluster-mns - # - record: - # service: my-self-vc-ns/my-self-vc-svc - # target: - # mode: self - # service: dns-test/nginx-svc - replicas: 1 - # The nodeSelector example below specifices that coredns should only be scheduled to nodes with the arm64 label - # nodeSelector: - # kubernetes.io/arch: arm64 - # image: my-core-dns-image:latest - # config: |- - # .:1053 { - # ... - # CoreDNS service configurations - service: - type: ClusterIP - # Configuration for LoadBalancer service type - externalIPs: [] - externalTrafficPolicy: "" - # Extra Annotations - annotations: {} - resources: - limits: - cpu: 1000m - memory: 512Mi - requests: - # ensure that cpu/memory requests are high enough. - # for example gke wants minimum 10m/32Mi here! - cpu: 20m - memory: 64Mi -# if below option is configured, it will override the coredns manifests with the following string -# manifests: |- -# apiVersion: ... -# ... - podAnnotations: {} - podLabels: {} - -# If enabled will deploy vcluster in an isolated mode with pod security -# standards, limit ranges and resource quotas -isolation: - enabled: false - namespace: null - - podSecurityStandard: baseline - - # If enabled will add node/proxy permission to the cluster role - # in isolation mode - nodeProxyPermission: - enabled: false - - resourceQuota: - enabled: true - quota: - requests.cpu: 10 - requests.memory: 20Gi - requests.storage: "100Gi" - requests.ephemeral-storage: 60Gi - limits.cpu: 20 - limits.memory: 40Gi - limits.ephemeral-storage: 160Gi - services.nodeports: 0 - services.loadbalancers: 1 - count/endpoints: 40 - count/pods: 20 - count/services: 20 - count/secrets: 100 - count/configmaps: 100 - count/persistentvolumeclaims: 20 - scopeSelector: - matchExpressions: - scopes: - - limitRange: - enabled: true - default: - ephemeral-storage: 8Gi - memory: 512Mi - cpu: "1" - defaultRequest: - ephemeral-storage: 3Gi - memory: 128Mi - cpu: 100m - - networkPolicy: - enabled: true - outgoingConnections: - ipBlock: - cidr: 0.0.0.0/0 - except: - - 100.64.0.0/10 - - 127.0.0.0/8 - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - -# manifests to setup when initializing a vcluster -init: - manifests: |- - --- - # The contents of manifests-template will be templated using helm - # this allows you to use helm values inside, e.g.: {{ .Release.Name }} - manifestsTemplate: '' - helm: [] - # - bundle: - base64-encoded .tar.gz file content (optional - overrides chart.repo) - # chart: - # name: REQUIRED - # version: REQUIRED - # repo: (optional when bundle is used) - # username: (if required for repo) - # password: (if required for repo) - # insecure: boolean (if required for repo) - # release: - # name: REQUIRED - # namespace: REQUIRED - # timeout: number - # values: |- string YAML object - # foo: bar - # valuesTemplate: |- string YAML object - # foo: {{ .Release.Name }} - -multiNamespaceMode: - enabled: false - -# list of {validating/mutating}webhooks that the syncer should proxy. -# This is a PRO only feature. -centralAdmission: - validatingWebhooks: [] - mutatingWebhooks: [] - -telemetry: - disabled: false - instanceCreator: "helm" - platformUserID: "" - platformInstanceID: "" - machineID: "" diff --git a/charts/k3s/.helmignore b/charts/k3s/.helmignore deleted file mode 100644 index b6a3eb5b3..000000000 --- a/charts/k3s/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store - -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ - -# Common backup files -*.swp -*.bak -*.tmp -*~ - -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/k3s/Chart.yaml b/charts/k3s/Chart.yaml deleted file mode 100644 index 516d9b124..000000000 --- a/charts/k3s/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v2 -name: vcluster -description: vcluster - Virtual Kubernetes Clusters -home: https://vcluster.com -icon: https://static.loft.sh/branding/logos/vcluster/vertical/vcluster_vertical.svg -keywords: - - developer - - development - - sharing - - share - - multi-tenancy - - tenancy - - cluster - - space - - namespace - - vcluster - - vclusters -maintainers: - - name: Loft Labs, Inc. - email: info@loft.sh - url: https://twitter.com/loft_sh -sources: - - https://github.com/loft-sh/vcluster -type: application - -version: 0.0.1 # version is auto-generated by release pipeline diff --git a/charts/k3s/README.md b/charts/k3s/README.md deleted file mode 100644 index 5125c17c0..000000000 --- a/charts/k3s/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -# vcluster - -## **[GitHub](https://github.com/loft-sh/vcluster)** • **[Website](https://www.vcluster.com)** • **[Quickstart](https://www.vcluster.com/docs/getting-started/setup)** • **[Documentation](https://www.vcluster.com/docs/what-are-virtual-clusters)** • **[Blog](https://loft.sh/blog)** • **[Twitter](https://twitter.com/loft_sh)** • **[Slack](https://slack.loft.sh/)** - -Create fully functional virtual Kubernetes clusters - Each vcluster runs inside a namespace of the underlying k8s cluster. It's cheaper than creating separate full-blown clusters and it offers better multi-tenancy and isolation than regular namespaces. - -## Prerequisites - -- Kubernetes 1.18+ -- Helm 3+ - -## Get Helm Repository Info - -```bash -helm repo add loft-sh https://charts.loft.sh -helm repo update -``` - -See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation. - -## Install Helm Chart - -```bash -helm upgrade [RELEASE_NAME] loft-sh/vcluster -n [RELEASE_NAMESPACE] --create-namespace --install -``` - -See [vcluster docs](https://vcluster.com/docs) for configuration options. - -See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation. - -## Connect to the vcluster - -In order to connect to the installed vcluster, please install [vcluster cli](https://www.vcluster.com/docs/getting-started/setup) and run: - -```bash -vcluster connect [RELEASE_NAME] -n [RELEASE_NAMESPACE] -``` - -## Uninstall Helm Chart - -```bash -helm uninstall [RELEASE_NAME] -``` - -This removes all the Kubernetes components associated with the chart and deletes the release. - -See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation. - -### Why Virtual Kubernetes Clusters? - -- **Cluster Scoped Resources**: much more powerful than simple namespaces (virtual clusters allow users to use CRDs, namespaces, cluster roles etc.) -- **Ease of Use**: usable in any Kubernetes cluster and created in seconds either via a single command or [cluster-api](https://github.com/loft-sh/cluster-api-provider-vcluster) -- **Cost Efficient**: much cheaper and efficient than "real" clusters (single pod and shared resources just like for namespaces) -- **Lightweight**: built upon the ultra-fast k3s distribution with minimal overhead per virtual cluster (other distributions work as well) -- **Strict isolation**: complete separate Kubernetes control plane and access point for each vcluster while still being able to share certain services of the underlying host cluster -- **Cluster Wide Permissions**: allow users to install apps which require cluster-wide permissions while being limited to actually just one namespace within the host cluster -- **Great for Testing**: allow you to test different Kubernetes versions inside a single host cluster which may have a different version than the virtual clusters - -Learn more on [www.vcluster.com](https://vcluster.com). - -![vcluster Intro](https://github.com/loft-sh/vcluster/raw/main/docs/static/media/vcluster-comparison.png) - -Learn more in the [documentation](https://vcluster.com/docs/what-are-virtual-clusters). diff --git a/charts/k3s/templates/NOTES.txt b/charts/k3s/templates/NOTES.txt deleted file mode 100644 index 6cf960d62..000000000 --- a/charts/k3s/templates/NOTES.txt +++ /dev/null @@ -1,10 +0,0 @@ -Thank you for installing vcluster. - -Your vcluster is named {{ .Release.Name }} in namespace {{ .Release.Namespace }}. - -To connect to the vcluster, use vcluster CLI (https://www.vcluster.com/docs/getting-started/setup): - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} -- kubectl get ns - - -For more information, please take a look at the vcluster docs at https://www.vcluster.com/docs diff --git a/charts/k3s/templates/_coredns.tpl b/charts/k3s/templates/_coredns.tpl deleted file mode 100644 index c86a8bcd4..000000000 --- a/charts/k3s/templates/_coredns.tpl +++ /dev/null @@ -1,52 +0,0 @@ -{{/* - Define a common coredns config -*/}} -{{- define "vcluster.corefile" -}} -Corefile: |- - {{- if .Values.coredns.config }} -{{ .Values.coredns.config | indent 8 }} - {{- else }} - .:1053 { - errors - health - ready - rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local - kubernetes cluster.local in-addr.arpa ip6.arpa { - {{- if .Values.pro }} - {{- if .Values.coredns.integrated }} - kubeconfig /data/k3s-config/kube-config.yaml - {{- end }} - {{- end }} - pods insecure - {{- if or (.Values.fallbackHostDns) (and .Values.coredns.integrated .Values.coredns.plugin.enabled) }} - fallthrough cluster.local in-addr.arpa ip6.arpa - {{- else }} - fallthrough in-addr.arpa ip6.arpa - {{- end }} - } - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - vcluster {{ toYaml .Values.coredns.plugin.config | b64enc }} - {{- end }} - hosts /etc/NodeHosts { - ttl 60 - reload 15s - fallthrough - } - prometheus :9153 - {{- if .Values.fallbackHostDns }} - forward . {{`{{.HOST_CLUSTER_DNS}}`}} - {{- else if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} - forward . /etc/resolv.conf {{ .Values.coredns.fallback }} { - policy sequential - } - {{- else }} - forward . /etc/resolv.conf - {{- end }} - cache 30 - loop - loadbalance - } - - import /etc/coredns/custom/*.server - {{- end }} -{{- end -}} diff --git a/charts/k3s/templates/_helpers.tpl b/charts/k3s/templates/_helpers.tpl deleted file mode 100644 index 50f08d6f9..000000000 --- a/charts/k3s/templates/_helpers.tpl +++ /dev/null @@ -1,196 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "vcluster.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Whether the ingressclasses syncer should be enabled -*/}} -{{- define "vcluster.syncIngressclassesEnabled" -}} -{{- if or - (.Values.sync.ingressclasses).enabled - (and - .Values.sync.ingresses.enabled - (not .Values.sync.ingressclasses)) -}} - {{- true -}} -{{- end -}} -{{- end -}} - -{{/* -Whether to create a cluster role or not -*/}} -{{- define "vcluster.createClusterRole" -}} -{{- if or - (not (empty (include "vcluster.serviceMapping.fromHost" . ))) - (not (empty (include "vcluster.plugin.clusterRoleExtraRules" . ))) - (not (empty (include "vcluster.generic.clusterRoleExtraRules" . ))) - .Values.rbac.clusterRole.create - .Values.sync.hoststorageclasses.enabled - (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") - (include "vcluster.syncIngressclassesEnabled" . ) - .Values.pro - .Values.sync.nodes.enabled - .Values.sync.persistentvolumes.enabled - .Values.sync.storageclasses.enabled - .Values.sync.priorityclasses.enabled - .Values.sync.volumesnapshots.enabled - .Values.proxy.metricsServer.nodes.enabled - .Values.multiNamespaceMode.enabled - .Values.coredns.plugin.enabled -}} -{{- true -}} -{{- end -}} -{{- end -}} - -{{- define "vcluster.clusterRoleName" -}} -{{- printf "vc-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "vcluster.clusterRoleNameMultinamespace" -}} -{{- printf "vc-mn-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Syncer flags for enabling/disabling controllers -Prints only the flags that modify the defaults: -- when default controller has enabled: false => `- "--sync=-controller` -- when non-default controller has enabled: true => `- "--sync=controller` -*/}} -{{- define "vcluster.syncer.syncArgs" -}} -{{- $defaultEnabled := list "services" "configmaps" "secrets" "endpoints" "pods" "events" "persistentvolumeclaims" "fake-nodes" "fake-persistentvolumes" -}} -{{- if and (hasKey .Values.sync.nodes "enableScheduler") .Values.sync.nodes.enableScheduler -}} - {{- $defaultEnabled = concat $defaultEnabled (list "csinodes" "csidrivers" "csistoragecapacities" ) -}} -{{- end -}} -{{- range $key, $val := .Values.sync }} -{{- if and (has $key $defaultEnabled) (not $val.enabled) }} -- --sync=-{{ $key }} -{{- else if and (not (has $key $defaultEnabled)) ($val.enabled)}} -{{- if eq $key "legacy-storageclasses" }} -- --sync=hoststorageclasses -{{- else }} -- --sync={{ $key }} -{{- end -}} -{{- end -}} -{{- end -}} -{{- if not (include "vcluster.syncIngressclassesEnabled" . ) }} -- --sync=-ingressclasses -{{- end -}} -{{- end -}} - -{{/* - Cluster role rules defined by plugins -*/}} -{{- define "vcluster.plugin.clusterRoleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.clusterRole }} -{{- if $container.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.clusterRoleExtraRules" -}} -{{- if .Values.sync.generic.clusterRole }} -{{- if .Values.sync.generic.clusterRole.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined on global level -*/}} -{{- define "vcluster.rbac.clusterRoleExtraRules" -}} -{{- if .Values.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - - -{{/* - Role rules defined on global level -*/}} -{{- define "vcluster.rbac.roleExtraRules" -}} -{{- if .Values.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined by plugins -*/}} -{{- define "vcluster.plugin.roleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.role }} -{{- if $container.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.roleExtraRules" -}} -{{- if .Values.sync.generic.role }} -{{- if .Values.sync.generic.role.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Virtual cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromVirtual" -}} -{{- range $key, $value := .Values.mapServices.fromVirtual }} -- '--map-virtual-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} - -{{/* - Host cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromHost" -}} -{{- range $key, $value := .Values.mapServices.fromHost }} -- '--map-host-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} diff --git a/charts/k3s/templates/_migrate.tpl b/charts/k3s/templates/_migrate.tpl deleted file mode 100644 index 34bef9a19..000000000 --- a/charts/k3s/templates/_migrate.tpl +++ /dev/null @@ -1,6 +0,0 @@ -{{/* - handles both replicas and syncer.replicas -*/}} -{{- define "vcluster.replicas" -}} -{{ if .Values.replicas }}{{ .Values.replicas }}{{ else }}{{ .Values.syncer.replicas }}{{ end }} -{{- end }} diff --git a/charts/k3s/templates/_plugin.tpl b/charts/k3s/templates/_plugin.tpl deleted file mode 100644 index b1ecd2fa2..000000000 --- a/charts/k3s/templates/_plugin.tpl +++ /dev/null @@ -1,188 +0,0 @@ -{{/* - Plugin config definition -*/}} -{{- define "vcluster.plugins.config" -}} -{{- $pluginFound := false -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} -{{- $pluginFound = true -}} -{{- end }} -{{- if $pluginFound }} -- name: PLUGIN_CONFIG - value: |- -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} - {{ $key }}: {{ toYaml $container.config | nindent 6 }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume mount definition -*/}} -{{- define "vcluster.plugins.volumeMounts" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- mountPath: /plugins - name: plugins -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume definition -*/}} -{{- define "vcluster.plugins.volumes" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- name: plugins - emptyDir: {} -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin init container definition -*/}} -{{- define "vcluster.plugins.initContainers" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if or $container.command $container.args }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- else }} - command: ["sh"] - args: ["-c", "cp -r /plugin /plugins/{{ $key }}"] - {{- end }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - {{- if $container.volumeMounts }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- else }} - volumeMounts: - - mountPath: /plugins - name: plugins - {{- end }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} -{{- end }} -{{- end -}} - -{{/* - Extra Syncer Args for the legacy Plugins -*/}} -{{- define "vcluster.legacyPlugins.args" -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{- continue }} -{{- end }} -{{- if not $container.optional }} -- --plugins={{ $key }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Sidecar container definition for the legacy syncer parts -*/}} -{{- define "vcluster.legacyPlugins.containers" -}} -{{- $counter := -1 -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{ continue }} -{{- end }} -{{- $counter = add1 $counter }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if $container.workingDir }} - workingDir: {{ $container.workingDir }} - {{- end }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- if $container.terminationMessagePath }} - terminationMessagePath: {{ $container.terminationMessagePath }} - {{- end }} - {{- if $container.terminationMessagePolicy }} - terminationMessagePolicy: {{ $container.terminationMessagePolicy }} - {{- end }} - env: - - name: VCLUSTER_PLUGIN_ADDRESS - value: "localhost:{{ add 14000 $counter }}" - - name: VCLUSTER_PLUGIN_NAME - value: "{{ $key }}" - {{- if $container.env }} -{{ toYaml $container.env | indent 4 }} - {{- end }} - envFrom: -{{ toYaml $container.envFrom | indent 4 }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - lifecycle: -{{ toYaml $container.lifecycle | indent 4 }} - livenessProbe: -{{ toYaml $container.livenessProbe | indent 4 }} - readinessProbe: -{{ toYaml $container.readinessProbe | indent 4 }} - startupProbe: -{{ toYaml $container.startupProbe | indent 4 }} - volumeDevices: -{{ toYaml $container.volumeDevices | indent 4 }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} - {{- end }} -{{- end }} - - diff --git a/charts/k3s/templates/_storage.tpl b/charts/k3s/templates/_storage.tpl deleted file mode 100644 index b10675ab6..000000000 --- a/charts/k3s/templates/_storage.tpl +++ /dev/null @@ -1,20 +0,0 @@ -{{/* - storage size -*/}} -{{- define "vcluster.storage.size" -}} -{{if .Values.storage }}{{ .Values.storage.size }}{{ else }}{{ .Values.syncer.storage.size }}{{ end }} -{{- end -}} - -{{/* - storage persistence -*/}} -{{- define "vcluster.storage.persistence" -}} -{{if .Values.storage }}{{ .Values.storage.persistence }}{{ else }}{{ .Values.syncer.storage.persistence }}{{ end }} -{{- end -}} - -{{/* - storage classname -*/}} -{{- define "vcluster.storage.className" -}} -{{if .Values.storage }}{{ .Values.storage.className }}{{ else }}{{ .Values.syncer.storage.className }}{{ end }} -{{- end -}} diff --git a/charts/k3s/templates/coredns.yaml b/charts/k3s/templates/coredns.yaml deleted file mode 100644 index cc25dd17f..000000000 --- a/charts/k3s/templates/coredns.yaml +++ /dev/null @@ -1,231 +0,0 @@ -{{- if not .Values.headless }} -{{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-coredns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{- if .Values.coredns.manifests }} - coredns.yaml: |- -{{ .Values.coredns.manifests | indent 4 }} -{{- else }} - coredns.yaml: |- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: coredns - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns - subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: -{{ include "vcluster.corefile" . | indent 6 }} - NodeHosts: "" - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/name: "CoreDNS" - spec: - replicas: {{ .Values.coredns.replicas }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - {{- if .Values.coredns.podAnnotations }} - annotations: -{{ toYaml .Values.coredns.podAnnotations | indent 12 }} - {{- end }} - labels: - k8s-app: kube-dns - {{- range $k, $v := .Values.coredns.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - priorityClassName: "system-cluster-critical" - serviceAccountName: coredns - nodeSelector: - kubernetes.io/os: linux - {{- if .Values.coredns.nodeSelector }} -{{ toYaml .Values.coredns.nodeSelector | indent 12 }} - {{- end }} - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - k8s-app: kube-dns - {{- if .Values.isolation.enabled }} - securityContext: - seccompProfile: - type: RuntimeDefault - {{- end }} - containers: - - name: coredns - {{- if .Values.coredns.image }} - image: {{ .Values.defaultImageRegistry }}{{ .Values.coredns.image }} - {{- else }} - image: {{`{{.IMAGE}}`}} - {{- end }} - imagePullPolicy: IfNotPresent - resources: -{{ toYaml .Values.coredns.resources | indent 16}} - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - - name: custom-config-volume - mountPath: /etc/coredns/custom - readOnly: true - securityContext: - runAsNonRoot: true - runAsUser: {{`{{.RUN_AS_USER}}`}} - runAsGroup: {{`{{.RUN_AS_GROUP}}`}} - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 2 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile - - key: NodeHosts - path: NodeHosts - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - selector: - k8s-app: kube-dns - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/k3s/templates/ingress.yaml b/charts/k3s/templates/ingress.yaml deleted file mode 100644 index d629f096f..000000000 --- a/charts/k3s/templates/ingress.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - {{- $annotations := merge .Values.ingress.annotations .Values.globalAnnotations }} - {{- if .Values.ingress.tls }} - {{- $annotations = omit $annotations "nginx.ingress.kubernetes.io/ssl-passthrough" }} - {{- end }} - {{- if $annotations }} - annotations: - {{- toYaml $annotations | nindent 4 }} - {{- end }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - {{- if .Values.ingress.ingressClassName }} - ingressClassName: {{ .Values.ingress.ingressClassName | quote }} - {{- end }} - rules: - - host: {{ .Values.ingress.host | quote }} - http: - paths: - - backend: - service: - name: {{ .Release.Name }} - port: - name: https - path: / - pathType: {{ .Values.ingress.pathType }} - {{- with .Values.ingress.tls }} - tls: - {{- toYaml . | nindent 4 }} - {{- end -}} -{{- end }} diff --git a/charts/k3s/templates/init-configmap.yaml b/charts/k3s/templates/init-configmap.yaml deleted file mode 100644 index 34498fd14..000000000 --- a/charts/k3s/templates/init-configmap.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- if .Values.init.manifests }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-init-manifests - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: - manifests: |- - {{ .Values.init.manifests | nindent 4 | trim }} - {{ tpl .Values.init.manifestsTemplate $ | nindent 4 | trim }} - {{- if .Values.init.helm }} - charts: |- - {{- range .Values.init.helm }} - {{- /* only render this chart entry if either of chart or bundle is defined */}} - {{- if .chart }} - - name: {{ .chart.name }} - repo: {{ .chart.repo }} - version: {{ .chart.version }} - {{- if .chart.username }} - username: {{ .chart.username }} - {{- end }} - {{- if .chart.password }} - password: {{ .chart.password }} - {{- end }} - {{- if .insecure }} - insecure: true - {{- end}} - {{- end }} - {{- if .bundle }} - bundle: {{ .bundle }} - {{- end }} - {{- if or .chart .bundle }} - {{- if or .values .valuesTemplate }} - values: |- - {{ (.values | default "") | nindent 8 | trim }} - {{ tpl (.valuesTemplate | default "") $ | nindent 8 | trim }} - {{- end}} - {{- if .release }} - timeout: {{ .timeout | default "120s" | quote }} - releaseName: {{ .release.name }} - releaseNamespace: {{ .release.namespace }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/k3s/templates/integrated-coredns.yaml b/charts/k3s/templates/integrated-coredns.yaml deleted file mode 100644 index 16907cba0..000000000 --- a/charts/k3s/templates/integrated-coredns.yaml +++ /dev/null @@ -1,64 +0,0 @@ -{{- if .Values.pro }} -{{- if .Values.coredns.integrated }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-dns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{ include "vcluster.corefile" . | indent 2 }} - coredns.yaml: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: - NodeHosts: "" - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} diff --git a/charts/k3s/templates/limitrange.yaml b/charts/k3s/templates/limitrange.yaml deleted file mode 100644 index ed219d063..000000000 --- a/charts/k3s/templates/limitrange.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.limitRange.enabled }} -apiVersion: v1 -kind: LimitRange -metadata: - name: {{ .Release.Name }}-limit-range - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - limits: - - default: - {{- range $key, $val := .Values.isolation.limitRange.default }} - {{ $key }}: {{ $val | quote }} - {{- end }} - defaultRequest: - {{- range $key, $val := .Values.isolation.limitRange.defaultRequest }} - {{ $key }}: {{ $val | quote }} - {{- end }} - type: Container -{{- end }} diff --git a/charts/k3s/templates/networkpolicy.yaml b/charts/k3s/templates/networkpolicy.yaml deleted file mode 100644 index e40d63356..000000000 --- a/charts/k3s/templates/networkpolicy.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-workloads - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - egress: - # Allows outgoing connections to the vcluster control plane - - ports: - - port: 443 - - port: 8443 - to: - - podSelector: - matchLabels: - release: {{ .Release.Name }} - # Allows outgoing connections to DNS server - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Allows outgoing connections to the internet or - # other vcluster workloads - - to: - - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - - ipBlock: - cidr: {{ .Values.isolation.networkPolicy.outgoingConnections.ipBlock.cidr }} - except: - {{- range .Values.isolation.networkPolicy.outgoingConnections.ipBlock.except }} - - {{ . }} - {{- end }} - policyTypes: - - Egress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-control-plane - namespace: {{ .Release.Namespace }} -spec: - podSelector: - matchLabels: - release: {{ .Release.Name }} - egress: - # Allows outgoing connections to all pods with - # port 443, 8443 or 6443. This is needed for host Kubernetes - # access - - ports: - - port: 443 - - port: 8443 - - port: 6443 - # Allows outgoing connections to all vcluster workloads - # or kube system dns server - - to: - - podSelector: {} - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: 'kube-system' - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Egress -{{- end }} diff --git a/charts/k3s/templates/rbac/clusterrole.yaml b/charts/k3s/templates/rbac/clusterrole.yaml deleted file mode 100644 index 984267cae..000000000 --- a/charts/k3s/templates/rbac/clusterrole.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: -{{- if .Values.pro }} - - apiGroups: ["cluster.loft.sh", "storage.loft.sh"] - resources: ["features", "virtualclusters"] - verbs: ["get", "list", "watch"] -{{- end }} - {{- if or .Values.pro .Values.sync.nodes.enabled }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: [ "pods", "nodes/metrics", "nodes/stats"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.coredns.plugin.enabled }} - - apiGroups: [""] - resources: [ "pods"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled (or (not .Values.isolation.enabled) (and .Values.isolation.nodeProxyPermission.enabled .Values.isolation.enabled)) }} - - apiGroups: [""] - resources: ["nodes/proxy"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled .Values.sync.nodes.syncNodeChanges }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["update", "patch"] - {{- end }} - {{- if .Values.sync.persistentvolumes.enabled }} - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if .Values.sync.nodes.enableScheduler }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses","csinodes","csidrivers","csistoragecapacities"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if (include "vcluster.syncIngressclassesEnabled" . ) }} - - apiGroups: ["networking.k8s.io"] - resources: ["ingressclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.storageclasses.enabled }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if or .Values.sync.hoststorageclasses.enabled (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") .Values.rbac.clusterRole.create }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.priorityclasses.enabled }} - - apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if (not (empty (include "vcluster.serviceMapping.fromHost" . ))) }} - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.proxy.metricsServer.nodes.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["nodes"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.clusterRoleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/k3s/templates/rbac/clusterrolebinding.yaml b/charts/k3s/templates/rbac/clusterrolebinding.yaml deleted file mode 100644 index 0a5645d28..000000000 --- a/charts/k3s/templates/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ template "vcluster.clusterRoleName" . }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/k3s/templates/rbac/role.yaml b/charts/k3s/templates/rbac/role.yaml deleted file mode 100644 index de05bbb62..000000000 --- a/charts/k3s/templates/rbac/role.yaml +++ /dev/null @@ -1,104 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRole -{{- else -}} -kind: Role -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} -{{- end }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: - {{- if .Values.pro }} - - apiGroups: [""] - {{- $resources := list "configmaps" "secrets" "services" "pods" "pods/attach" "pods/portforward" "pods/exec" "persistentvolumeclaims" }} - {{- range $excluded := .Values.rbac.role.excludedApiResources }} - {{- $resources = without $resources $excluded }} - {{- end}} - resources: {{ $resources | toJson }} - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- else }} - - apiGroups: [""] - resources: ["configmaps", "secrets", "services", "pods", "pods/attach", "pods/portforward", "pods/exec", "persistentvolumeclaims"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.status }} - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.ephemeralContainers }} - - apiGroups: [""] - resources: ["pods/ephemeralcontainers"] - verbs: ["patch", "update"] - {{- end }} - {{- if or .Values.sync.endpoints.enabled .Values.headless }} - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["create", "delete", "patch", "update"] - {{- end }} - {{- if gt (int .Values.syncer.replicas) 1 }} - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: [""] - resources: ["endpoints", "events", "pods/log"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.ingresses.enabled}} - - apiGroups: ["networking.k8s.io"] - resources: ["ingresses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "deployments"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.networkpolicies.enabled }} - - apiGroups: ["networking.k8s.io"] - resources: ["networkpolicies"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.serviceaccounts.enabled }} - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.poddisruptionbudgets.enabled }} - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.openshift.enable }} - {{- if .Values.sync.endpoints.enabled }} - - apiGroups: [""] - resources: ["endpoints/restricted"] - verbs: ["create"] - {{- end }} - {{- end }} - {{- if .Values.proxy.metricsServer.pods.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["pods"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.roleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.roleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.roleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/k3s/templates/rbac/rolebinding.yaml b/charts/k3s/templates/rbac/rolebinding.yaml deleted file mode 100644 index 676e5002a..000000000 --- a/charts/k3s/templates/rbac/rolebinding.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRoleBinding -{{- else -}} -kind: RoleBinding -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -{{- end }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: -{{- if .Values.multiNamespaceMode.enabled }} - kind: ClusterRole - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - kind: Role - name: {{ .Release.Name }} -{{- end }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/k3s/templates/resourcequota.yaml b/charts/k3s/templates/resourcequota.yaml deleted file mode 100644 index b19d89fff..000000000 --- a/charts/k3s/templates/resourcequota.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.resourceQuota.enabled }} -apiVersion: v1 -kind: ResourceQuota -metadata: - name: {{ .Release.Name }}-quota - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - hard: - {{- range $key, $val := .Values.isolation.resourceQuota.quota }} - {{ $key }}: {{ $val | quote }} - {{- end }} - - {{- if .Values.isolation.resourceQuota.scopeSelector.matchExpressions }} - scopeSelector: - matchExpressions: - {{- toYaml .Values.isolation.resourceQuota.scopeSelector.matchExpressions | nindent 4 }} - {{- end}} - - {{- if .Values.isolation.resourceQuota.scopes }} - scopes: - {{- toYaml .Values.isolation.resourceQuota.scopes | nindent 4 }} - {{- end}} - -{{- end }} diff --git a/charts/k3s/templates/service-monitor.yaml b/charts/k3s/templates/service-monitor.yaml deleted file mode 100644 index 683506ab5..000000000 --- a/charts/k3s/templates/service-monitor.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if .Values.monitoring.serviceMonitor.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: vcluster - release: "{{ .Release.Name }}" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: "{{ .Release.Service }}" - endpoints: - - interval: 30s - port: https - path: /metrics - scheme: https - tlsConfig: - ca: - secret: - name: vc-{{ .Release.Name }} - key: certificate-authority - cert: - secret: - name: vc-{{ .Release.Name }} - key: client-certificate - keySecret: - name: vc-{{ .Release.Name }} - key: client-key -{{- end }} diff --git a/charts/k3s/templates/service.yaml b/charts/k3s/templates/service.yaml deleted file mode 100644 index b575a9fe1..000000000 --- a/charts/k3s/templates/service.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: {{ if eq .Values.service.type "LoadBalancer" -}}ClusterIP{{- else }} {{- .Values.service.type }} {{- end }} - ports: - - name: https - port: 443 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.httpsNodePort }} - protocol: TCP - - name: kubelet - port: 10250 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.kubeletNodePort }} - protocol: TCP - {{- if .Values.service.externalIPs }} - externalIPs: - {{- range $f := .Values.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- if eq .Values.service.type "NodePort" }} - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} ---- -{{ if eq .Values.service.type "LoadBalancer" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-lb - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.service.loadBalancerAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: LoadBalancer - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.service.loadBalancerClass }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range $f := .Values.service.loadBalancerSourceRanges }} - - "{{ $f }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/k3s/templates/serviceaccount.yaml b/charts/k3s/templates/serviceaccount.yaml deleted file mode 100644 index 504998037..000000000 --- a/charts/k3s/templates/serviceaccount.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} -{{- end }} diff --git a/charts/k3s/templates/statefulset-service.yaml b/charts/k3s/templates/statefulset-service.yaml deleted file mode 100644 index 7523bf36e..000000000 --- a/charts/k3s/templates/statefulset-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-headless - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "vcluster.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - publishNotReadyAddresses: true - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.embeddedEtcd.enabled }} - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - {{- end }} - clusterIP: None - selector: - app: vcluster - release: "{{ .Release.Name }}" -{{- end }} diff --git a/charts/k3s/templates/syncer.yaml b/charts/k3s/templates/syncer.yaml deleted file mode 100644 index a83a8e250..000000000 --- a/charts/k3s/templates/syncer.yaml +++ /dev/null @@ -1,431 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.labels }} -{{ toYaml .Values.labels | indent 4 }} -{{- end }} -{{- $annotations := merge .Values.annotations .Values.globalAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - serviceName: {{ .Release.Name }}-headless - {{- if .Values.autoDeletePersistentVolumeClaims }} - {{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} - persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete - {{- end }} - {{- end }} - replicas: {{include "vcluster.replicas" .}} - selector: - matchLabels: - app: vcluster - release: {{ .Release.Name }} - {{- if (hasKey .Values "volumeClaimTemplates") }} - volumeClaimTemplates: -{{ toYaml .Values.volumeClaimTemplates | indent 4 }} - {{- else if (eq ( include "vcluster.storage.persistence" . ) "true" ) }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if ( include "vcluster.storage.className" . ) }} - storageClassName: {{ include "vcluster.storage.className" . }} - {{- end }} - resources: - requests: - storage: {{ include "vcluster.storage.size" . }} - {{- end }} - template: - metadata: - {{- if .Values.podAnnotations }} - annotations: -{{ toYaml .Values.podAnnotations | indent 8 }} - {{- end }} - labels: - app: vcluster - release: {{ .Release.Name }} - {{- range $k, $v := .Values.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - terminationGracePeriodSeconds: 10 - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} - {{- if .Values.affinity }} - affinity: -{{ toYaml .Values.affinity | indent 8 }} - {{- else if (gt (int ( include "vcluster.replicas" . )) 1) }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - # if possible avoid scheduling more than one pod on one node - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: "kubernetes.io/hostname" - # if possible avoid scheduling pod onto node that is in the same zone as one or more vcluster pods are running - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: topology.kubernetes.io/zone - {{- end }} - tolerations: -{{ toYaml .Values.tolerations | indent 8 }} - {{- if .Values.serviceAccount.name }} - serviceAccountName: {{ .Values.serviceAccount.name }} - {{- else }} - serviceAccountName: vc-{{ .Release.Name }} - {{- end }} - volumes: - {{- include "vcluster.plugins.volumes" . | indent 8 }} - - name: helm-cache - emptyDir: {} - - name: binaries - emptyDir: {} - - name: tmp - emptyDir: {} - - name: config - emptyDir: {} - {{- if .Values.syncer.volumes }} -{{ toYaml .Values.syncer.volumes | indent 8 }} - {{- end }} - {{- if .Values.volumes }} -{{ toYaml .Values.volumes | indent 8 }} - {{- end }} - {{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} - - name: coredns - configMap: - name: {{ .Release.Name }}-coredns - {{- else if .Values.coredns.integrated }} - - name: coredns - configMap: - name: {{ .Release.Name }}-dns - {{- end }} - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - {{- if not (eq ( include "vcluster.storage.persistence" . ) "true" ) }} - - name: data - emptyDir: {} - {{- end }} - {{- if and .Values.noopSyncer.enabled .Values.noopSyncer.secret }} - - name: noopsyncer-cluster-certs - secret: - secretName: {{ .Values.noopSyncer.secret }} - optional: true - {{- end }} - {{- if .Values.priorityClassName }} - priorityClassName: {{ .Values.priorityClassName }} - {{- end }} - {{- if or .Values.podSecurityContext .Values.fsGroup }} - {{- $fsContext := dict "fsGroup" .Values.fsGroup }} - {{- with merge .Values.podSecurityContext $fsContext }} - securityContext: -{{ toYaml . | indent 8 }} - {{- end }} - {{- end }} - {{- if and (not .Values.vcluster.disabled) (not .Values.noopSyncer.enabled) }} - initContainers: - {{- include "vcluster.plugins.initContainers" . | indent 6 }} - - image: {{ .Values.defaultImageRegistry }}{{ .Values.vcluster.image }} - name: vcluster - # k3s has a problem running as pid 1 and disabled agents on cgroupv2 - # nodes as it will try to evacuate the cgroups there. Starting k3s - # through a shell makes it non pid 1 and prevents this from happening - command: - - /bin/sh - args: - - -c - - "cp /bin/k3s /binaries/k3s" - {{- if .Values.vcluster.imagePullPolicy }} - imagePullPolicy: {{ .Values.vcluster.imagePullPolicy }} - {{- end }} - securityContext: -{{ toYaml .Values.securityContext | indent 10 }} - volumeMounts: - - name: binaries - mountPath: /binaries - resources: -{{ toYaml .Values.vcluster.resources | indent 10 }} - {{- end }} - containers: - {{- if not .Values.syncer.disabled }} - - name: syncer - {{- if .Values.syncer.image }} - image: "{{ .Values.defaultImageRegistry }}{{ .Values.syncer.image }}" - {{- else }} - {{- if .Values.pro }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster-pro:{{ .Chart.Version }}" - {{- else }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster:{{ .Chart.Version }}" - {{- end }} - {{- end }} - {{- if .Values.syncer.workingDir }} - workingDir: {{ .Values.syncer.workingDir }} - {{- end }} - {{- if .Values.syncer.command }} - command: - {{- range $f := .Values.syncer.command }} - - {{ $f | quote }} - {{- end }} - {{- end }} - args: - - --name={{ .Release.Name }} - - --kube-config=/data/k3s-config/kube-config.yaml - - --service-account=vc-workload-{{ .Release.Name }} - {{- include "vcluster.legacyPlugins.args" . | indent 10 }} - {{- if .Values.pro }} - {{- if .Values.proLicenseSecret }} - - --pro-license-secret={{ .Values.proLicenseSecret }} - {{- end }} - {{- if .Values.embeddedEtcd.enabled }} - - --etcd-embedded - - --etcd-replicas={{ include "vcluster.replicas" . }} - {{- end }} - {{- end }} - {{- if .Values.sync.nodes.enableScheduler }} - - --enable-scheduler - {{- end }} - {{- if .Values.defaultImageRegistry }} - - --default-image-registry={{ .Values.defaultImageRegistry }} - {{- end }} - {{- if .Values.syncer.kubeConfigContextName }} - - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }} - {{- end }} - {{- if (gt (int ( include "vcluster.replicas" . )) 1) }} - - --leader-elect=true - {{- else }} - - --leader-elect=false - {{- end }} - {{- if .Values.ingress.enabled }} - - --tls-san={{ .Values.ingress.host }} - {{- end }} - {{- if .Values.isolation.enabled }} - - --enforce-pod-security-standard={{ .Values.isolation.podSecurityStandard }} - {{- end}} - {{- include "vcluster.syncer.syncArgs" . | indent 10 }} - {{- if .Values.sync.nodes.syncAllNodes }} - - --sync-all-nodes - {{- end }} - {{- if .Values.sync.nodes.nodeSelector }} - - --node-selector={{ .Values.sync.nodes.nodeSelector }} - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - --multi-namespace-mode=true - {{- end }} - {{- if .Values.sync.configmaps.all }} - - --sync-all-configmaps=true - {{- end }} - {{- if .Values.sync.secrets.all }} - - --sync-all-secrets=true - {{- end }} - {{- if not .Values.sync.nodes.fakeKubeletIPs }} - - --fake-kubelet-ips=false - {{- end }} - {{- if or .Values.proxy.metricsServer.nodes.enabled .Values.proxy.metricsServer.pods.enabled }} - - --proxy-metrics-server=true - {{- end }} - {{- if .Values.coredns.integrated }} - - --integrated-coredns=true - {{- end }} - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - - --use-coredns-plugin=true - {{- end }} - {{- if .Values.noopSyncer.enabled }} - - --noop-syncer=true - {{- if .Values.noopSyncer.synck8sService }} - - --sync-k8s-service=true - {{- end }} - {{- end }} - {{- if .Values.centralAdmission.validatingWebhooks }} - {{- range .Values.centralAdmission.validatingWebhooks }} - - --enforce-validating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- if .Values.centralAdmission.mutatingWebhooks }} - {{- range .Values.centralAdmission.mutatingWebhooks }} - - --enforce-mutating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- range $f := .Values.syncer.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- include "vcluster.serviceMapping.fromHost" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromVirtual" . | indent 10 }} - {{- if .Values.syncer.livenessProbe }} - {{- if .Values.syncer.livenessProbe.enabled }} - livenessProbe: - httpGet: - path: /healthz - port: 8443 - scheme: HTTPS - failureThreshold: 60 - initialDelaySeconds: 60 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.readinessProbe }} - {{- if .Values.syncer.readinessProbe.enabled }} - startupProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 300 - periodSeconds: 6 - readinessProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 60 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.imagePullPolicy }} - imagePullPolicy: {{ .Values.syncer.imagePullPolicy }} - {{- end }} - securityContext: -{{ toYaml .Values.securityContext | indent 10 }} - env: - {{- include "vcluster.plugins.config" . | indent 10 }} - - name: VCLUSTER_DISTRO - value: k3s - {{- if .Values.vcluster.env }} -{{ toYaml .Values.vcluster.env | indent 10 }} - {{- end }} - {{- if .Values.syncer.env }} -{{ toYaml .Values.syncer.env | indent 10 }} - {{- end }} - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: VCLUSTER_COMMAND - value: |- - command: - {{ range $f := .Values.vcluster.command }} - - {{ $f }} - {{- end }} - args: - {{- range $f := .Values.vcluster.baseArgs }} - - {{ $f }} - {{- end }} - {{- if not .Values.sync.nodes.enableScheduler }} - - --disable-scheduler - - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl - - --kube-apiserver-arg=endpoint-reconciler-type=none - {{- else }} - - --kube-controller-manager-arg=controllers=*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl - - --kube-apiserver-arg=endpoint-reconciler-type=none - - --kube-controller-manager-arg=node-monitor-grace-period=1h - - --kube-controller-manager-arg=node-monitor-period=1h - {{- end }} - {{- if .Values.pro }} - {{- if .Values.embeddedEtcd.enabled }} - - --datastore-endpoint=https://localhost:2379 - - --datastore-cafile=/data/pki/etcd/ca.crt - - --datastore-certfile=/data/pki/apiserver-etcd-client.crt - - --datastore-keyfile=/data/pki/apiserver-etcd-client.key - {{- end }} - {{- end }} - {{- range $f := .Values.vcluster.extraArgs }} - - {{ $f }} - {{- end }} - {{- if eq ( ( include "vcluster.replicas" . ) | toString | atoi) 1 }} - - name: VCLUSTER_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- end }} - {{- if .Values.sync.generic.config }} - - name: CONFIG - value: |- - {{- .Values.sync.generic.config | nindent 14 }} - {{- end }} - - name: VCLUSTER_TELEMETRY_CONFIG - value: {{ .Values.telemetry | toJson | quote }} - volumeMounts: - {{- include "vcluster.plugins.volumeMounts" . | indent 10 }} - - name: binaries - mountPath: /binaries - - name: helm-cache - mountPath: /.cache/helm - - name: config - mountPath: /etc/rancher - - name: tmp - mountPath: /tmp - {{- if or .Values.coredns.enabled .Values.coredns.integrated }} - - name: coredns - mountPath: /manifests/coredns - readOnly: true - - name: custom-config-volume - mountPath: /etc/coredns/custom - readOnly: true - {{- end }} - {{- if and .Values.noopSyncer.enabled .Values.noopSyncer.secret }} - - name: noopsyncer-cluster-certs - mountPath: /data/server/tls/request-header-ca.crt - subPath: requestHeaderCaCert - - name: noopsyncer-cluster-certs - mountPath: /data/server/tls/client-ca.crt - subPath: clientCaCert - - name: noopsyncer-cluster-certs - mountPath: /data/server/tls/server-ca.crt - subPath: serverCaCert - - name: noopsyncer-cluster-certs - mountPath: /data/server/tls/server-ca.key - subPath: serverCaKey - - name: noopsyncer-cluster-certs - mountPath: /data/server/cred/admin.kubeconfig - subPath: kubeConfig - {{- end }} -{{ toYaml .Values.syncer.volumeMounts | indent 10 }} - {{- if .Values.syncer.extraVolumeMounts }} -{{ toYaml .Values.syncer.extraVolumeMounts | indent 10 }} - {{- end }} - {{- if .Values.vcluster.volumeMounts }} -{{ toYaml .Values.vcluster.volumeMounts | indent 10 }} - {{- end }} - {{- if .Values.vcluster.extraVolumeMounts }} -{{ toYaml .Values.vcluster.extraVolumeMounts | indent 10 }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} -{{- include "vcluster.legacyPlugins.containers" . | indent 6 }} -{{- end }} diff --git a/charts/k3s/templates/token-secret.yaml b/charts/k3s/templates/token-secret.yaml deleted file mode 100644 index a87189fd8..000000000 --- a/charts/k3s/templates/token-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if .Values.k3sToken }} -apiVersion: v1 -kind: Secret -metadata: - name: "vc-k3s-{{ .Release.Name }}" - namespace: {{ .Release.Namespace }} -type: Opaque -data: - token: {{ .Values.k3sToken | b64enc | quote }} -{{- end }} diff --git a/charts/k3s/templates/workloadserviceaccount.yaml b/charts/k3s/templates/workloadserviceaccount.yaml deleted file mode 100644 index 8d05fc1e8..000000000 --- a/charts/k3s/templates/workloadserviceaccount.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-workload-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.workloadServiceAccount.annotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} diff --git a/charts/k3s/tests/README.md b/charts/k3s/tests/README.md deleted file mode 100644 index bd826bdbe..000000000 --- a/charts/k3s/tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Add [unittest plugin](https://github.com/helm-unittest/helm-unittest) via: -``` -helm plugin install https://github.com/helm-unittest/helm-unittest.git -``` - -Run tests via: -``` -helm unittest charts/k3s -d -``` diff --git a/charts/k3s/tests/clusterrole_test.yaml b/charts/k3s/tests/clusterrole_test.yaml deleted file mode 100644 index 3385c786c..000000000 --- a/charts/k3s/tests/clusterrole_test.yaml +++ /dev/null @@ -1,41 +0,0 @@ -suite: ClusterRole -templates: - - rbac/clusterrole.yaml - -tests: - - it: should create clusterrole - set: - rbac: - clusterRole: - create: true - asserts: - - hasDocuments: - count: 1 - - it: should not create clusterrole - set: - rbac: - clusterRole: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - clusterRole: - create: true - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/k3s/tests/legacy-plugins_test.yaml b/charts/k3s/tests/legacy-plugins_test.yaml deleted file mode 100644 index 6977c7562..000000000 --- a/charts/k3s/tests/legacy-plugins_test.yaml +++ /dev/null @@ -1,77 +0,0 @@ -suite: Legacy Plugins -templates: - - syncer.yaml - -tests: - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - bootstrap-with-deployment2: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment2" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[2].name - value: bootstrap-with-deployment2 - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[2].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - equal: - path: spec.template.spec.containers[2].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14001 - - - it: should check no legacy plugin rendering - asserts: - - hasDocuments: - count: 1 - - lengthEqual: - path: spec.template.spec.containers - count: 1 diff --git a/charts/k3s/tests/plugins_test.yaml b/charts/k3s/tests/plugins_test.yaml deleted file mode 100644 index 2ce3149dc..000000000 --- a/charts/k3s/tests/plugins_test.yaml +++ /dev/null @@ -1,100 +0,0 @@ -suite: Plugins -templates: - - syncer.yaml - -tests: - - it: should check plugin config rendering - set: - plugin: - plugin1: - version: v2 - config: - myConfig: true - plugin2: - version: v2 - image: test - plugin3: - version: v2 - image: test123 - config: - myOtherConfig: - - test123 - - test456 - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.containers[0].env[0].name - value: PLUGIN_CONFIG - - equal: - path: spec.template.spec.containers[0].env[0].value - value: |- - plugin1: - myConfig: true - plugin3: - myOtherConfig: - - test123 - - test456 - - - it: should check plugin rendering - set: - plugin: - bootstrap-with-deployment: - version: v2 - image: test - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.volumes[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins diff --git a/charts/k3s/tests/role_test.yaml b/charts/k3s/tests/role_test.yaml deleted file mode 100644 index 5eeedba74..000000000 --- a/charts/k3s/tests/role_test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -suite: Role -templates: - - rbac/role.yaml - -tests: - - it: should not create role - set: - rbac: - role: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - role: - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/k3s/tests/syncer_test.yaml b/charts/k3s/tests/syncer_test.yaml deleted file mode 100644 index 685792854..000000000 --- a/charts/k3s/tests/syncer_test.yaml +++ /dev/null @@ -1,65 +0,0 @@ -suite: Syncer -templates: - - syncer.yaml - -tests: - - it: should pass pro license secret as a flag - set: - pro: true - proLicenseSecret: "my-test-secret" - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--pro-license-secret=my-test-secret" - count: 1 - - it: should have pvc if persistence is on - set: - storage: - persistence: true - - asserts: - - hasDocuments: - count: 1 - - isNotEmpty: - path: spec.volumeClaimTemplates - - - it: should not have pvc if persistence is off - set: - storage: - persistence: false - - asserts: - - hasDocuments: - count: 1 - - isNull: - path: spec.volumeClaimTemplates - - - it: should not have emptyDir data volume if persistence is on - set: - storage: - persistence: true - - asserts: - - hasDocuments: - count: 1 - - notContains: - path: .spec.template.spec.volumes - content: - name: data - emptyDir: {} - - - it: should have emptyDir data volume if persistence is off - set: - storage: - persistence: false - - asserts: - - hasDocuments: - count: 1 - - contains: - path: .spec.template.spec.volumes - content: - name: data - emptyDir: {} diff --git a/charts/k3s/values.yaml b/charts/k3s/values.yaml deleted file mode 100644 index 70bdbf2b5..000000000 --- a/charts/k3s/values.yaml +++ /dev/null @@ -1,539 +0,0 @@ -# These annotations will be applied to all objects created in this chart -globalAnnotations: {} - -# If vCluster.Pro is enabled -pro: false - -# Defines where vCluster should search for a license secret. If you are using the vCluster.Pro control-plane, -# this is optional. vCluster by default will lookout for a secret called vc-VCLUSTER_NAME-license and if found use that. -# You can also use a secret within another namespace by using the format NAMESPACE/NAME. If the secret is in another -# namespace, please enable clusterRole.create and define an extra clusterRole.extraRules to allow vCluster to retrieve -# secrets within the cluster. -proLicenseSecret: "" - -# If true, will deploy vcluster in headless mode, which means no deployment -# or statefulset is created. -headless: false - -monitoring: - serviceMonitor: - enabled: false - -# DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed -# images within the vcluster will not be rewritten. -defaultImageRegistry: "" - -# Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml' -plugin: {} -# Manually configure a plugin called test -# test: -# image: ... -# env: ... -# rbac: -# clusterRole: -# extraRules: ... -# role: -# extraRules: ... - -# Resource syncers that should be enabled/disabled. -# Enabling syncers will impact RBAC Role and ClusterRole permissions. -# To disable a syncer set "enabled: false". -# See docs for details - https://www.vcluster.com/docs/architecture/synced-resources -sync: - services: - enabled: true - configmaps: - enabled: true - all: false - secrets: - all: false - enabled: true - endpoints: - enabled: true - pods: - enabled: true - ephemeralContainers: false - status: false - events: - enabled: true - persistentvolumeclaims: - enabled: true - ingresses: - enabled: false - ingressclasses: {} - # By default IngressClasses sync is enabled when the Ingress sync is enabled - # but it can be explicitly disabled by setting: - # enabled: false - fake-nodes: - enabled: true # will be ignored if nodes.enabled = true - fake-persistentvolumes: - enabled: true # will be ignored if persistentvolumes.enabled = true - nodes: - fakeKubeletIPs: true - enabled: false - # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster - # will sync all nodes instead of only the ones where some pods are running. - syncAllNodes: false - # nodeSelector is used to limit which nodes get synced to the vcluster, - # and which nodes are used to run vcluster pods. - # A valid string representation of a label selector must be used. - nodeSelector: "" - # if true, vcluster will run with a scheduler and node changes are possible - # from within the virtual cluster. This is useful if you would like to - # taint, drain and label nodes from within the virtual cluster - enableScheduler: false - # DEPRECATED: use enable scheduler instead - # syncNodeChanges allows vcluster user edits of the nodes to be synced down to the host nodes. - # Write permissions on node resource will be given to the vcluster. - syncNodeChanges: false - persistentvolumes: - enabled: false - storageclasses: - enabled: false - # formerly named - "legacy-storageclasses" - hoststorageclasses: - enabled: false - priorityclasses: - enabled: false - networkpolicies: - enabled: false - volumesnapshots: - enabled: false - poddisruptionbudgets: - enabled: false - serviceaccounts: - enabled: false - # generic CRD configuration - generic: - config: |- - --- - -# If enabled, will fallback to host dns for resolving domains. This -# is useful if using istio or dapr in the host cluster and sidecar -# containers cannot connect to the central instance. Its also useful -# if you want to access host cluster services from within the vcluster. -fallbackHostDns: false - -# Map Services between host and virtual cluster -mapServices: - # Services that should get mapped from the - # virtual cluster to the host cluster. - # vcluster will make sure to sync the service - # ip to the host cluster automatically as soon - # as the service exists. - # For example: - # fromVirtual: - # - from: my-namespace/name - # to: host-service - fromVirtual: [] - # Same as from virtual, but instead sync services - # from the host cluster into the virtual cluster. - # If the namespace does not exist, vcluster will - # also create the namespace for the service. - fromHost: [] - -# Proxy metrics server from host to virtual -proxy: - metricsServer: - nodes: - enabled: false - pods: - enabled: false - -# Syncer configuration -syncer: - replicas: 1 - # Image to use for the syncer - # image: ghcr.io/loft-sh/vcluster - extraArgs: [] - imagePullPolicy: "" - env: [] - livenessProbe: - enabled: true - readinessProbe: - enabled: true - volumeMounts: - - mountPath: /data - name: data - # readOnly: true - extraVolumeMounts: [] - resources: - limits: - ephemeral-storage: 8Gi - memory: 2Gi - requests: - ephemeral-storage: 200Mi - cpu: 200m - memory: 256Mi - kubeConfigContextName: "my-vcluster" - serviceAnnotations: {} - # Storage settings for the vcluster - storage: - # If this is disabled, vcluster will use an emptyDir instead - # of a PersistentVolumeClaim - persistence: true - # Size of the persistent volume claim - size: 5Gi - # Optional StorageClass used for the pvc - # if empty default StorageClass defined in your host cluster will be used - #className: - -# Virtual Cluster (k3s) configuration -vcluster: - # Image to use for the virtual cluster - image: rancher/k3s:v1.29.0-k3s1 - imagePullPolicy: "" - command: - - /binaries/k3s - baseArgs: - - server - - --write-kubeconfig=/data/k3s-config/kube-config.yaml - - --data-dir=/data - - --disable=traefik,servicelb,metrics-server,local-storage,coredns - - --disable-network-policy - - --disable-agent - - --disable-cloud-controller - - --egress-selector-mode=disabled - - --flannel-backend=none - - --kube-apiserver-arg=bind-address=127.0.0.1 - extraArgs: [] - # Deprecated: Use syncer.extraVolumeMounts instead - extraVolumeMounts: [] - # Deprecated: Use syncer.volumeMounts instead - volumeMounts: [] - # Deprecated: Use syncer.env instead - env: [] - resources: - limits: - cpu: 100m - memory: 256Mi - requests: - cpu: 40m - memory: 64Mi - -# Embedded etcd settings -embeddedEtcd: - # If embedded etcd should be enabled, this is a PRO only feature - enabled: false - -# list of {validating/mutating}webhooks that the syncer should proxy. -# This is a PRO only feature. -centralAdmission: - validatingWebhooks: [] - mutatingWebhooks: [] - - -# Extra volumes that should be created for the StatefulSet -volumes: [] - -# Service account that should be used by the vcluster -serviceAccount: - create: true - # Optional name of the service account to use - # name: default - # Optional pull secrets - # imagePullSecrets: - # - name: my-pull-secret - -# Service account that should be used by the pods synced by vcluster -workloadServiceAccount: - # This is not supported in multi-namespace mode - annotations: {} - -# Roles & ClusterRoles for the vcluster -rbac: - clusterRole: - # You don't need to toggle this as necessary cluster roles are created based on the enabled syncers (.sync.*.enabled). - # This only makes sense to enable if you want to use the extraRules below. - create: false - # Extra Rules for the cluster role - extraRules: [] - role: - # Disable this only if you don't want vCluster to create a role. This will break most functionality if disabled. - create: true - # Extra Rules for the role - extraRules: [] - # all entries in excludedApiResources will be excluded from the Role created for vcluster - excludedApiResources: - # - pods/exec - -# If vCluster persistent volume claims should get deleted automatically. -autoDeletePersistentVolumeClaims: false - -# NodeSelector used to schedule the vcluster -nodeSelector: {} - -# Affinity to apply to the vcluster statefulset -affinity: {} - -# PriorityClassName to apply to the vcluster statefulset -priorityClassName: "" - -# Tolerations to apply to the vcluster statefulset -tolerations: [] - -# Extra Labels for the stateful set -labels: {} -podLabels: {} - -# Extra Annotations for the stateful set -annotations: {} -podAnnotations: {} - -# The k3s token to use. If empty will generate one automatically -k3sToken: "" - -# Service configurations -service: - type: ClusterIP - - # Optional configuration - # A list of IP addresses for which nodes in the cluster will also accept traffic for this service. - # These IPs are not managed by Kubernetes; e.g., an external load balancer. - externalIPs: [] - - # Optional configuration for LoadBalancer & NodePort service types - # Route external traffic to node-local or cluster-wide endpoints [ Local | Cluster ] - externalTrafficPolicy: "" - - # Optional configuration for LoadBalancer service type - # Specify IP of load balancer to be created - loadBalancerIP: "" - # CIDR block(s) for the service allowlist - loadBalancerSourceRanges: [] - # Set the loadBalancerClass if using an external load balancer controller - loadBalancerClass: "" - # Set loadBalancer specific annotations on the Kubernetes service - loadBalancerAnnotations: {} - -# Configure the ingress resource that allows you to access the vcluster -ingress: - # Enable ingress record generation - enabled: false - # Ingress path type - pathType: ImplementationSpecific - ingressClassName: "" - host: vcluster.local - annotations: - nginx.ingress.kubernetes.io/backend-protocol: HTTPS - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - # Ingress TLS configuration - tls: [] - # - secretName: tls-vcluster.local - # hosts: - # - vcluster.local - -# Configure SecurityContext of the containers in the VCluster pod -securityContext: - allowPrivilegeEscalation: false - # capabilities: - # drop: - # - all - # readOnlyRootFilesystem will be set to true by default at a later release - # currently leaving it undefined for backwards compatibility with older vcluster cli versions - # readOnlyRootFilesystem: true - - # To run vcluster pod as non-root uncomment runAsUser and runAsNonRoot values. - # Update the runAsUser value if your cluster has limitations on user UIDs. - # For installation on OpenShift leave the runAsUser undefined (commented out). - # runAsUser: 12345 - # runAsNonRoot: true - runAsUser: 0 - runAsGroup: 0 - -# PodSecurityContext holds pod-level security attributes and common container settings for the vCluster pod -podSecurityContext: {} - -# Set "enable" to true when running vcluster in an OpenShift host -# This will add an extra rule to the deployed role binding in order -# to manage service endpoints -openshift: - enable: false - -# If enabled will deploy the coredns configmap -coredns: - # If CoreDns is enabled - enabled: true - # Pro only feature - integrated: false - # only used if isolation is enabled - fallback: 8.8.8.8 - plugin: - enabled: false - config: [] - # example configuration for plugin syntax, will be documented in detail - # - record: - # fqdn: google.com - # target: - # mode: url - # url: google.co.in - # - record: - # service: my-namespace/my-svc # dns-test/nginx-svc - # target: - # mode: host - # service: dns-test/nginx-svc - # - record: - # service: my-namespace-lb/my-svc-lb - # target: - # mode: host - # service: dns-test-exposed-lb/nginx-svc-exposed-lb - # - record: - # service: my-ns-external-name/my-svc-external-name - # target: - # mode: host - # service: dns-test-external-name/nginx-svc-external-name - # - record: - # service: my-ns-in-vcluster/my-svc-vcluster - # target: - # mode: vcluster # can be tested only manually for now - # vcluster: test-vcluster-ns/test-vcluster - # service: dns-test-in-vcluster-ns/test-in-vcluster-service - # - record: - # service: my-ns-in-vcluster-mns/my-svc-mns - # target: - # mode: vcluster # can be tested only manually for now - # service: dns-test-in-vcluster-mns/test-in-vcluster-svc-mns - # vcluster: test-vcluster-ns-mns/test-vcluster-mns - # - record: - # service: my-self-vc-ns/my-self-vc-svc - # target: - # mode: self - # service: dns-test/nginx-svc - replicas: 1 - # The nodeSelector example below specifices that coredns should only be scheduled to nodes with the arm64 label - # nodeSelector: - # kubernetes.io/arch: arm64 - # image: my-core-dns-image:latest - # config: |- - # .:1053 { - # ... - # CoreDNS service configurations - service: - type: ClusterIP - # Configuration for LoadBalancer service type - externalIPs: [] - externalTrafficPolicy: "" - # Extra Annotations - annotations: {} - resources: - limits: - cpu: 1000m - memory: 170Mi - requests: - cpu: 3m - memory: 16Mi -# if below option is configured, it will override the coredns manifests with the following string -# manifests: |- -# apiVersion: ... -# ... - podAnnotations: {} - podLabels: {} - -# If enabled will deploy vcluster in an isolated mode with pod security -# standards, limit ranges and resource quotas -isolation: - enabled: false - namespace: null - - podSecurityStandard: baseline - - # If enabled will add node/proxy permission to the cluster role - # in isolation mode - nodeProxyPermission: - enabled: false - - resourceQuota: - enabled: true - quota: - requests.cpu: 10 - requests.memory: 20Gi - requests.storage: "100Gi" - requests.ephemeral-storage: 60Gi - limits.cpu: 20 - limits.memory: 40Gi - limits.ephemeral-storage: 160Gi - services.nodeports: 0 - services.loadbalancers: 1 - count/endpoints: 40 - count/pods: 20 - count/services: 20 - count/secrets: 100 - count/configmaps: 100 - count/persistentvolumeclaims: 20 - scopeSelector: - matchExpressions: - scopes: - - limitRange: - enabled: true - default: - ephemeral-storage: 8Gi - memory: 512Mi - cpu: "1" - defaultRequest: - ephemeral-storage: 3Gi - memory: 128Mi - cpu: 100m - - networkPolicy: - enabled: true - outgoingConnections: - ipBlock: - cidr: 0.0.0.0/0 - except: - - 100.64.0.0/10 - - 127.0.0.0/8 - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - -# manifests to setup when initializing a vcluster -init: - manifests: |- - --- - # The contents of manifests-template will be templated using helm - # this allows you to use helm values inside, e.g.: {{ .Release.Name }} - manifestsTemplate: '' - helm: [] - # - bundle: - base64-encoded .tar.gz file content (optional - overrides chart.repo) - # chart: - # name: REQUIRED - # version: REQUIRED - # repo: (optional when bundle is used) - # username: (if required for repo) - # password: (if required for repo) - # insecure: boolean (if required for repo) - # release: - # name: REQUIRED - # namespace: REQUIRED - # timeout: number - # values: |- string YAML object - # foo: bar - # valuesTemplate: |- string YAML object - # foo: {{ .Release.Name }} - -multiNamespaceMode: - enabled: false - -telemetry: - disabled: false - instanceCreator: "helm" - platformUserID: "" - platformInstanceID: "" - machineID: "" - -noopSyncer: - enabled: false - synck8sService: false - - # Secret containing remote cluster configuration - # (should have the following keys defined) - # serverCaCert: - # serverCaKey: - # clientCaCert: - # requestHeaderCaCert: - # kubeConfig: - # - #secret: "" - diff --git a/charts/k8s/.helmignore b/charts/k8s/.helmignore deleted file mode 100644 index b6a3eb5b3..000000000 --- a/charts/k8s/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store - -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ - -# Common backup files -*.swp -*.bak -*.tmp -*~ - -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/k8s/Chart.yaml b/charts/k8s/Chart.yaml deleted file mode 100644 index 2eda61d64..000000000 --- a/charts/k8s/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v2 -name: vcluster-k8s -description: vcluster - Virtual Kubernetes Clusters (k8s) -home: https://vcluster.com -icon: https://static.loft.sh/branding/logos/vcluster/vertical/vcluster_vertical.svg -keywords: - - developer - - development - - sharing - - share - - multi-tenancy - - tenancy - - cluster - - space - - namespace - - vcluster - - vclusters -maintainers: - - name: Loft Labs, Inc. - email: info@loft.sh - url: https://twitter.com/loft_sh -sources: - - https://github.com/loft-sh/vcluster -type: application - -version: 0.0.1 # version is auto-generated by release pipeline diff --git a/charts/k8s/README.md b/charts/k8s/README.md deleted file mode 100644 index 48207ef35..000000000 --- a/charts/k8s/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -# vcluster (k8s) - -## **[GitHub](https://github.com/loft-sh/vcluster)** • **[Website](https://www.vcluster.com)** • **[Quickstart](https://www.vcluster.com/docs/getting-started/setup)** • **[Documentation](https://www.vcluster.com/docs/what-are-virtual-clusters)** • **[Blog](https://loft.sh/blog)** • **[Twitter](https://twitter.com/loft_sh)** • **[Slack](https://slack.loft.sh/)** - -Create fully functional virtual Kubernetes clusters - Each vcluster runs inside a namespace of the underlying k8s cluster. It's cheaper than creating separate full-blown clusters and it offers better multi-tenancy and isolation than regular namespaces. - -## Prerequisites - -- Kubernetes 1.18+ -- Helm 3+ - -## Get Helm Repository Info - -```bash -helm repo add loft-sh https://charts.loft.sh -helm repo update -``` - -See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation. - -## Install Helm Chart - -```bash -helm upgrade [RELEASE_NAME] loft-sh/vcluster-k8s -n [RELEASE_NAMESPACE] --create-namespace --install -``` - -See [vcluster docs](https://vcluster.com/docs) for configuration options. - -See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation. - -## Connect to the vcluster - -In order to connect to the installed vcluster, please install [vcluster cli](https://www.vcluster.com/docs/getting-started/setup) and run: - -```bash -vcluster connect [RELEASE_NAME] -n [RELEASE_NAMESPACE] -``` - -## Uninstall Helm Chart - -```bash -helm uninstall [RELEASE_NAME] -``` - -This removes all the Kubernetes components associated with the chart and deletes the release. - -See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation. - -### Why Virtual Kubernetes Clusters? - -- **Cluster Scoped Resources**: much more powerful than simple namespaces (virtual clusters allow users to use CRDs, namespaces, cluster roles etc.) -- **Ease of Use**: usable in any Kubernetes cluster and created in seconds either via a single command or [cluster-api](https://github.com/loft-sh/cluster-api-provider-vcluster) -- **Cost Efficient**: much cheaper and efficient than "real" clusters (single pod and shared resources just like for namespaces) -- **Lightweight**: built upon the ultra-fast k3s distribution with minimal overhead per virtual cluster (other distributions work as well) -- **Strict isolation**: complete separate Kubernetes control plane and access point for each vcluster while still being able to share certain services of the underlying host cluster -- **Cluster Wide Permissions**: allow users to install apps which require cluster-wide permissions while being limited to actually just one namespace within the host cluster -- **Great for Testing**: allow you to test different Kubernetes versions inside a single host cluster which may have a different version than the virtual clusters - -Learn more on [www.vcluster.com](https://vcluster.com). - -![vcluster Intro](https://github.com/loft-sh/vcluster/raw/main/docs/static/media/vcluster-comparison.png) - -Learn more in the [documentation](https://vcluster.com/docs/what-are-virtual-clusters). diff --git a/charts/k8s/templates/NOTES.txt b/charts/k8s/templates/NOTES.txt deleted file mode 100644 index 6cf960d62..000000000 --- a/charts/k8s/templates/NOTES.txt +++ /dev/null @@ -1,10 +0,0 @@ -Thank you for installing vcluster. - -Your vcluster is named {{ .Release.Name }} in namespace {{ .Release.Namespace }}. - -To connect to the vcluster, use vcluster CLI (https://www.vcluster.com/docs/getting-started/setup): - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} - $ vcluster connect {{ .Release.Name }} -n {{ .Release.Namespace }} -- kubectl get ns - - -For more information, please take a look at the vcluster docs at https://www.vcluster.com/docs diff --git a/charts/k8s/templates/_coredns.tpl b/charts/k8s/templates/_coredns.tpl deleted file mode 100644 index f5b1aff93..000000000 --- a/charts/k8s/templates/_coredns.tpl +++ /dev/null @@ -1,52 +0,0 @@ -{{/* - Define a common coredns config -*/}} -{{- define "vcluster.corefile" -}} -Corefile: |- - {{- if .Values.coredns.config }} -{{ .Values.coredns.config | indent 8 }} - {{- else }} - .:1053 { - errors - health - ready - rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local - kubernetes cluster.local in-addr.arpa ip6.arpa { - {{- if .Values.pro }} - {{- if .Values.coredns.integrated }} - kubeconfig /pki/admin.conf - {{- end }} - {{- end }} - pods insecure - {{- if .Values.fallbackHostDns }} - fallthrough cluster.local in-addr.arpa ip6.arpa - {{- else }} - fallthrough in-addr.arpa ip6.arpa - {{- end }} - } - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - vcluster {{ toYaml .Values.coredns.plugin.config | b64enc }} - {{- end }} - hosts /etc/NodeHosts { - ttl 60 - reload 15s - fallthrough - } - prometheus :9153 - {{- if .Values.fallbackHostDns }} - forward . {{`{{.HOST_CLUSTER_DNS}}`}} - {{- else if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} - forward . /etc/resolv.conf {{ .Values.coredns.fallback }} { - policy sequential - } - {{- else }} - forward . /etc/resolv.conf - {{- end }} - cache 30 - loop - loadbalance - } - - import /etc/coredns/custom/*.server - {{- end }} -{{- end -}} diff --git a/charts/k8s/templates/_helpers.tpl b/charts/k8s/templates/_helpers.tpl deleted file mode 100644 index 50f08d6f9..000000000 --- a/charts/k8s/templates/_helpers.tpl +++ /dev/null @@ -1,196 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "vcluster.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Whether the ingressclasses syncer should be enabled -*/}} -{{- define "vcluster.syncIngressclassesEnabled" -}} -{{- if or - (.Values.sync.ingressclasses).enabled - (and - .Values.sync.ingresses.enabled - (not .Values.sync.ingressclasses)) -}} - {{- true -}} -{{- end -}} -{{- end -}} - -{{/* -Whether to create a cluster role or not -*/}} -{{- define "vcluster.createClusterRole" -}} -{{- if or - (not (empty (include "vcluster.serviceMapping.fromHost" . ))) - (not (empty (include "vcluster.plugin.clusterRoleExtraRules" . ))) - (not (empty (include "vcluster.generic.clusterRoleExtraRules" . ))) - .Values.rbac.clusterRole.create - .Values.sync.hoststorageclasses.enabled - (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") - (include "vcluster.syncIngressclassesEnabled" . ) - .Values.pro - .Values.sync.nodes.enabled - .Values.sync.persistentvolumes.enabled - .Values.sync.storageclasses.enabled - .Values.sync.priorityclasses.enabled - .Values.sync.volumesnapshots.enabled - .Values.proxy.metricsServer.nodes.enabled - .Values.multiNamespaceMode.enabled - .Values.coredns.plugin.enabled -}} -{{- true -}} -{{- end -}} -{{- end -}} - -{{- define "vcluster.clusterRoleName" -}} -{{- printf "vc-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "vcluster.clusterRoleNameMultinamespace" -}} -{{- printf "vc-mn-%s-v-%s" .Release.Name .Release.Namespace | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Syncer flags for enabling/disabling controllers -Prints only the flags that modify the defaults: -- when default controller has enabled: false => `- "--sync=-controller` -- when non-default controller has enabled: true => `- "--sync=controller` -*/}} -{{- define "vcluster.syncer.syncArgs" -}} -{{- $defaultEnabled := list "services" "configmaps" "secrets" "endpoints" "pods" "events" "persistentvolumeclaims" "fake-nodes" "fake-persistentvolumes" -}} -{{- if and (hasKey .Values.sync.nodes "enableScheduler") .Values.sync.nodes.enableScheduler -}} - {{- $defaultEnabled = concat $defaultEnabled (list "csinodes" "csidrivers" "csistoragecapacities" ) -}} -{{- end -}} -{{- range $key, $val := .Values.sync }} -{{- if and (has $key $defaultEnabled) (not $val.enabled) }} -- --sync=-{{ $key }} -{{- else if and (not (has $key $defaultEnabled)) ($val.enabled)}} -{{- if eq $key "legacy-storageclasses" }} -- --sync=hoststorageclasses -{{- else }} -- --sync={{ $key }} -{{- end -}} -{{- end -}} -{{- end -}} -{{- if not (include "vcluster.syncIngressclassesEnabled" . ) }} -- --sync=-ingressclasses -{{- end -}} -{{- end -}} - -{{/* - Cluster role rules defined by plugins -*/}} -{{- define "vcluster.plugin.clusterRoleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.clusterRole }} -{{- if $container.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.clusterRoleExtraRules" -}} -{{- if .Values.sync.generic.clusterRole }} -{{- if .Values.sync.generic.clusterRole.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Cluster role rules defined on global level -*/}} -{{- define "vcluster.rbac.clusterRoleExtraRules" -}} -{{- if .Values.rbac.clusterRole.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.clusterRole.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - - -{{/* - Role rules defined on global level -*/}} -{{- define "vcluster.rbac.roleExtraRules" -}} -{{- if .Values.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := .Values.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined by plugins -*/}} -{{- define "vcluster.plugin.roleExtraRules" -}} -{{- range $key, $container := .Values.plugin }} -{{- if $container.rbac }} -{{- if $container.rbac.role }} -{{- if $container.rbac.role.extraRules }} -{{- range $ruleIndex, $rule := $container.rbac.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Role rules defined in generic syncer -*/}} -{{- define "vcluster.generic.roleExtraRules" -}} -{{- if .Values.sync.generic.role }} -{{- if .Values.sync.generic.role.extraRules}} -{{- range $ruleIndex, $rule := .Values.sync.generic.role.extraRules }} -- {{ toJson $rule }} -{{- end }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Virtual cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromVirtual" -}} -{{- range $key, $value := .Values.mapServices.fromVirtual }} -- '--map-virtual-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} - -{{/* - Host cluster service mapping -*/}} -{{- define "vcluster.serviceMapping.fromHost" -}} -{{- range $key, $value := .Values.mapServices.fromHost }} -- '--map-host-service={{ $value.from }}={{ $value.to }}' -{{- end }} -{{- end -}} diff --git a/charts/k8s/templates/_kind.tpl b/charts/k8s/templates/_kind.tpl deleted file mode 100644 index 23009704f..000000000 --- a/charts/k8s/templates/_kind.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* - deployment kind -*/}} -{{- define "vcluster.kind" -}} -{{ if and .Values.embeddedEtcd.enabled .Values.pro }}StatefulSet{{ else }}Deployment{{ end }} -{{- end -}} - -{{/* - service name for statefulset -*/}} -{{- define "vcluster.statefulset.serviceName" }} -{{- if .Values.embeddedEtcd.enabled }} -serviceName: {{ .Release.Name }}-headless -{{- end }} -{{- end -}} - -{{/* - volumeClaimTemplate -*/}} -{{- define "vcluster.statefulset.volumeClaimTemplate" }} -{{- if .Values.embeddedEtcd.enabled }} -{{- if .Values.autoDeletePersistentVolumeClaims }} -{{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} -persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete -{{- end }} -{{- end }} -{{- if (hasKey .Values "volumeClaimTemplates") }} -volumeClaimTemplates: -{{ toYaml .Values.volumeClaimTemplates | indent 4 }} -{{- else if .Values.syncer.storage.persistence }} -volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if .Values.syncer.storage.className }} - storageClassName: {{ .Values.syncer.storage.className }} - {{- end }} - resources: - requests: - storage: {{ .Values.syncer.storage.size }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - deployment strategy -*/}} -{{- define "vcluster.deployment.strategy" }} -{{- if not .Values.embeddedEtcd.enabled }} -strategy: - rollingUpdate: - maxSurge: 1 - {{- if (eq (int .Values.syncer.replicas) 1) }} - maxUnavailable: 0 - {{- else }} - maxUnavailable: 1 - {{- end }} - type: RollingUpdate -{{- end }} -{{- end -}} diff --git a/charts/k8s/templates/_plugin.tpl b/charts/k8s/templates/_plugin.tpl deleted file mode 100644 index b1ecd2fa2..000000000 --- a/charts/k8s/templates/_plugin.tpl +++ /dev/null @@ -1,188 +0,0 @@ -{{/* - Plugin config definition -*/}} -{{- define "vcluster.plugins.config" -}} -{{- $pluginFound := false -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} -{{- $pluginFound = true -}} -{{- end }} -{{- if $pluginFound }} -- name: PLUGIN_CONFIG - value: |- -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.config) }} -{{- continue }} -{{- end }} - {{ $key }}: {{ toYaml $container.config | nindent 6 }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume mount definition -*/}} -{{- define "vcluster.plugins.volumeMounts" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- mountPath: /plugins - name: plugins -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin volume definition -*/}} -{{- define "vcluster.plugins.volumes" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- name: plugins - emptyDir: {} -{{- break }} -{{- end }} -{{- end -}} - -{{/* - Plugin init container definition -*/}} -{{- define "vcluster.plugins.initContainers" -}} -{{- range $key, $container := .Values.plugin }} -{{- if or (ne $container.version "v2") (not $container.image) }} -{{- continue }} -{{- end }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if or $container.command $container.args }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- else }} - command: ["sh"] - args: ["-c", "cp -r /plugin /plugins/{{ $key }}"] - {{- end }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - {{- if $container.volumeMounts }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- else }} - volumeMounts: - - mountPath: /plugins - name: plugins - {{- end }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} -{{- end }} -{{- end -}} - -{{/* - Extra Syncer Args for the legacy Plugins -*/}} -{{- define "vcluster.legacyPlugins.args" -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{- continue }} -{{- end }} -{{- if not $container.optional }} -- --plugins={{ $key }} -{{- end }} -{{- end }} -{{- end -}} - -{{/* - Sidecar container definition for the legacy syncer parts -*/}} -{{- define "vcluster.legacyPlugins.containers" -}} -{{- $counter := -1 -}} -{{- range $key, $container := .Values.plugin }} -{{- if eq $container.version "v2" }} -{{ continue }} -{{- end }} -{{- $counter = add1 $counter }} -- image: {{ $.Values.defaultImageRegistry }}{{ $container.image }} - {{- if $container.name }} - name: {{ $container.name | quote }} - {{- else }} - name: {{ $key | quote }} - {{- end }} - {{- if $container.imagePullPolicy }} - imagePullPolicy: {{ $container.imagePullPolicy }} - {{- end }} - {{- if $container.workingDir }} - workingDir: {{ $container.workingDir }} - {{- end }} - {{- if $container.command }} - command: - {{- range $commandIndex, $command := $container.command }} - - {{ $command | quote }} - {{- end }} - {{- end }} - {{- if $container.args }} - args: - {{- range $argIndex, $arg := $container.args }} - - {{ $arg | quote }} - {{- end }} - {{- end }} - {{- if $container.terminationMessagePath }} - terminationMessagePath: {{ $container.terminationMessagePath }} - {{- end }} - {{- if $container.terminationMessagePolicy }} - terminationMessagePolicy: {{ $container.terminationMessagePolicy }} - {{- end }} - env: - - name: VCLUSTER_PLUGIN_ADDRESS - value: "localhost:{{ add 14000 $counter }}" - - name: VCLUSTER_PLUGIN_NAME - value: "{{ $key }}" - {{- if $container.env }} -{{ toYaml $container.env | indent 4 }} - {{- end }} - envFrom: -{{ toYaml $container.envFrom | indent 4 }} - securityContext: -{{ toYaml $container.securityContext | indent 4 }} - lifecycle: -{{ toYaml $container.lifecycle | indent 4 }} - livenessProbe: -{{ toYaml $container.livenessProbe | indent 4 }} - readinessProbe: -{{ toYaml $container.readinessProbe | indent 4 }} - startupProbe: -{{ toYaml $container.startupProbe | indent 4 }} - volumeDevices: -{{ toYaml $container.volumeDevices | indent 4 }} - volumeMounts: -{{ toYaml $container.volumeMounts | indent 4 }} - {{- if $container.resources }} - resources: -{{ toYaml $container.resources | indent 4 }} - {{- end }} - {{- end }} -{{- end }} - - diff --git a/charts/k8s/templates/coredns.yaml b/charts/k8s/templates/coredns.yaml deleted file mode 100644 index cc25dd17f..000000000 --- a/charts/k8s/templates/coredns.yaml +++ /dev/null @@ -1,231 +0,0 @@ -{{- if not .Values.headless }} -{{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-coredns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{- if .Values.coredns.manifests }} - coredns.yaml: |- -{{ .Values.coredns.manifests | indent 4 }} -{{- else }} - coredns.yaml: |- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: coredns - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns - subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: -{{ include "vcluster.corefile" . | indent 6 }} - NodeHosts: "" - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/name: "CoreDNS" - spec: - replicas: {{ .Values.coredns.replicas }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - {{- if .Values.coredns.podAnnotations }} - annotations: -{{ toYaml .Values.coredns.podAnnotations | indent 12 }} - {{- end }} - labels: - k8s-app: kube-dns - {{- range $k, $v := .Values.coredns.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - priorityClassName: "system-cluster-critical" - serviceAccountName: coredns - nodeSelector: - kubernetes.io/os: linux - {{- if .Values.coredns.nodeSelector }} -{{ toYaml .Values.coredns.nodeSelector | indent 12 }} - {{- end }} - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - k8s-app: kube-dns - {{- if .Values.isolation.enabled }} - securityContext: - seccompProfile: - type: RuntimeDefault - {{- end }} - containers: - - name: coredns - {{- if .Values.coredns.image }} - image: {{ .Values.defaultImageRegistry }}{{ .Values.coredns.image }} - {{- else }} - image: {{`{{.IMAGE}}`}} - {{- end }} - imagePullPolicy: IfNotPresent - resources: -{{ toYaml .Values.coredns.resources | indent 16}} - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - - name: custom-config-volume - mountPath: /etc/coredns/custom - readOnly: true - securityContext: - runAsNonRoot: true - runAsUser: {{`{{.RUN_AS_USER}}`}} - runAsGroup: {{`{{.RUN_AS_GROUP}}`}} - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 2 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 3 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile - - key: NodeHosts - path: NodeHosts - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - selector: - k8s-app: kube-dns - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/k8s/templates/etcd-service.yaml b/charts/k8s/templates/etcd-service.yaml deleted file mode 100644 index cbef5e804..000000000 --- a/charts/k8s/templates/etcd-service.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if or (and (not .Values.embeddedEtcd.enabled) (not .Values.headless) (not .Values.etcd.disabled)) .Values.embeddedEtcd.migrateFromEtcd }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-etcd - namespace: {{ .Release.Namespace }} - labels: - app: vcluster-etcd - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.etcd.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: ClusterIP - ports: - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - selector: - app: vcluster-etcd - release: {{ .Release.Name }} -{{- end }} diff --git a/charts/k8s/templates/etcd-statefulset-service.yaml b/charts/k8s/templates/etcd-statefulset-service.yaml deleted file mode 100644 index 3ea17586a..000000000 --- a/charts/k8s/templates/etcd-statefulset-service.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{ if or (and (not .Values.embeddedEtcd.enabled) (not .Values.headless) (not .Values.etcd.disabled)) .Values.embeddedEtcd.migrateFromEtcd }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-etcd-headless - namespace: {{ .Release.Namespace }} - labels: - app: vcluster-etcd - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.etcd.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - publishNotReadyAddresses: true - ports: - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - clusterIP: None - selector: - app: vcluster-etcd - release: "{{ .Release.Name }}" -{{ end }} diff --git a/charts/k8s/templates/etcd-statefulset.yaml b/charts/k8s/templates/etcd-statefulset.yaml deleted file mode 100644 index cd06fce5f..000000000 --- a/charts/k8s/templates/etcd-statefulset.yaml +++ /dev/null @@ -1,205 +0,0 @@ -{{ if or (and (not .Values.embeddedEtcd.enabled) (not .Values.headless ) (not .Values.etcd.disabled )) .Values.embeddedEtcd.migrateFromEtcd }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Release.Name }}-etcd - namespace: {{ .Release.Namespace }} - labels: - app: vcluster-etcd - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.etcd.labels }} -{{ toYaml .Values.etcd.labels | indent 4 }} -{{- end }} - {{- $annotations := merge .Values.globalAnnotations .Values.etcd.annotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - serviceName: {{ .Release.Name }}-etcd-headless - {{- if .Values.autoDeletePersistentVolumeClaims }} - {{- if ge (int .Capabilities.KubeVersion.Minor) 27 }} - persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete - {{- end }} - {{- end }} - replicas: {{ .Values.etcd.replicas }} - podManagementPolicy: Parallel - selector: - matchLabels: - app: vcluster-etcd - release: {{ .Release.Name }} - {{- if (hasKey .Values.etcd "volumeClaimTemplates") }} - volumeClaimTemplates: -{{ toYaml .Values.etcd.volumeClaimTemplates | indent 4 }} - {{- else if .Values.etcd.storage.persistence }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if .Values.etcd.storage.className}} - storageClassName: {{ .Values.etcd.storage.className }} - {{- end }} - resources: - requests: - storage: {{ .Values.etcd.storage.size }} - {{- end }} - template: - metadata: - {{- if .Values.etcd.podAnnotations }} - annotations: -{{ toYaml .Values.etcd.podAnnotations | indent 8 }} - {{- end }} - labels: - app: vcluster-etcd - release: {{ .Release.Name }} - {{- range $k, $v := .Values.etcd.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - terminationGracePeriodSeconds: 10 - {{- if .Values.etcd.affinity }} - affinity: -{{ toYaml .Values.etcd.affinity | indent 8 }} - {{- else }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - # if possible avoid scheduling more than one pod on one node - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster-etcd - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: "kubernetes.io/hostname" - # if possible avoid scheduling pod onto node that is in the same zone as one or more vcluster pods are running - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster-etcd - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: topology.kubernetes.io/zone - {{- end }} - {{- if .Values.etcd.topologySpreadConstraints }} - topologySpreadConstraints: -{{ toYaml .Values.etcd.topologySpreadConstraints | indent 8 }} - {{- end }} - nodeSelector: -{{ toYaml .Values.etcd.nodeSelector | indent 8 }} - tolerations: -{{ toYaml .Values.etcd.tolerations | indent 8 }} - automountServiceAccountToken: false - {{- if .Values.serviceAccount.name }} - serviceAccountName: {{ .Values.serviceAccount.name }} - {{- else }} - serviceAccountName: vc-{{ .Release.Name }} - {{- end }} - volumes: - - name: certs - secret: - secretName: {{ .Release.Name }}-certs - {{- if .Values.etcd.volumes }} -{{ toYaml .Values.etcd.volumes | indent 8 }} - {{- end }} - {{- if not .Values.etcd.storage.persistence }} - - name: data - emptyDir: {} - {{- end }} - {{- if .Values.etcd.priorityClassName }} - priorityClassName: {{ .Values.etcd.priorityClassName }} - {{- end }} - {{- if .Values.etcd.fsGroup }} - securityContext: - fsGroup: {{ .Values.etcd.fsGroup }} - {{- end }} - containers: - - name: etcd - image: "{{ .Values.defaultImageRegistry }}{{ .Values.etcd.image }}" - command: - - etcd - - '--cert-file=/run/config/pki/etcd-server.crt' - - '--client-cert-auth=true' - - '--data-dir=/var/lib/etcd' - - '--advertise-client-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2379' - - '--initial-advertise-peer-urls=https://$(NAME).{{ .Release.Name }}-etcd-headless.{{ .Release.Namespace }}:2380' - {{- $releaseName := .Release.Name -}} - {{- $releaseNamespace := .Release.Namespace }} - - '--initial-cluster={{ range $index := untilStep 0 (int .Values.etcd.replicas) 1 }}{{ if (ne (int $index) 0) }},{{ end }}{{ $releaseName }}-etcd-{{ $index }}=https://{{ $releaseName }}-etcd-{{ $index }}.{{ $releaseName }}-etcd-headless.{{ $releaseNamespace }}:2380{{ end }}' - - '--initial-cluster-token={{ .Release.Name }}' - - '--initial-cluster-state=new' - - '--listen-client-urls=https://0.0.0.0:2379' - - '--listen-metrics-urls=http://0.0.0.0:2381' - - '--listen-peer-urls=https://0.0.0.0:2380' - - '--key-file=/run/config/pki/etcd-server.key' - - '--name=$(NAME)' - - '--peer-cert-file=/run/config/pki/etcd-peer.crt' - - '--peer-client-cert-auth=true' - - '--peer-key-file=/run/config/pki/etcd-peer.key' - - '--peer-trusted-ca-file=/run/config/pki/etcd-ca.crt' - - '--snapshot-count=10000' - - '--trusted-ca-file=/run/config/pki/etcd-ca.crt' - {{- range $f := .Values.etcd.extraArgs }} - - {{ $f | quote }} - {{- end }} - securityContext: -{{ toYaml .Values.etcd.securityContext | indent 10 }} - env: - - name: NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if .Values.etcd.env }} -{{ toYaml .Values.etcd.env | indent 10 }} - {{- end }} - volumeMounts: - - name: data - mountPath: /var/lib/etcd - - mountPath: /run/config/pki - name: certs - readOnly: true - {{- if .Values.etcd.volumeMounts }} -{{ toYaml .Values.etcd.volumeMounts | indent 10 }} - {{- end }} - resources: -{{ toYaml .Values.etcd.resources | indent 10 }} - livenessProbe: - httpGet: - path: /health - port: 2381 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 15 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 8 - startupProbe: - httpGet: - path: /health - port: 2381 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 15 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 24 - {{- if .Values.etcd.imagePullPolicy }} - imagePullPolicy: {{ .Values.etcd.imagePullPolicy }} - {{- end }} -{{ end }} diff --git a/charts/k8s/templates/ingress.yaml b/charts/k8s/templates/ingress.yaml deleted file mode 100644 index e5caff5b4..000000000 --- a/charts/k8s/templates/ingress.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - {{- $annotations := merge .Values.ingress.annotations .Values.globalAnnotations }} - {{- if .Values.ingress.tls }} - {{- $annotations = omit $annotations "nginx.ingress.kubernetes.io/ssl-passthrough" }} - {{- end }} - {{- if $annotations }} - annotations: - {{ toYaml $annotations | nindent 4 }} - {{- end }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - {{- if .Values.ingress.ingressClassName }} - ingressClassName: {{ .Values.ingress.ingressClassName | quote }} - {{- end }} - rules: - - host: {{ .Values.ingress.host | quote }} - http: - paths: - - backend: - service: - name: {{ .Release.Name }} - port: - name: https - path: / - pathType: {{ .Values.ingress.pathType }} - {{- with .Values.ingress.tls }} - tls: - {{- toYaml . | nindent 4 }} - {{- end -}} -{{- end }} diff --git a/charts/k8s/templates/init-configmap.yaml b/charts/k8s/templates/init-configmap.yaml deleted file mode 100644 index 34498fd14..000000000 --- a/charts/k8s/templates/init-configmap.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- if .Values.init.manifests }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-init-manifests - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: - manifests: |- - {{ .Values.init.manifests | nindent 4 | trim }} - {{ tpl .Values.init.manifestsTemplate $ | nindent 4 | trim }} - {{- if .Values.init.helm }} - charts: |- - {{- range .Values.init.helm }} - {{- /* only render this chart entry if either of chart or bundle is defined */}} - {{- if .chart }} - - name: {{ .chart.name }} - repo: {{ .chart.repo }} - version: {{ .chart.version }} - {{- if .chart.username }} - username: {{ .chart.username }} - {{- end }} - {{- if .chart.password }} - password: {{ .chart.password }} - {{- end }} - {{- if .insecure }} - insecure: true - {{- end}} - {{- end }} - {{- if .bundle }} - bundle: {{ .bundle }} - {{- end }} - {{- if or .chart .bundle }} - {{- if or .values .valuesTemplate }} - values: |- - {{ (.values | default "") | nindent 8 | trim }} - {{ tpl (.valuesTemplate | default "") $ | nindent 8 | trim }} - {{- end}} - {{- if .release }} - timeout: {{ .timeout | default "120s" | quote }} - releaseName: {{ .release.name }} - releaseNamespace: {{ .release.namespace }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/k8s/templates/integrated-coredns.yaml b/charts/k8s/templates/integrated-coredns.yaml deleted file mode 100644 index 16907cba0..000000000 --- a/charts/k8s/templates/integrated-coredns.yaml +++ /dev/null @@ -1,64 +0,0 @@ -{{- if .Values.pro }} -{{- if .Values.coredns.integrated }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-dns - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -data: -{{ include "vcluster.corefile" . | indent 2 }} - coredns.yaml: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: - NodeHosts: "" - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - {{- if .Values.coredns.service.annotations }} -{{ toYaml .Values.coredns.service.annotations | indent 8 }} - {{- end }} - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - type: {{ .Values.coredns.service.type }} - {{- if (eq (.Values.coredns.service.type) "LoadBalancer") }} - {{- if .Values.coredns.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.coredns.service.externalTrafficPolicy }} - {{- end }} - {{- if .Values.coredns.service.externalIPs }} - externalIPs: - {{- range $f := .Values.coredns.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- end }} - ports: - - name: dns - port: 53 - targetPort: 1053 - protocol: UDP - - name: dns-tcp - port: 53 - targetPort: 1053 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP -{{- end }} -{{- end }} diff --git a/charts/k8s/templates/limitrange.yaml b/charts/k8s/templates/limitrange.yaml deleted file mode 100644 index ed219d063..000000000 --- a/charts/k8s/templates/limitrange.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.limitRange.enabled }} -apiVersion: v1 -kind: LimitRange -metadata: - name: {{ .Release.Name }}-limit-range - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - limits: - - default: - {{- range $key, $val := .Values.isolation.limitRange.default }} - {{ $key }}: {{ $val | quote }} - {{- end }} - defaultRequest: - {{- range $key, $val := .Values.isolation.limitRange.defaultRequest }} - {{ $key }}: {{ $val | quote }} - {{- end }} - type: Container -{{- end }} diff --git a/charts/k8s/templates/networkpolicy.yaml b/charts/k8s/templates/networkpolicy.yaml deleted file mode 100644 index 81886898b..000000000 --- a/charts/k8s/templates/networkpolicy.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-workloads - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - egress: - # Allows outgoing connections to the vcluster control plane - - ports: - - port: 443 - - port: 8443 - to: - - podSelector: - matchLabels: - release: {{ .Release.Name }} - # Allows outgoing connections to DNS server - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Allows outgoing connections to the internet or - # other vcluster workloads - - to: - - podSelector: - matchLabels: - vcluster.loft.sh/managed-by: {{ .Release.Name }} - - ipBlock: - cidr: {{ .Values.isolation.networkPolicy.outgoingConnections.ipBlock.cidr }} - except: - {{- range .Values.isolation.networkPolicy.outgoingConnections.ipBlock.except }} - - {{ . }} - {{- end }} - policyTypes: - - Egress ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ .Release.Name }}-control-plane - namespace: {{ .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - podSelector: - matchLabels: - release: {{ .Release.Name }} - egress: - # Allows outgoing connections to all pods with - # port 443, 8443 or 6443. This is needed for host Kubernetes - # access - - ports: - - port: 443 - - port: 8443 - - port: 6443 - # Allows outgoing connections to all vcluster workloads - # or kube system dns server - - to: - - podSelector: {} - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: 'kube-system' - podSelector: - matchLabels: - k8s-app: kube-dns - policyTypes: - - Egress -{{- end }} \ No newline at end of file diff --git a/charts/k8s/templates/rbac/clusterrole.yaml b/charts/k8s/templates/rbac/clusterrole.yaml deleted file mode 100644 index 984267cae..000000000 --- a/charts/k8s/templates/rbac/clusterrole.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: -{{- if .Values.pro }} - - apiGroups: ["cluster.loft.sh", "storage.loft.sh"] - resources: ["features", "virtualclusters"] - verbs: ["get", "list", "watch"] -{{- end }} - {{- if or .Values.pro .Values.sync.nodes.enabled }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: [ "pods", "nodes/metrics", "nodes/stats"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.coredns.plugin.enabled }} - - apiGroups: [""] - resources: [ "pods"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled (or (not .Values.isolation.enabled) (and .Values.isolation.nodeProxyPermission.enabled .Values.isolation.enabled)) }} - - apiGroups: [""] - resources: ["nodes/proxy"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if and .Values.sync.nodes.enabled .Values.sync.nodes.syncNodeChanges }} - - apiGroups: [""] - resources: ["nodes", "nodes/status"] - verbs: ["update", "patch"] - {{- end }} - {{- if .Values.sync.persistentvolumes.enabled }} - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if .Values.sync.nodes.enableScheduler }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses","csinodes","csidrivers","csistoragecapacities"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if (include "vcluster.syncIngressclassesEnabled" . ) }} - - apiGroups: ["networking.k8s.io"] - resources: ["ingressclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.storageclasses.enabled }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - {{- end }} - {{- if or .Values.sync.hoststorageclasses.enabled (index ((index .Values.sync "legacy-storageclasses") | default (dict "enabled" false)) "enabled") .Values.rbac.clusterRole.create }} - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.sync.priorityclasses.enabled }} - - apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if (not (empty (include "vcluster.serviceMapping.fromHost" . ))) }} - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["get", "watch", "list"] - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.proxy.metricsServer.nodes.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["nodes"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.clusterRoleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.clusterRoleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/k8s/templates/rbac/clusterrolebinding.yaml b/charts/k8s/templates/rbac/clusterrolebinding.yaml deleted file mode 100644 index 0a5645d28..000000000 --- a/charts/k8s/templates/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if (include "vcluster.createClusterRole" . ) -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "vcluster.clusterRoleName" . }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ template "vcluster.clusterRoleName" . }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/k8s/templates/rbac/role.yaml b/charts/k8s/templates/rbac/role.yaml deleted file mode 100644 index de05bbb62..000000000 --- a/charts/k8s/templates/rbac/role.yaml +++ /dev/null @@ -1,104 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRole -{{- else -}} -kind: Role -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} -{{- end }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -rules: - {{- if .Values.pro }} - - apiGroups: [""] - {{- $resources := list "configmaps" "secrets" "services" "pods" "pods/attach" "pods/portforward" "pods/exec" "persistentvolumeclaims" }} - {{- range $excluded := .Values.rbac.role.excludedApiResources }} - {{- $resources = without $resources $excluded }} - {{- end}} - resources: {{ $resources | toJson }} - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- else }} - - apiGroups: [""] - resources: ["configmaps", "secrets", "services", "pods", "pods/attach", "pods/portforward", "pods/exec", "persistentvolumeclaims"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.status }} - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.pods.ephemeralContainers }} - - apiGroups: [""] - resources: ["pods/ephemeralcontainers"] - verbs: ["patch", "update"] - {{- end }} - {{- if or .Values.sync.endpoints.enabled .Values.headless }} - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["create", "delete", "patch", "update"] - {{- end }} - {{- if gt (int .Values.syncer.replicas) 1 }} - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: [""] - resources: ["endpoints", "events", "pods/log"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.ingresses.enabled}} - - apiGroups: ["networking.k8s.io"] - resources: ["ingresses"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "deployments"] - verbs: ["get", "list", "watch"] - {{- if .Values.sync.networkpolicies.enabled }} - - apiGroups: ["networking.k8s.io"] - resources: ["networkpolicies"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.volumesnapshots.enabled }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.serviceaccounts.enabled }} - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.sync.poddisruptionbudgets.enabled }} - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - {{- end }} - {{- if .Values.openshift.enable }} - {{- if .Values.sync.endpoints.enabled }} - - apiGroups: [""] - resources: ["endpoints/restricted"] - verbs: ["create"] - {{- end }} - {{- end }} - {{- if .Values.proxy.metricsServer.pods.enabled }} - - apiGroups: ["metrics.k8s.io"] - resources: ["pods"] - verbs: ["get", "list"] - {{- end }} - {{- include "vcluster.plugin.roleExtraRules" . | indent 2 }} - {{- include "vcluster.generic.roleExtraRules" . | indent 2 }} - {{- include "vcluster.rbac.roleExtraRules" . | indent 2 }} -{{- end }} diff --git a/charts/k8s/templates/rbac/rolebinding.yaml b/charts/k8s/templates/rbac/rolebinding.yaml deleted file mode 100644 index 676e5002a..000000000 --- a/charts/k8s/templates/rbac/rolebinding.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.rbac.role.create }} -{{- if .Values.multiNamespaceMode.enabled }} -kind: ClusterRoleBinding -{{- else -}} -kind: RoleBinding -{{- end }} -apiVersion: rbac.authorization.k8s.io/v1 -metadata: -{{- if .Values.multiNamespaceMode.enabled }} - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -{{- end }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -subjects: - - kind: ServiceAccount - {{- if .Values.serviceAccount.name }} - name: {{ .Values.serviceAccount.name }} - {{- else }} - name: vc-{{ .Release.Name }} - {{- end }} - namespace: {{ .Release.Namespace }} -roleRef: -{{- if .Values.multiNamespaceMode.enabled }} - kind: ClusterRole - name: {{ template "vcluster.clusterRoleNameMultinamespace" . }} -{{- else }} - kind: Role - name: {{ .Release.Name }} -{{- end }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/charts/k8s/templates/resourcequota.yaml b/charts/k8s/templates/resourcequota.yaml deleted file mode 100644 index b19d89fff..000000000 --- a/charts/k8s/templates/resourcequota.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if and .Values.isolation.enabled .Values.isolation.resourceQuota.enabled }} -apiVersion: v1 -kind: ResourceQuota -metadata: - name: {{ .Release.Name }}-quota - namespace: {{ .Values.isolation.namespace | default .Release.Namespace }} - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -spec: - hard: - {{- range $key, $val := .Values.isolation.resourceQuota.quota }} - {{ $key }}: {{ $val | quote }} - {{- end }} - - {{- if .Values.isolation.resourceQuota.scopeSelector.matchExpressions }} - scopeSelector: - matchExpressions: - {{- toYaml .Values.isolation.resourceQuota.scopeSelector.matchExpressions | nindent 4 }} - {{- end}} - - {{- if .Values.isolation.resourceQuota.scopes }} - scopes: - {{- toYaml .Values.isolation.resourceQuota.scopes | nindent 4 }} - {{- end}} - -{{- end }} diff --git a/charts/k8s/templates/service-monitor.yaml b/charts/k8s/templates/service-monitor.yaml deleted file mode 100644 index b6917ae27..000000000 --- a/charts/k8s/templates/service-monitor.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.monitoring.serviceMonitor.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: vcluster - release: "{{ .Release.Name }}" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: "{{ .Release.Service }}" - endpoints: - - interval: 30s - port: https - path: /metrics - scheme: https - tlsConfig: - ca: - secret: - name: vc-{{ .Release.Name }} - key: certificate-authority - cert: - secret: - name: vc-{{ .Release.Name }} - key: client-certificate - keySecret: - name: vc-{{ .Release.Name }} - key: client-key - serverName: 127.0.0.1 -{{- end }} diff --git a/charts/k8s/templates/service.yaml b/charts/k8s/templates/service.yaml deleted file mode 100644 index b575a9fe1..000000000 --- a/charts/k8s/templates/service.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: {{ if eq .Values.service.type "LoadBalancer" -}}ClusterIP{{- else }} {{- .Values.service.type }} {{- end }} - ports: - - name: https - port: 443 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.httpsNodePort }} - protocol: TCP - - name: kubelet - port: 10250 - {{- if not .Values.headless }} - targetPort: 8443 - {{- end }} - nodePort: {{ .Values.service.kubeletNodePort }} - protocol: TCP - {{- if .Values.service.externalIPs }} - externalIPs: - {{- range $f := .Values.service.externalIPs }} - - {{ $f }} - {{- end }} - {{- end }} - {{- if eq .Values.service.type "NodePort" }} - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} ---- -{{ if eq .Values.service.type "LoadBalancer" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-lb - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.service.loadBalancerAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - type: LoadBalancer - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} - {{- if not .Values.headless }} - selector: - app: vcluster - release: {{ .Release.Name }} - {{- end }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.service.loadBalancerClass }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- range $f := .Values.service.loadBalancerSourceRanges }} - - "{{ $f }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/k8s/templates/serviceaccount.yaml b/charts/k8s/templates/serviceaccount.yaml deleted file mode 100644 index 504998037..000000000 --- a/charts/k8s/templates/serviceaccount.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- if .Values.globalAnnotations }} - annotations: -{{ toYaml .Values.globalAnnotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} -{{- end }} diff --git a/charts/k8s/templates/statefulset-service.yaml b/charts/k8s/templates/statefulset-service.yaml deleted file mode 100644 index e7b5d3d91..000000000 --- a/charts/k8s/templates/statefulset-service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if eq ( include "vcluster.kind" . ) "StatefulSet" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-headless - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "vcluster.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.serviceAnnotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - publishNotReadyAddresses: true - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP - {{- if .Values.embeddedEtcd.enabled }} - - name: etcd - port: 2379 - targetPort: 2379 - protocol: TCP - - name: peer - port: 2380 - targetPort: 2380 - protocol: TCP - {{- end }} - clusterIP: None - selector: - app: vcluster - release: "{{ .Release.Name }}" -{{- end }} diff --git a/charts/k8s/templates/syncer.yaml b/charts/k8s/templates/syncer.yaml deleted file mode 100644 index a59945763..000000000 --- a/charts/k8s/templates/syncer.yaml +++ /dev/null @@ -1,493 +0,0 @@ -{{- if not .Values.headless }} -apiVersion: apps/v1 -kind: {{ include "vcluster.kind" . }} -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.syncer.labels }} -{{ toYaml .Values.syncer.labels | indent 4 }} -{{- end }} - {{- $annotations := merge .Values.globalAnnotations .Values.syncer.annotations }} - {{- if $annotations}} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -spec: - replicas: {{ .Values.syncer.replicas }} - {{- include "vcluster.deployment.strategy" . | indent 2 }} - {{- include "vcluster.statefulset.serviceName" . | indent 2 }} - {{- include "vcluster.statefulset.volumeClaimTemplate" . | indent 2 }} - selector: - matchLabels: - app: vcluster - release: {{ .Release.Name }} - template: - metadata: - {{- if .Values.syncer.podAnnotations }} - annotations: -{{ toYaml .Values.syncer.podAnnotations | indent 8 }} - {{- end }} - labels: - app: vcluster - release: {{ .Release.Name }} - {{- range $k, $v := .Values.syncer.podLabels }} - {{ $k }}: {{ $v | quote }} - {{- end }} - spec: - terminationGracePeriodSeconds: 10 - {{- if .Values.syncer.affinity }} - affinity: -{{ toYaml .Values.syncer.affinity | indent 8 }} - {{- else if (gt (int .Values.syncer.replicas) 1) }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - # if possible avoid scheduling more than one pod on one node - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: "kubernetes.io/hostname" - # if possible avoid scheduling pod onto node that is in the same zone as one or more vcluster pods are running - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vcluster - - key: release - operator: In - values: - - {{ .Release.Name }} - topologyKey: topology.kubernetes.io/zone - {{- end }} - {{- if .Values.syncer.topologySpreadConstraints }} - topologySpreadConstraints: -{{ toYaml .Values.syncer.topologySpreadConstraints | indent 8 }} - {{- end }} - nodeSelector: -{{ toYaml .Values.syncer.nodeSelector | indent 8 }} - tolerations: -{{ toYaml .Values.syncer.tolerations | indent 8 }} - {{- if .Values.serviceAccount.name }} - serviceAccountName: {{ .Values.serviceAccount.name }} - {{- else }} - serviceAccountName: vc-{{ .Release.Name }} - {{- end }} - {{- if .Values.syncer.podSecurityContext }} - securityContext: -{{ toYaml .Values.syncer.podSecurityContext | indent 8 }} - {{- end }} - volumes: - {{- include "vcluster.plugins.volumes" . | indent 8 }} - - name: helm-cache - emptyDir: {} - - name: tmp - emptyDir: {} - - name: certs - emptyDir: {} - {{- if .Values.volumes }} -{{ toYaml .Values.volumes | indent 8 }} - {{- end }} - - name: binaries - emptyDir: {} - {{- if .Values.syncer.volumes }} -{{ toYaml .Values.syncer.volumes | indent 8 }} - {{- end }} - {{- if and .Values.coredns.enabled (not .Values.coredns.integrated) }} - - name: coredns - configMap: - name: {{ .Release.Name }}-coredns - {{- else if .Values.coredns.integrated }} - - name: coredns - configMap: - name: {{ .Release.Name }}-dns - {{- end }} - - name: custom-config-volume - configMap: - name: coredns-custom - optional: true - {{- if .Values.syncer.priorityClassName }} - priorityClassName: {{ .Values.syncer.priorityClassName }} - {{- end }} - initContainers: - {{- include "vcluster.plugins.initContainers" . | indent 6 }} - # this is needed because the k8s containers are distroless and thus we don't have any - # way of copying the binaries otherwise - - name: vcluster-copy - {{- if .Values.syncer.image }} - image: "{{ .Values.defaultImageRegistry }}{{ .Values.syncer.image }}" - {{- else }} - {{- if .Values.pro }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster-pro:{{ .Chart.Version }}" - {{- else }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster:{{ .Chart.Version }}" - {{- end }} - {{- end }} - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /bin/sh - args: - - -c - - "cp /vcluster /binaries/vcluster" - {{- if .Values.syncer.imagePullPolicy }} - imagePullPolicy: {{ .Values.syncer.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- if not .Values.controller.disabled }} - - name: kube-controller-manager - image: "{{ .Values.defaultImageRegistry }}{{ .Values.controller.image }}" - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /binaries/vcluster - args: - - cp - - /usr/local/bin/kube-controller-manager - - /binaries/kube-controller-manager - {{- if .Values.controller.imagePullPolicy }} - imagePullPolicy: {{ .Values.controller.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} - {{- if or (not .Values.scheduler.disabled) .Values.sync.nodes.enableScheduler }} - - name: kube-scheduler-manager - image: "{{ .Values.defaultImageRegistry }}{{ .Values.scheduler.image }}" - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /binaries/vcluster - args: - - cp - - /usr/local/bin/kube-scheduler - - /binaries/kube-scheduler - {{- if .Values.scheduler.imagePullPolicy }} - imagePullPolicy: {{ .Values.scheduler.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} - {{- if not .Values.api.disabled }} - - name: kube-apiserver - image: "{{ .Values.defaultImageRegistry }}{{ .Values.api.image }}" - volumeMounts: - - mountPath: /binaries - name: binaries - command: - - /binaries/vcluster - args: - - cp - - /usr/local/bin/kube-apiserver - - /binaries/kube-apiserver - {{- if .Values.api.imagePullPolicy }} - imagePullPolicy: {{ .Values.api.imagePullPolicy }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} - {{- end }} - containers: - - name: syncer - {{- if .Values.syncer.image }} - image: "{{ .Values.defaultImageRegistry }}{{ .Values.syncer.image }}" - {{- else }} - {{- if .Values.pro }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster-pro:{{ .Chart.Version }}" - {{- else }} - image: "{{ .Values.defaultImageRegistry }}ghcr.io/loft-sh/vcluster:{{ .Chart.Version }}" - {{- end }} - {{- end }} - {{- if .Values.syncer.workingDir }} - workingDir: {{ .Values.syncer.workingDir }} - {{- end }} - {{- if .Values.syncer.command }} - command: - {{- range $f := .Values.syncer.command }} - - {{ $f | quote }} - {{- end }} - {{- end }} - args: - - --name={{ .Release.Name }} - - --request-header-ca-cert=/pki/front-proxy-ca.crt - - --client-ca-cert=/pki/ca.crt - - --server-ca-cert=/pki/ca.crt - - --server-ca-key=/pki/ca.key - - --kube-config=/pki/admin.conf - - --service-account=vc-workload-{{ .Release.Name }} - {{- if .Values.embeddedEtcd.migrateFromEtcd }} - - --migrate-from=https://{{ .Release.Name }}-etcd:2379 - {{- end }} - {{- include "vcluster.legacyPlugins.args" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromHost" . | indent 10 }} - {{- include "vcluster.serviceMapping.fromVirtual" . | indent 10 }} - {{- if or (not .Values.scheduler.disabled) .Values.sync.nodes.enableScheduler }} - - --enable-scheduler - {{- end }} - {{- if .Values.defaultImageRegistry }} - - --default-image-registry={{ .Values.defaultImageRegistry }} - {{- end }} - {{- if .Values.syncer.kubeConfigContextName }} - - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }} - {{- end }} - {{- if .Values.pro }} - {{- if .Values.proLicenseSecret }} - - --pro-license-secret={{ .Values.proLicenseSecret }} - {{- end }} - {{- if .Values.embeddedEtcd.enabled }} - - --etcd-embedded - - --etcd-replicas={{ .Values.syncer.replicas }} - {{- end }} - {{- end }} - {{- if (gt (int .Values.syncer.replicas) 1)}} - - --leader-elect=true - {{- else }} - - --leader-elect=false - {{- end }} - {{- if .Values.ingress.enabled }} - - --tls-san={{ .Values.ingress.host }} - {{- end }} - {{- if .Values.isolation.enabled }} - - --enforce-pod-security-standard={{ .Values.isolation.podSecurityStandard }} - {{- end}} - {{- include "vcluster.syncer.syncArgs" . | indent 10 -}} - {{- if .Values.sync.nodes.syncAllNodes }} - - --sync-all-nodes - {{- end }} - {{- if .Values.sync.nodes.nodeSelector }} - - --node-selector={{ .Values.sync.nodes.nodeSelector }} - {{- end }} - {{- if .Values.multiNamespaceMode.enabled }} - - --multi-namespace-mode=true - {{- end }} - {{- if .Values.sync.configmaps.all }} - - --sync-all-configmaps=true - {{- end }} - {{- if .Values.sync.secrets.all }} - - --sync-all-secrets=true - {{- end }} - {{- if not .Values.sync.nodes.fakeKubeletIPs }} - - --fake-kubelet-ips=false - {{- end }} - {{- if or .Values.proxy.metricsServer.nodes.enabled .Values.proxy.metricsServer.pods.enabled }} - - --proxy-metrics-server=true - {{- end }} - {{- if .Values.coredns.integrated }} - - --integrated-coredns=true - {{- end }} - {{- if and .Values.coredns.integrated .Values.coredns.plugin.enabled }} - - --use-coredns-plugin=true - {{- end }} - {{- if .Values.centralAdmission.validatingWebhooks }} - {{- range .Values.centralAdmission.validatingWebhooks }} - - --enforce-validating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- if .Values.centralAdmission.mutatingWebhooks }} - {{- range .Values.centralAdmission.mutatingWebhooks }} - - --enforce-mutating-hook={{ . | toYaml | b64enc }} - {{- end }} - {{- end }} - {{- range $f := .Values.syncer.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- if .Values.syncer.livenessProbe }} - {{- if .Values.syncer.livenessProbe.enabled }} - livenessProbe: - httpGet: - path: /healthz - port: 8443 - scheme: HTTPS - failureThreshold: 10 - initialDelaySeconds: 60 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.readinessProbe }} - {{- if .Values.syncer.readinessProbe.enabled }} - startupProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 300 - periodSeconds: 6 - readinessProbe: - httpGet: - path: /readyz - port: 8443 - scheme: HTTPS - failureThreshold: 30 - periodSeconds: 2 - {{- end }} - {{- end }} - {{- if .Values.syncer.imagePullPolicy }} - imagePullPolicy: {{ .Values.syncer.imagePullPolicy }} - {{- end }} - securityContext: -{{ toYaml .Values.syncer.securityContext | indent 10 }} - env: - {{- include "vcluster.plugins.config" . | indent 10 }} - - name: VCLUSTER_DISTRO - value: k8s - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - {{- if eq (.Values.syncer.replicas | toString | atoi) 1 }} - - name: VCLUSTER_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- end }} - {{- if .Values.syncer.env }} -{{ toYaml .Values.syncer.env | indent 10 }} - {{- end }} - {{- if .Values.sync.generic.config }} - - name: CONFIG - value: |- - {{- .Values.sync.generic.config | nindent 14 }} - {{- end }} - - name: VCLUSTER_TELEMETRY_CONFIG - value: {{ .Values.telemetry | toJson | quote }} - {{- if not .Values.api.disabled }} - - name: APISERVER_COMMAND - value: |- - command: - - /binaries/kube-apiserver - - '--advertise-address=127.0.0.1' - - '--bind-address=127.0.0.1' - - '--allow-privileged=true' - - '--authorization-mode=RBAC' - - '--client-ca-file=/pki/ca.crt' - - '--enable-bootstrap-token-auth=true' - - '--etcd-cafile=/pki/etcd/ca.crt' - - '--etcd-certfile=/pki/apiserver-etcd-client.crt' - - '--etcd-keyfile=/pki/apiserver-etcd-client.key' - {{- if .Values.embeddedEtcd.enabled }} - - '--etcd-servers=https://127.0.0.1:2379' - {{- else }} - - '--etcd-servers=https://{{ .Release.Name }}-etcd:2379' - {{- end }} - - '--proxy-client-cert-file=/pki/front-proxy-client.crt' - - '--proxy-client-key-file=/pki/front-proxy-client.key' - - '--requestheader-allowed-names=front-proxy-client' - - '--requestheader-client-ca-file=/pki/front-proxy-ca.crt' - - '--requestheader-extra-headers-prefix=X-Remote-Extra-' - - '--requestheader-group-headers=X-Remote-Group' - - '--requestheader-username-headers=X-Remote-User' - - '--secure-port=6443' - - '--service-account-issuer=https://kubernetes.default.svc.cluster.local' - - '--service-account-key-file=/pki/sa.pub' - - '--service-account-signing-key-file=/pki/sa.key' - - '--tls-cert-file=/pki/apiserver.crt' - - '--tls-private-key-file=/pki/apiserver.key' - - '--watch-cache=false' - - '--endpoint-reconciler-type=none' - {{- range $f := .Values.api.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- end }} - {{- if not .Values.controller.disabled }} - - name: CONTROLLER_COMMAND - value: |- - command: - - /binaries/kube-controller-manager - - '--authentication-kubeconfig=/pki/controller-manager.conf' - - '--authorization-kubeconfig=/pki/controller-manager.conf' - - '--bind-address=127.0.0.1' - - '--client-ca-file=/pki/ca.crt' - - '--cluster-name=kubernetes' - - '--cluster-signing-cert-file=/pki/ca.crt' - - '--cluster-signing-key-file=/pki/ca.key' - {{- if or (not .Values.scheduler.disabled) .Values.sync.nodes.enableScheduler }} - - '--controllers=*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' - - '--node-monitor-grace-period=1h' - - '--node-monitor-period=1h' - {{- else }} - - '--controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' - {{- end }} - - '--horizontal-pod-autoscaler-sync-period=60s' - - '--kubeconfig=/pki/controller-manager.conf' - {{- if (gt (int .Values.syncer.replicas) 1) }} - - '--leader-elect=true' - {{- else }} - - '--leader-elect=false' - {{- end }} - - '--node-monitor-grace-period=180s' - - '--node-monitor-period=30s' - - '--pvclaimbinder-sync-period=60s' - - '--requestheader-client-ca-file=/pki/front-proxy-ca.crt' - - '--root-ca-file=/pki/ca.crt' - - '--service-account-private-key-file=/pki/sa.key' - - '--use-service-account-credentials=true' - {{- range $f := .Values.controller.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- end }} - {{- if or (not .Values.scheduler.disabled) .Values.sync.nodes.enableScheduler }} - - name: SCHEDULER_COMMAND - value: |- - command: - - /binaries/kube-scheduler - - '--authentication-kubeconfig=/pki/scheduler.conf' - - '--authorization-kubeconfig=/pki/scheduler.conf' - - '--bind-address=127.0.0.1' - - '--kubeconfig=/pki/scheduler.conf' - {{- if (gt (int .Values.syncer.replicas) 1) }} - - '--leader-elect=true' - {{- else }} - - '--leader-elect=false' - {{- end }} - {{- range $f := .Values.scheduler.extraArgs }} - - {{ $f | quote }} - {{- end }} - {{- end }} - volumeMounts: - {{- include "vcluster.plugins.volumeMounts" . | indent 10 }} - {{- if eq ( include "vcluster.kind" . ) "StatefulSet" }} - - name: data - mountPath: /data - {{- end }} - - name: helm-cache - mountPath: /.cache/helm - - name: tmp - mountPath: /tmp - - mountPath: /pki - name: certs - - mountPath: /binaries - name: binaries - {{- if .Values.coredns.enabled }} - - name: coredns - mountPath: /manifests/coredns - readOnly: true - {{- end }} - {{- if .Values.syncer.volumeMounts }} -{{ toYaml .Values.syncer.volumeMounts | indent 10 }} - {{- end }} - {{- if .Values.syncer.extraVolumeMounts }} -{{ toYaml .Values.syncer.extraVolumeMounts | indent 10 }} - {{- end }} - resources: -{{ toYaml .Values.syncer.resources | indent 10 }} -{{- include "vcluster.legacyPlugins.containers" . | indent 6 }} -{{- end }} diff --git a/charts/k8s/templates/workloadserviceaccount.yaml b/charts/k8s/templates/workloadserviceaccount.yaml deleted file mode 100644 index 8d05fc1e8..000000000 --- a/charts/k8s/templates/workloadserviceaccount.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vc-workload-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: vcluster - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - {{- $annotations := merge .Values.globalAnnotations .Values.workloadServiceAccount.annotations }} - {{- if $annotations }} - annotations: -{{ toYaml $annotations | indent 4 }} - {{- end }} -{{- if .Values.serviceAccount.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} -{{- end }} diff --git a/charts/k8s/tests/README.md b/charts/k8s/tests/README.md deleted file mode 100644 index 24d888a08..000000000 --- a/charts/k8s/tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Add [unittest plugin](https://github.com/helm-unittest/helm-unittest) via: -``` -helm plugin install https://github.com/helm-unittest/helm-unittest.git -``` - -Run tests via: -``` -helm unittest charts/k8s -d -``` diff --git a/charts/k8s/tests/clusterrole_test.yaml b/charts/k8s/tests/clusterrole_test.yaml deleted file mode 100644 index 3385c786c..000000000 --- a/charts/k8s/tests/clusterrole_test.yaml +++ /dev/null @@ -1,41 +0,0 @@ -suite: ClusterRole -templates: - - rbac/clusterrole.yaml - -tests: - - it: should create clusterrole - set: - rbac: - clusterRole: - create: true - asserts: - - hasDocuments: - count: 1 - - it: should not create clusterrole - set: - rbac: - clusterRole: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - clusterRole: - create: true - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/k8s/tests/etcd_test.yaml b/charts/k8s/tests/etcd_test.yaml deleted file mode 100644 index 931ce5d34..000000000 --- a/charts/k8s/tests/etcd_test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -suite: Etcd -templates: - - etcd-statefulset.yaml - - etcd-service.yaml - - etcd-statefulset-service.yaml - -tests: - - it: should have etcd when migratefrometcd is true - set: - pro: true - embeddedEtcd: - enabled: true - migrateFromEtcd: true - asserts: - - hasDocuments: - count: 1 - - it: shouldn't have etcd when migratefrometcd is false - set: - pro: true - embeddedEtcd: - enabled: true - migrateFromEtcd: false - asserts: - - hasDocuments: - count: 0 - diff --git a/charts/k8s/tests/legacy-plugins_test.yaml b/charts/k8s/tests/legacy-plugins_test.yaml deleted file mode 100644 index 6977c7562..000000000 --- a/charts/k8s/tests/legacy-plugins_test.yaml +++ /dev/null @@ -1,77 +0,0 @@ -suite: Legacy Plugins -templates: - - syncer.yaml - -tests: - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - - it: should check legacy plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - bootstrap-with-deployment2: - image: test - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment" - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--plugins=bootstrap-with-deployment2" - count: 1 - - equal: - path: spec.template.spec.containers[1].name - value: bootstrap-with-deployment - - equal: - path: spec.template.spec.containers[2].name - value: bootstrap-with-deployment2 - - equal: - path: spec.template.spec.containers[1].image - value: test - - equal: - path: spec.template.spec.containers[2].image - value: test - - equal: - path: spec.template.spec.containers[1].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14000 - - equal: - path: spec.template.spec.containers[2].env[0] - value: - name: VCLUSTER_PLUGIN_ADDRESS - value: localhost:14001 - - - it: should check no legacy plugin rendering - asserts: - - hasDocuments: - count: 1 - - lengthEqual: - path: spec.template.spec.containers - count: 1 diff --git a/charts/k8s/tests/plugins_test.yaml b/charts/k8s/tests/plugins_test.yaml deleted file mode 100644 index 2ce3149dc..000000000 --- a/charts/k8s/tests/plugins_test.yaml +++ /dev/null @@ -1,100 +0,0 @@ -suite: Plugins -templates: - - syncer.yaml - -tests: - - it: should check plugin config rendering - set: - plugin: - plugin1: - version: v2 - config: - myConfig: true - plugin2: - version: v2 - image: test - plugin3: - version: v2 - image: test123 - config: - myOtherConfig: - - test123 - - test456 - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.containers[0].env[0].name - value: PLUGIN_CONFIG - - equal: - path: spec.template.spec.containers[0].env[0].value - value: |- - plugin1: - myConfig: true - plugin3: - myOtherConfig: - - test123 - - test456 - - - it: should check plugin rendering - set: - plugin: - bootstrap-with-deployment: - version: v2 - image: test - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.template.spec.initContainers[0].image - value: test - - equal: - path: spec.template.spec.volumes[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - set: - plugin: - bootstrap-with-deployment: - image: test - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins - - - it: should check no plugin rendering - asserts: - - hasDocuments: - count: 1 - - notEqual: - path: spec.template.spec.initContainers[0].image - value: test - - notEqual: - path: spec.template.spec.volumes[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].name - value: plugins - - notEqual: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /plugins diff --git a/charts/k8s/tests/role_test.yaml b/charts/k8s/tests/role_test.yaml deleted file mode 100644 index 5eeedba74..000000000 --- a/charts/k8s/tests/role_test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -suite: Role -templates: - - rbac/role.yaml - -tests: - - it: should not create role - set: - rbac: - role: - create: false - asserts: - - hasDocuments: - count: 0 - - it: should contain extra rule - set: - rbac: - role: - extraRules: - - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - asserts: - - hasDocuments: - count: 1 - - contains: - path: rules - content: - apiGroups: ["test"] - resources: ["tests"] - verbs: ["test"] - count: 1 - diff --git a/charts/k8s/tests/syncer_test.yaml b/charts/k8s/tests/syncer_test.yaml deleted file mode 100644 index 7713b2e10..000000000 --- a/charts/k8s/tests/syncer_test.yaml +++ /dev/null @@ -1,52 +0,0 @@ -suite: Syncer -templates: - - syncer.yaml - -tests: - - it: should pass pro license secret as a flag - set: - pro: true - proLicenseSecret: "my-test-secret" - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].args - content: "--pro-license-secret=my-test-secret" - count: 1 - - it: should be a statefulset when embeddedEtcd is enabled - set: - pro: true - embeddedEtcd: - enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: StatefulSet - - exists: - path: spec.serviceName - - exists: - path: spec.volumeClaimTemplates - - notExists: - path: spec.strategy - - equal: - path: spec.template.spec.containers[0].volumeMounts[0].mountPath - value: /data - - - it: should be a deployment when embeddedEtcd is disabled - set: - pro: true - embeddedEtcd: - enabled: false - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Deployment - - notExists: - path: spec.serviceName - - notExists: - path: spec.volumeClaimTemplates - - exists: - path: spec.strategy diff --git a/charts/k8s/values.yaml b/charts/k8s/values.yaml deleted file mode 100644 index ed5dd2736..000000000 --- a/charts/k8s/values.yaml +++ /dev/null @@ -1,524 +0,0 @@ -# DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed -# images within the vcluster will not be rewritten. -defaultImageRegistry: "" - -# Global annotations to add to all objects -globalAnnotations: {} - -# If vCluster.Pro is enabled -pro: false - -# Defines where vCluster should search for a license secret. If you are using the vCluster.Pro control-plane, -# this is optional. vCluster by default will lookout for a secret called vc-VCLUSTER_NAME-license and if found use that. -# You can also use a secret within another namespace by using the format NAMESPACE/NAME. If the secret is in another -# namespace, please enable clusterRole.create and define an extra clusterRole.extraRules to allow vCluster to retrieve -# secrets within the cluster. -proLicenseSecret: "" - -# Embedded etcd settings -embeddedEtcd: - # If embedded etcd should be enabled, this is a PRO only feature - enabled: false - # To use if embeddedEtcd is enabled and you want to migrate the data - # from the external etcd - migrateFromEtcd: false - -# If true, will deploy vcluster in headless mode, which means no deployment -# or statefulset is created. -headless: false - -monitoring: - serviceMonitor: - enabled: false - -# Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml' -plugin: {} -# Manually configure a plugin called test -# test: -# image: ... -# env: ... -# rbac: -# clusterRole: -# extraRules: ... -# role: -# extraRules: ... - -# Extra Annotations for the stateful set -annotations: {} -podAnnotations: {} - -# Resource syncers that should be enabled/disabled. -# Enabling syncers will impact RBAC Role and ClusterRole permissions. -# To disable a syncer set "enabled: false". -# See docs for details - https://www.vcluster.com/docs/architecture/synced-resources -sync: - services: - enabled: true - configmaps: - enabled: true - all: false - secrets: - enabled: true - all: false - endpoints: - enabled: true - pods: - enabled: true - ephemeralContainers: false - status: false - events: - enabled: true - persistentvolumeclaims: - enabled: true - ingresses: - enabled: false - ingressclasses: {} - # By default IngressClasses sync is enabled when the Ingress sync is enabled - # but it can be explicitly disabled by setting: - # enabled: false - fake-nodes: - enabled: true # will be ignored if nodes.enabled = true - fake-persistentvolumes: - enabled: true # will be ignored if persistentvolumes.enabled = true - nodes: - fakeKubeletIPs: true - enabled: false - # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster - # will sync all nodes instead of only the ones where some pods are running. - syncAllNodes: false - # nodeSelector is used to limit which nodes get synced to the vcluster, - # and which nodes are used to run vcluster pods. - # A valid string representation of a label selector must be used. - nodeSelector: "" - # if true, vcluster will run with a scheduler and node changes are possible - # from within the virtual cluster. This is useful if you would like to - # taint, drain and label nodes from within the virtual cluster - enableScheduler: false - # DEPRECATED: use enable scheduler instead - # syncNodeChanges allows vcluster user edits of the nodes to be synced down to the host nodes. - # Write permissions on node resource will be given to the vcluster. - syncNodeChanges: false - persistentvolumes: - enabled: false - storageclasses: - enabled: false - # formerly named - "legacy-storageclasses" - hoststorageclasses: - enabled: false - priorityclasses: - enabled: false - networkpolicies: - enabled: false - volumesnapshots: - enabled: false - poddisruptionbudgets: - enabled: false - serviceaccounts: - enabled: false - # generic CRD configuration - generic: - config: |- - --- - -# If enabled, will fallback to host dns for resolving domains. This -# is useful if using istio or dapr in the host cluster and sidecar -# containers cannot connect to the central instance. Its also useful -# if you want to access host cluster services from within the vcluster. -fallbackHostDns: false - -# Map Services between host and virtual cluster -mapServices: - # Services that should get mapped from the - # virtual cluster to the host cluster. - # vcluster will make sure to sync the service - # ip to the host cluster automatically as soon - # as the service exists. - # For example: - # fromVirtual: - # - from: my-namespace/name - # to: host-service - fromVirtual: [] - # Same as from virtual, but instead sync services - # from the host cluster into the virtual cluster. - # If the namespace does not exist, vcluster will - # also create the namespace for the service. - fromHost: [] - -proxy: - metricsServer: - nodes: - enabled: false - pods: - enabled: false - -# Syncer configuration -syncer: - # Image to use for the syncer - # image: ghcr.io/loft-sh/vcluster - imagePullPolicy: "" - extraArgs: [] - volumeMounts: [] - extraVolumeMounts: [] - env: [] - livenessProbe: - enabled: true - readinessProbe: - enabled: true - resources: - limits: - ephemeral-storage: 8Gi - cpu: 1000m - memory: 2Gi - requests: - ephemeral-storage: 200Mi - # ensure that cpu/memory requests are high enough. - # for example gke wants minimum 10m/32Mi here! - cpu: 20m - memory: 256Mi - # Extra volumes - volumes: [] - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used to schedule the syncer - nodeSelector: {} - # Affinity to apply to the syncer deployment - affinity: {} - # Tolerations to apply to the syncer deployment - tolerations: [] - # Extra Labels for the syncer deployment - labels: {} - # Extra Annotations for the syncer deployment - annotations: {} - podAnnotations: {} - podLabels: {} - priorityClassName: "" - kubeConfigContextName: "my-vcluster" - # Security context configuration - securityContext: {} - podSecurityContext: - allowPrivilegeEscalation: false - runAsUser: 0 - runAsGroup: 0 - serviceAnnotations: {} - # Storage settings for the vcluster - storage: - # If this is disabled, vcluster will use an emptyDir instead - # of a PersistentVolumeClaim - persistence: true - # Size of the persistent volume claim - size: 5Gi - # Optional StorageClass used for the pvc - # if empty default StorageClass defined in your host cluster will be used - #className: - -# Etcd settings -etcd: - image: registry.k8s.io/etcd:3.5.10-0 - imagePullPolicy: "" - # The amount of replicas to run - replicas: 1 - # NodeSelector used - nodeSelector: {} - # Affinity to apply - affinity: {} - # Tolerations to apply - tolerations: [] - # Extra Labels - labels: {} - # Extra Annotations - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 20m - memory: 150Mi - # Storage settings for the etcd - storage: - # If this is disabled, vcluster will use an emptyDir instead - # of a PersistentVolumeClaim - persistence: true - # Size of the persistent volume claim - size: 5Gi - # Optional StorageClass used for the pvc - # if empty default StorageClass defined in your host cluster will be used - #className: - priorityClassName: "" - securityContext: {} - serviceAnnotations: {} - autoDeletePersistentVolumeClaims: false - -# Kubernetes Controller Manager settings -controller: - image: registry.k8s.io/kube-controller-manager:v1.29.0 - imagePullPolicy: "" - -# Kubernetes Scheduler settings. Only enabled if sync.nodes.enableScheduler is true -scheduler: - image: registry.k8s.io/kube-scheduler:v1.29.0 - imagePullPolicy: "" - disabled: true - -# Kubernetes API Server settings -api: - image: registry.k8s.io/kube-apiserver:v1.29.0 - imagePullPolicy: "" - extraArgs: [] - -# Service account that should be used by the vcluster -serviceAccount: - create: true - # Optional name of the service account to use - # name: default - # Optional pull secrets - # imagePullSecrets: - # - name: my-pull-secret - -# Service account that should be used by the pods synced by vcluster -workloadServiceAccount: - # This is not supported in multi-namespace mode - annotations: {} - -# Roles & ClusterRoles for the vcluster -rbac: - clusterRole: - # You don't need to toggle this as necessary cluster roles are created based on the enabled syncers (.sync.*.enabled). - # This only makes sense to enable if you want to use the extraRules below. - create: false - # Extra Rules for the cluster role - extraRules: [] - role: - # Disable this only if you don't want vCluster to create a role. This will break most functionality if disabled. - create: true - # Extra Rules for the role - extraRules: [] - # all entries in excludedApiResources will be excluded from the Role created for vcluster - excludedApiResources: - # - pods/exec - -# Syncer service configurations -service: - type: ClusterIP - - # Optional configuration - # A list of IP addresses for which nodes in the cluster will also accept traffic for this service. - # These IPs are not managed by Kubernetes; e.g., an external load balancer. - externalIPs: [] - - # Optional configuration for LoadBalancer & NodePort service types - # Route external traffic to node-local or cluster-wide endpoints [ Local | Cluster ] - externalTrafficPolicy: "" - - # Optional configuration for LoadBalancer service type - # Specify IP of load balancer to be created - loadBalancerIP: "" - # CIDR block(s) for the service allowlist - loadBalancerSourceRanges: [] - # Set the loadBalancerClass if using an external load balancer controller - loadBalancerClass: "" - # Set loadBalancer specific annotations on the Kubernetes service - loadBalancerAnnotations: {} - -# Configure the ingress resource that allows you to access the vcluster -ingress: - # Enable ingress record generation - enabled: false - # Ingress path type - pathType: ImplementationSpecific - ingressClassName: "" - host: vcluster.local - annotations: - nginx.ingress.kubernetes.io/backend-protocol: HTTPS - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - # Ingress TLS configuration - tls: [] - # - secretName: tls-vcluster.local - # hosts: - # - vcluster.local - -# Set "enable" to true when running vcluster in an OpenShift host -# This will add an extra rule to the deployed role binding in order -# to manage service endpoints -openshift: - enable: false - -# If enabled will deploy the coredns configmap -coredns: - # If CoreDns is enabled - enabled: true - # Pro only feature - integrated: false - # only used if isolation is enabled - fallback: 8.8.8.8 - plugin: - enabled: false - config: [] - # example configuration for plugin syntax, will be documented in detail - # - record: - # fqdn: google.com - # target: - # mode: url - # url: google.co.in - # - record: - # service: my-namespace/my-svc # dns-test/nginx-svc - # target: - # mode: host - # service: dns-test/nginx-svc - # - record: - # service: my-namespace-lb/my-svc-lb - # target: - # mode: host - # service: dns-test-exposed-lb/nginx-svc-exposed-lb - # - record: - # service: my-ns-external-name/my-svc-external-name - # target: - # mode: host - # service: dns-test-external-name/nginx-svc-external-name - # - record: - # service: my-ns-in-vcluster/my-svc-vcluster - # target: - # mode: vcluster # can be tested only manually for now - # vcluster: test-vcluster-ns/test-vcluster - # service: dns-test-in-vcluster-ns/test-in-vcluster-service - # - record: - # service: my-ns-in-vcluster-mns/my-svc-mns - # target: - # mode: vcluster # can be tested only manually for now - # service: dns-test-in-vcluster-mns/test-in-vcluster-svc-mns - # vcluster: test-vcluster-ns-mns/test-vcluster-mns - # - record: - # service: my-self-vc-ns/my-self-vc-svc - # target: - # mode: self - # service: dns-test/nginx-svc - replicas: 1 - # The nodeSelector example below specifices that coredns should only be scheduled to nodes with the arm64 label - # nodeSelector: - # kubernetes.io/arch: arm64 - # image: my-core-dns-image:latest - # config: |- - # .:1053 { - # ... - # CoreDNS service configurations - service: - type: ClusterIP - # Configuration for LoadBalancer service type - externalIPs: [] - externalTrafficPolicy: "" - # Extra Annotations - annotations: {} - resources: - limits: - cpu: 1000m - memory: 170Mi - requests: - cpu: 3m - memory: 16Mi -# if below option is configured, it will override the coredns manifests with the following string -# manifests: |- -# apiVersion: ... -# ... - podAnnotations: {} - podLabels: {} - -# If enabled will deploy vcluster in an isolated mode with pod security -# standards, limit ranges and resource quotas -isolation: - enabled: false - namespace: null - - podSecurityStandard: baseline - - # If enabled will add node/proxy permission to the cluster role - # in isolation mode - nodeProxyPermission: - enabled: false - - resourceQuota: - enabled: true - quota: - requests.cpu: 10 - requests.memory: 20Gi - requests.storage: "100Gi" - requests.ephemeral-storage: 60Gi - limits.cpu: 20 - limits.memory: 40Gi - limits.ephemeral-storage: 160Gi - services.nodeports: 0 - services.loadbalancers: 1 - count/endpoints: 40 - count/pods: 20 - count/services: 20 - count/secrets: 100 - count/configmaps: 100 - count/persistentvolumeclaims: 20 - scopeSelector: - matchExpressions: - scopes: - - limitRange: - enabled: true - default: - ephemeral-storage: 8Gi - memory: 512Mi - cpu: "1" - defaultRequest: - ephemeral-storage: 3Gi - memory: 128Mi - cpu: 100m - - networkPolicy: - enabled: true - outgoingConnections: - ipBlock: - cidr: 0.0.0.0/0 - except: - - 100.64.0.0/10 - - 127.0.0.0/8 - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - -# manifests to setup when initializing a vcluster -init: - manifests: |- - --- - # The contents of manifests-template will be templated using helm - # this allows you to use helm values inside, e.g.: {{ .Release.Name }} - manifestsTemplate: '' - helm: [] - # - bundle: - base64-encoded .tar.gz file content (optional - overrides chart.repo) - # chart: - # name: REQUIRED - # version: REQUIRED - # repo: (optional when bundle is used) - # username: (if required for repo) - # password: (if required for repo) - # insecure: boolean (if required for repo) - # release: - # name: REQUIRED - # namespace: REQUIRED - # timeout: number - # values: |- string YAML object - # foo: bar - # valuesTemplate: |- string YAML object - # foo: {{ .Release.Name }} - -multiNamespaceMode: - enabled: false - - -# list of {validating/mutating}webhooks that the syncer should proxy. -# This is a PRO only feature. -centralAdmission: - validatingWebhooks: [] - mutatingWebhooks: [] - -telemetry: - disabled: false - instanceCreator: "helm" - platformUserID: "" - platformInstanceID: "" - machineID: "" - diff --git a/cmd/vcluster/cmd/start.go b/cmd/vcluster/cmd/start.go index 797bf332c..0be795922 100644 --- a/cmd/vcluster/cmd/start.go +++ b/cmd/vcluster/cmd/start.go @@ -45,7 +45,7 @@ func NewStartCommand() *cobra.Command { }() // parse vCluster config - vClusterConfig, err := config.ParseConfig(startOptions.Config) + vClusterConfig, err := config.ParseConfig(startOptions.Config, os.Getenv("VCLUSTER_NAME")) if err != nil { return err } @@ -69,6 +69,9 @@ func ExecuteStart(ctx context.Context, vConfig *config.VirtualClusterConfig) err if vConfig.ServiceName == "" { vConfig.ServiceName = translate.VClusterName } + if vConfig.ControlPlane.Advanced.WorkloadServiceAccount.Name == "" { + vConfig.ControlPlane.Advanced.WorkloadServiceAccount.Name = "vc-workload-" + vConfig.Name + } // get current namespace controlPlaneConfig, controlPlaneNamespace, controlPlaneService, workloadConfig, workloadNamespace, workloadService, err := pro.GetRemoteClient(vConfig) @@ -94,10 +97,6 @@ func ExecuteStart(ctx context.Context, vConfig *config.VirtualClusterConfig) err plugin.DefaultManager.SetProFeatures(pro.LicenseFeatures()) // get host cluster config and tweak rate-limiting configuration - workloadClient, err := kubernetes.NewForConfig(workloadConfig) - if err != nil { - return err - } controlPlaneClient, err := kubernetes.NewForConfig(controlPlaneConfig) if err != nil { return err @@ -106,12 +105,10 @@ func ExecuteStart(ctx context.Context, vConfig *config.VirtualClusterConfig) err // check if we should create certs err = setup.Initialize( ctx, - workloadClient, controlPlaneClient, - workloadNamespace, controlPlaneNamespace, translate.VClusterName, - options, + vConfig, ) if err != nil { return fmt.Errorf("initialize: %w", err) @@ -120,7 +117,7 @@ func ExecuteStart(ctx context.Context, vConfig *config.VirtualClusterConfig) err // build controller context controllerCtx, err := setup.NewControllerContext( ctx, - options, + vConfig, workloadNamespace, workloadConfig, scheme.Scheme, diff --git a/config/config.go b/config/config.go index 179e45e9b..da2e82f3b 100644 --- a/config/config.go +++ b/config/config.go @@ -4,12 +4,12 @@ import "regexp" type Config struct { ExportKubeConfig ExportKubeConfig `json:"exportKubeConfig,omitempty"` + ControlPlane ControlPlane `json:"controlPlane,omitempty"` Sync Sync `json:"sync,omitempty"` Observability Observability `json:"observability,omitempty"` Networking Networking `json:"networking,omitempty"` Plugin map[string]Plugin `json:"plugin,omitempty"` Plugins map[string]Plugins `json:"plugins,omitempty"` - ControlPlane ControlPlane `json:"controlPlane,omitempty"` Policies Policies `json:"policies,omitempty"` RBAC RBAC `json:"rbac,omitempty"` @@ -17,6 +17,10 @@ type Config struct { Telemetry Telemetry `json:"telemetry,omitempty"` Experimental Experimental `json:"experimental,omitempty"` Platform Platform `json:"platform,omitempty"` + + // legacy for compatibility + ServiceCIDR string `json:"serviceCIDR,omitempty"` + Pro bool `json:"pro,omitempty"` } type ExportKubeConfig struct { @@ -79,11 +83,10 @@ type SyncAllResource struct { type SyncPods struct { Enabled bool `json:"enabled,omitempty"` - WorkloadServiceAccount string `json:"workloadServiceAccount,omitempty"` - TranslateImage map[string]string `json:"translateImage,omitempty"` - EnforceTolerations []string `json:"enforceTolerations,omitempty"` // validate format - UseSecretsForSATokens bool `json:"useSecretsForSATokens,omitempty"` - RewriteHosts SyncRewriteHosts `json:"rewriteHosts,omitempty"` + TranslateImage map[string]string `json:"translateImage,omitempty"` + EnforceTolerations []string `json:"enforceTolerations,omitempty"` // validate format + UseSecretsForSATokens bool `json:"useSecretsForSATokens,omitempty"` + RewriteHosts SyncRewriteHosts `json:"rewriteHosts,omitempty"` } type SyncRewriteHosts struct { @@ -128,13 +131,13 @@ type MetricsProxy struct { type Networking struct { ReplicateServices ReplicateServices `json:"replicateServices,omitempty"` - ResolveServices ResolveServices `json:"resolveServices,omitempty"` + ResolveServices []ResolveServices `json:"resolveServices,omitempty"` Advanced NetworkingAdvanced `json:"advanced,omitempty"` } type ReplicateServices struct { - ToHost ServiceMapping `json:"toHost,omitempty"` - FromHost ServiceMapping `json:"fromHost,omitempty"` + ToHost []ServiceMapping `json:"toHost,omitempty"` + FromHost []ServiceMapping `json:"fromHost,omitempty"` } type ServiceMapping struct { @@ -255,11 +258,12 @@ type Distro struct { K3S DistroK3s `json:"k3s,omitempty"` K8S DistroK8s `json:"k8s,omitempty"` K0S DistroK0s `json:"k0s,omitempty"` - EKS DistroEks `json:"eks,omitempty"` + EKS DistroK8s `json:"eks,omitempty"` } type DistroK3s struct { - Enabled bool `json:"enabled,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Token string `json:"token,omitempty"` DistroCommon `json:",inline"` DistroContainer `json:",inline"` } @@ -267,25 +271,18 @@ type DistroK3s struct { type DistroK8s struct { Enabled bool `json:"enabled,omitempty"` DistroCommon `json:",inline"` - APIServer DistroContainer `json:"apiServer,omitempty"` - ControllerManager DistroContainer `json:"controllerManager,omitempty"` - Scheduler DistroContainer `json:"scheduler,omitempty"` + APIServer DistroContainerDisabled `json:"apiServer,omitempty"` + ControllerManager DistroContainerDisabled `json:"controllerManager,omitempty"` + Scheduler DistroContainer `json:"scheduler,omitempty"` } type DistroK0s struct { - Enabled bool `json:"enabled,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Config string `json:"config,omitempty"` DistroCommon `json:",inline"` DistroContainer `json:",inline"` } -type DistroEks struct { - Enabled bool `json:"enabled,omitempty"` - DistroCommon `json:",inline"` - APIServer DistroContainer `json:"apiServer,omitempty"` - ControllerManager DistroContainer `json:"controllerManager,omitempty"` - Scheduler DistroContainer `json:"scheduler,omitempty"` -} - type DistroCommon struct { Env []map[string]interface{} `json:"env,omitempty"` SecurityContext map[string]interface{} `json:"securityContext,omitempty"` @@ -296,7 +293,14 @@ type DistroContainer struct { Image Image `json:"image,omitempty"` ImagePullPolicy string `json:"imagePullPolicy,omitempty"` Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` + ExtraArgs []string `json:"extraArgs,omitempty"` +} + +type DistroContainerDisabled struct { + Disabled bool `json:"disabled,omitempty"` + Image Image `json:"image,omitempty"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + Command []string `json:"command,omitempty"` ExtraArgs []string `json:"extraArgs,omitempty"` } @@ -327,8 +331,8 @@ type BackingStore struct { } type EmbeddedEtcd struct { - Enabled bool `json:"enabled,omitempty"` - MigrateFromSqlite bool `json:"migrateFromSqlite,omitempty"` + Enabled bool `json:"enabled,omitempty"` + MigrateFromExternalEtcd bool `json:"migrateFromExternalEtcd,omitempty"` } type ExternalEtcd struct { @@ -348,15 +352,15 @@ type ExternalEtcdMetadata struct { } type HostPathMapper struct { - EnableSwitch `json:",inline"` - Central bool `json:"central,omitempty" product:"pro"` + Enabled bool `json:"enabled,omitempty"` + Central bool `json:"central,omitempty" product:"pro"` } type CoreDNS struct { - EnableSwitch `json:",inline"` - Embedded bool `json:"embedded,omitempty" product:"pro"` - Service CoreDNSService `json:"service,omitempty"` - Deployment CoreDNSDeployment `json:"deployment,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Embedded bool `json:"embedded,omitempty" product:"pro"` + Service CoreDNSService `json:"service,omitempty"` + Deployment CoreDNSDeployment `json:"deployment,omitempty"` } type CoreDNSService struct { @@ -370,13 +374,9 @@ type CoreDNSDeployment struct { } type ControlPlaneProxy struct { - BindAddress string `json:"bindAddress,omitempty"` - Port int `json:"port,omitempty"` - TLS ControlPlaneProxyTLS `json:"tls,omitempty"` -} - -type ControlPlaneProxyTLS struct { - ExtraSANs []string `json:"extraSANs,omitempty"` + BindAddress string `json:"bindAddress,omitempty"` + Port int `json:"port,omitempty"` + ExtraSANs []string `json:"extraSANs,omitempty"` } type ControlPlaneService struct { @@ -445,7 +445,7 @@ type VolumeMount struct { type ControlPlaneScheduling struct { NodeSelector map[string]interface{} `json:"nodeSelector,omitempty"` Affinity map[string]interface{} `json:"affinity,omitempty"` - Tolerations map[string]interface{} `json:"tolerations,omitempty"` + Tolerations []interface{} `json:"tolerations,omitempty"` PriorityClassName string `json:"priorityClassName,omitempty"` } @@ -594,7 +594,6 @@ type Experimental struct { Extended map[string]interface{} `json:",inline"` IsolatedControlPlane ExperimentalIsolatedControlPlane `json:"isolatedControlPlane,omitempty"` - ControlPlaneSettings ExperimentalControlPlaneSettings `json:"controlPlaneSettings,omitempty"` SyncSettings ExperimentalSyncSettings `json:"syncSettings,omitempty"` GenericSync ExperimentalGenericSync `json:"genericSync,omitempty"` Deploy ExperimentalDeploy `json:"deploy,omitempty"` @@ -605,25 +604,25 @@ type Experimental struct { type ExperimentalMultiNamespaceMode struct { Enabled bool `json:"enabled,omitempty"` + + NamespaceLabels map[string]string `json:"namespaceLabels,omitempty"` } type ExperimentalIsolatedControlPlane struct { Enabled bool `json:"enabled,omitempty"` KubeConfig string `json:"kubeConfig,omitempty"` -} - -type ExperimentalControlPlaneSettings struct { - RewriteKubernetesService bool `json:"rewriteKubernetesService,omitempty"` + Namespace string `json:"namespace,omitempty"` + Service string `json:"service,omitempty"` } type ExperimentalSyncSettings struct { - DisableSync bool `json:"disableSync,omitempty"` - Target ExperimentalSyncSettingsTarget `json:"target,omitempty"` -} + DisableSync bool `json:"disableSync,omitempty"` + RewriteKubernetesService bool `json:"rewriteKubernetesService,omitempty"` -type ExperimentalSyncSettingsTarget struct { - Namespace string `json:"namespace,omitempty"` + TargetNamespace string `json:"targetNamespace,omitempty"` + SetOwner bool `json:"setOwner,omitempty"` + SyncLabels []string `json:"syncLabels,omitempty"` } type ExperimentalDeploy struct { diff --git a/pkg/config/config.go b/pkg/config/config.go index d3a449b4b..96d391553 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -6,6 +6,14 @@ import ( "github.com/loft-sh/vcluster/config" ) +const ( + K3SDistro = "k3s" + K8SDistro = "k8s" + K0SDistro = "k0s" + EKSDistro = "eks" + Unknown = "unknown" +) + type VirtualClusterConfig struct { // Holds the vCluster config config.Config `json:",inline"` @@ -20,6 +28,69 @@ type VirtualClusterConfig struct { TargetNamespace string `json:"targetNamespace,omitempty"` } +func (v VirtualClusterConfig) Distro() string { + if v.Config.ControlPlane.Distro.K3S.Enabled { + return K3SDistro + } else if v.Config.ControlPlane.Distro.K0S.Enabled { + return K0SDistro + } else if v.Config.ControlPlane.Distro.K8S.Enabled { + return K8SDistro + } else if v.Config.ControlPlane.Distro.EKS.Enabled { + return EKSDistro + } + + return K3SDistro +} + +func (v VirtualClusterConfig) VirtualClusterKubeConfig() config.VirtualClusterKubeConfig { + distroConfig := config.VirtualClusterKubeConfig{} + switch v.Distro() { + case K3SDistro: + distroConfig = config.VirtualClusterKubeConfig{ + KubeConfig: "/data/server/cred/admin.kubeconfig", + ServerCAKey: "/data/server/tls/server-ca.key", + ServerCACert: "/data/server/tls/server-ca.crt", + ClientCACert: "/data/server/tls/client-ca.crt", + RequestHeaderCACert: "/data/server/tls/request-header-ca.crt", + } + case K0SDistro: + distroConfig = config.VirtualClusterKubeConfig{ + KubeConfig: "/data/k0s/pki/admin.conf", + ServerCAKey: "/data/k0s/pki/ca.key", + ServerCACert: "/data/k0s/pki/ca.crt", + ClientCACert: "/data/k0s/pki/ca.crt", + RequestHeaderCACert: "/data/k0s/pki/front-proxy-ca.crt", + } + case EKSDistro, K8SDistro: + distroConfig = config.VirtualClusterKubeConfig{ + KubeConfig: "/pki/admin.conf", + ServerCAKey: "/pki/ca.key", + ServerCACert: "/pki/ca.crt", + ClientCACert: "/pki/ca.crt", + RequestHeaderCACert: "/pki/front-proxy-ca.crt", + } + } + + retConfig := v.Config.Experimental.VirtualClusterKubeConfig + if retConfig.KubeConfig == "" { + retConfig.KubeConfig = distroConfig.KubeConfig + } + if retConfig.ClientCACert == "" { + retConfig.ClientCACert = distroConfig.ClientCACert + } + if retConfig.ServerCAKey == "" { + retConfig.ServerCAKey = distroConfig.ServerCAKey + } + if retConfig.ServerCACert == "" { + retConfig.ServerCACert = distroConfig.ServerCACert + } + if retConfig.RequestHeaderCACert == "" { + retConfig.RequestHeaderCACert = distroConfig.RequestHeaderCACert + } + + return retConfig +} + // LegacyOptions converts the config to the legacy cluster options func (v VirtualClusterConfig) LegacyOptions() (*LegacyVirtualClusterOptions, error) { legacyPlugins := []string{} @@ -41,47 +112,41 @@ func (v VirtualClusterConfig) LegacyOptions() (*LegacyVirtualClusterOptions, err nodeSelector = strings.Join(selectors, ",") } - retOptions := &LegacyVirtualClusterOptions{ + return &LegacyVirtualClusterOptions{ ProOptions: LegacyVirtualClusterProOptions{ - RemoteKubeConfig: "", - RemoteNamespace: "", - RemoteServiceName: "", - EnforceValidatingHooks: nil, - EnforceMutatingHooks: nil, - IntegratedCoredns: v.ControlPlane.CoreDNS.Embedded, - UseCoreDNSPlugin: false, - EtcdReplicas: 0, - EtcdEmbedded: v.ControlPlane.BackingStore.EmbeddedEtcd.Enabled, - MigrateFrom: "", - NoopSyncer: false, - SyncKubernetesService: false, + RemoteKubeConfig: v.Experimental.IsolatedControlPlane.KubeConfig, + RemoteNamespace: v.Experimental.IsolatedControlPlane.Namespace, + RemoteServiceName: v.Experimental.IsolatedControlPlane.Service, + IntegratedCoredns: v.ControlPlane.CoreDNS.Embedded, + EtcdReplicas: int(v.ControlPlane.StatefulSet.HighAvailability.Replicas), + EtcdEmbedded: v.ControlPlane.BackingStore.EmbeddedEtcd.Enabled, + NoopSyncer: !v.Experimental.SyncSettings.DisableSync, + SyncKubernetesService: v.Experimental.SyncSettings.RewriteKubernetesService, }, - Controllers: nil, - ServerCaCert: "", - ServerCaKey: "", - TLSSANs: nil, - RequestHeaderCaCert: "", - ClientCaCert: "", - KubeConfigPath: "", - KubeConfigContextName: "", - KubeConfigSecret: "", - KubeConfigSecretNamespace: "", - KubeConfigServer: "", - Tolerations: nil, + ServerCaCert: v.VirtualClusterKubeConfig().ServerCACert, + ServerCaKey: v.VirtualClusterKubeConfig().ServerCAKey, + TLSSANs: v.ControlPlane.Proxy.ExtraSANs, + RequestHeaderCaCert: v.VirtualClusterKubeConfig().RequestHeaderCACert, + ClientCaCert: v.VirtualClusterKubeConfig().ClientCACert, + KubeConfigPath: v.VirtualClusterKubeConfig().KubeConfig, + KubeConfigContextName: v.ExportKubeConfig.Context, + KubeConfigSecret: v.ExportKubeConfig.Secret.Name, + KubeConfigSecretNamespace: v.ExportKubeConfig.Secret.Namespace, + KubeConfigServer: v.ExportKubeConfig.Server, + Tolerations: v.Sync.ToHost.Pods.EnforceTolerations, BindAddress: v.ControlPlane.Proxy.BindAddress, Port: v.ControlPlane.Proxy.Port, Name: v.Name, TargetNamespace: v.TargetNamespace, ServiceName: v.ServiceName, - SetOwner: false, + SetOwner: v.Experimental.SyncSettings.SetOwner, SyncAllNodes: v.Sync.FromHost.Nodes.Real.SyncAll, EnableScheduler: v.ControlPlane.VirtualScheduler.Enabled, - DisableFakeKubelets: false, - FakeKubeletIPs: false, + DisableFakeKubelets: !v.Networking.Advanced.ProxyKubelets.ByIP && !v.Networking.Advanced.ProxyKubelets.ByHostname, + FakeKubeletIPs: v.Networking.Advanced.ProxyKubelets.ByIP, ClearNodeImages: v.Sync.FromHost.Nodes.Real.ClearImageStatus, - TranslateImages: nil, NodeSelector: nodeSelector, - ServiceAccount: v.Sync.ToHost.Pods.WorkloadServiceAccount, + ServiceAccount: v.ControlPlane.Advanced.WorkloadServiceAccount.Name, EnforceNodeSelector: true, PluginListenAddress: "localhost:10099", OverrideHosts: v.Sync.ToHost.Pods.RewriteHosts.Enabled, @@ -95,20 +160,15 @@ func (v VirtualClusterConfig) LegacyOptions() (*LegacyVirtualClusterOptions, err Plugins: legacyPlugins, DefaultImageRegistry: v.ControlPlane.Advanced.DefaultImageRegistry, EnforcePodSecurityStandard: v.Policies.PodSecurityStandard, - MapHostServices: nil, - MapVirtualServices: nil, - SyncLabels: nil, + SyncLabels: v.Experimental.SyncSettings.SyncLabels, MountPhysicalHostPaths: false, HostMetricsBindAddress: "0", VirtualMetricsBindAddress: "0", MultiNamespaceMode: v.Experimental.MultiNamespaceMode.Enabled, - NamespaceLabels: nil, SyncAllSecrets: v.Sync.ToHost.Secrets.All, SyncAllConfigMaps: v.Sync.ToHost.ConfigMaps.All, ProxyMetricsServer: v.Observability.Metrics.Proxy.Nodes.Enabled || v.Observability.Metrics.Proxy.Pods.Enabled, DeprecatedSyncNodeChanges: v.Sync.FromHost.Nodes.Real.SyncLabelsTaints, - } - - return retOptions, nil + }, nil } diff --git a/pkg/config/disable_missing_apis.go b/pkg/config/disable_missing_apis.go new file mode 100644 index 000000000..ded772dc6 --- /dev/null +++ b/pkg/config/disable_missing_apis.go @@ -0,0 +1,49 @@ +package config + +import ( + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "k8s.io/klog/v2" +) + +// DisableMissingAPIs checks if the apis are enabled, if any are missing, disable the syncer and print a log +func (v VirtualClusterConfig) DisableMissingAPIs(discoveryClient discovery.DiscoveryInterface) error { + resources, err := discoveryClient.ServerResourcesForGroupVersion("storage.k8s.io/v1") + if err != nil && !kerrors.IsNotFound(err) { + return err + } + + // check if found + if v.Sync.FromHost.CSINodes.Enabled && !findResource(resources, "csinodes") { + v.Sync.FromHost.CSINodes.Enabled = false + klog.Warningf("host kubernetes apiserver not advertising resource csinodes in GroupVersion storage.k8s.io/v1, disabling the syncer") + } + + // check if found + if v.Sync.FromHost.CSIDrivers.Enabled && !findResource(resources, "csidrivers") { + v.Sync.FromHost.CSIDrivers.Enabled = false + klog.Warningf("host kubernetes apiserver not advertising resource csidrivers in GroupVersion storage.k8s.io/v1, disabling the syncer") + } + + // check if found + if v.Sync.FromHost.CSIStorageCapacities.Enabled && !findResource(resources, "csistoragecapacities") { + v.Sync.FromHost.CSIStorageCapacities.Enabled = false + klog.Warningf("host kubernetes apiserver not advertising resource csistoragecapacities in GroupVersion storage.k8s.io/v1, disabling the syncer") + + } + + return nil +} + +func findResource(resources *metav1.APIResourceList, resourcePlural string) bool { + if resources != nil { + for _, r := range resources.APIResources { + if r.Name == resourcePlural { + return true + } + } + } + + return false +} diff --git a/pkg/config/flags.go b/pkg/config/flags.go index db4a1db73..52ab5183c 100644 --- a/pkg/config/flags.go +++ b/pkg/config/flags.go @@ -1,9 +1,6 @@ package config -import ( - "github.com/spf13/pflag" -) - +/* func AddProFlags(flags *pflag.FlagSet, options *VirtualClusterOptions) { flags.StringVar(&options.ProOptions.ProLicenseSecret, "pro-license-secret", "", "If set, vCluster.Pro will try to find this secret to retrieve the vCluster.Pro license.") @@ -103,4 +100,4 @@ func AddFlags(flags *pflag.FlagSet, options *VirtualClusterOptions) { flags.StringVar(&options.DeprecatedSuffix, "suffix", "", "DEPRECATED: use --name instead") flags.StringVar(&options.DeprecatedOwningStatefulSet, "owning-statefulset", "", "DEPRECATED: use --set-owner instead") flags.StringVar(&options.DeprecatedDisableSyncResources, "disable-sync-resources", "", "DEPRECATED: use --sync instead") -} +}*/ diff --git a/pkg/config/options.go b/pkg/config/options.go index 6b0ced1b1..0ea52cc35 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -9,9 +9,6 @@ type LegacyVirtualClusterOptions struct { // PRO Options ProOptions LegacyVirtualClusterProOptions `json:",inline"` - // OSS Options below - Controllers []string `json:"controllers,omitempty"` - ServerCaCert string `json:"serverCaCert,omitempty"` ServerCaKey string `json:"serverCaKey,omitempty"` TLSSANs []string `json:"tlsSans,omitempty"` @@ -35,12 +32,11 @@ type LegacyVirtualClusterOptions struct { SetOwner bool `json:"setOwner,omitempty"` - SyncAllNodes bool `json:"syncAllNodes,omitempty"` - EnableScheduler bool `json:"enableScheduler,omitempty"` - DisableFakeKubelets bool `json:"disableFakeKubelets,omitempty"` - FakeKubeletIPs bool `json:"fakeKubeletIPs,omitempty"` - ClearNodeImages bool `json:"clearNodeImages,omitempty"` - TranslateImages []string `json:"translateImages,omitempty"` + SyncAllNodes bool `json:"syncAllNodes,omitempty"` + EnableScheduler bool `json:"enableScheduler,omitempty"` + DisableFakeKubelets bool `json:"disableFakeKubelets,omitempty"` + FakeKubeletIPs bool `json:"fakeKubeletIPs,omitempty"` + ClearNodeImages bool `json:"clearNodeImages,omitempty"` NodeSelector string `json:"nodeSelector,omitempty"` EnforceNodeSelector bool `json:"enforceNodeSelector,omitempty"` @@ -63,9 +59,6 @@ type LegacyVirtualClusterOptions struct { EnforcePodSecurityStandard string `json:"enforcePodSecurityStandard,omitempty"` - MapHostServices []string `json:"mapHostServices,omitempty"` - MapVirtualServices []string `json:"mapVirtualServices,omitempty"` - SyncLabels []string `json:"syncLabels,omitempty"` // hostpath mapper options @@ -76,10 +69,9 @@ type LegacyVirtualClusterOptions struct { HostMetricsBindAddress string `json:"hostMetricsBindAddress,omitempty"` VirtualMetricsBindAddress string `json:"virtualMetricsBindAddress,omitempty"` - MultiNamespaceMode bool `json:"multiNamespaceMode,omitempty"` - NamespaceLabels []string `json:"namespaceLabels,omitempty"` - SyncAllSecrets bool `json:"syncAllSecrets,omitempty"` - SyncAllConfigMaps bool `json:"syncAllConfigMaps,omitempty"` + MultiNamespaceMode bool `json:"multiNamespaceMode,omitempty"` + SyncAllSecrets bool `json:"syncAllSecrets,omitempty"` + SyncAllConfigMaps bool `json:"syncAllConfigMaps,omitempty"` ProxyMetricsServer bool `json:"proxyMetricsServer,omitempty"` ServiceAccountTokenSecrets bool `json:"serviceAccountTokenSecrets,omitempty"` diff --git a/pkg/config/parse.go b/pkg/config/parse.go index 74373a13a..8eb516ec1 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -8,7 +8,7 @@ import ( "github.com/loft-sh/vcluster/config" ) -func ParseConfig(path string) (*VirtualClusterConfig, error) { +func ParseConfig(path, name string) (*VirtualClusterConfig, error) { rawFile, err := os.ReadFile(path) if err != nil { return nil, err @@ -20,9 +20,12 @@ func ParseConfig(path string) (*VirtualClusterConfig, error) { return nil, err } - retConfig, err := Convert(rawConfig) - if err != nil { - return nil, err + retConfig := &VirtualClusterConfig{ + Config: *rawConfig, + Name: name, + } + if name == "" { + return nil, fmt.Errorf("environment variable VCLUSTER_NAME is not defined") } err = ValidateConfig(retConfig) @@ -32,19 +35,3 @@ func ParseConfig(path string) (*VirtualClusterConfig, error) { return retConfig, nil } - -func Convert(config *config.Config) (*VirtualClusterConfig, error) { - vClusterName := os.Getenv("VCLUSTER_NAME") - if vClusterName == "" { - return nil, fmt.Errorf("environment variable VCLUSTER_NAME is not defined") - } - - retConfig := &VirtualClusterConfig{ - Config: *config, - Name: vClusterName, - } - - // convert legacy options - - return retConfig, nil -} diff --git a/pkg/config/pro_options.go b/pkg/config/pro_options.go index 287fabfcb..e8bd984f3 100644 --- a/pkg/config/pro_options.go +++ b/pkg/config/pro_options.go @@ -1,16 +1,12 @@ package config type LegacyVirtualClusterProOptions struct { - RemoteKubeConfig string `json:"remoteKubeConfig,omitempty"` - RemoteNamespace string `json:"remoteNamespace,omitempty"` - RemoteServiceName string `json:"remoteServiceName,omitempty"` - EnforceValidatingHooks []string `json:"enforceValidatingHooks"` - EnforceMutatingHooks []string `json:"enforceMutatingHooks"` - EtcdReplicas int `json:"etcdReplicas,omitempty"` - IntegratedCoredns bool `json:"integratedCoreDNS,omitempty"` - UseCoreDNSPlugin bool `json:"useCoreDNSPlugin,omitempty"` - EtcdEmbedded bool `json:"etcdEmbedded,omitempty"` - MigrateFrom string `json:"migrateFrom,omitempty"` + RemoteKubeConfig string `json:"remoteKubeConfig,omitempty"` + RemoteNamespace string `json:"remoteNamespace,omitempty"` + RemoteServiceName string `json:"remoteServiceName,omitempty"` + EtcdReplicas int `json:"etcdReplicas,omitempty"` + IntegratedCoredns bool `json:"integratedCoreDNS,omitempty"` + EtcdEmbedded bool `json:"etcdEmbedded,omitempty"` NoopSyncer bool `json:"noopSyncer,omitempty"` SyncKubernetesService bool `json:"synck8sService,omitempty"` diff --git a/pkg/config/validation.go b/pkg/config/validation.go index 763595da8..9407c08a2 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -8,27 +8,108 @@ import ( "github.com/ghodss/yaml" "github.com/loft-sh/vcluster/config" + "github.com/loft-sh/vcluster/pkg/util/toleration" "github.com/samber/lo" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/klog/v2" +) + +var allowedPodSecurityStandards = map[string]bool{ + "privileged": true, + "baseline": true, + "restricted": true, +} + +var ( + verbs = []string{"get", "list", "create", "update", "patch", "watch", "delete", "deletecollection"} ) func ValidateConfig(config *VirtualClusterConfig) error { + // check the value of pod security standard + if config.Policies.PodSecurityStandard != "" && !allowedPodSecurityStandards[config.Policies.PodSecurityStandard] { + return fmt.Errorf("invalid argument enforce-pod-security-standard=%s, must be one of: privileged, baseline, restricted", config.Policies.PodSecurityStandard) + } + + // parse tolerations + for _, t := range config.Sync.ToHost.Pods.EnforceTolerations { + _, err := toleration.ParseToleration(t) + if err != nil { + return err + } + } + + // check if enable scheduler works correctly + if config.ControlPlane.VirtualScheduler.Enabled && !config.Sync.FromHost.Nodes.Real.SyncAll && len(config.Sync.FromHost.Nodes.Real.Selector.Labels) == 0 { + config.Sync.FromHost.Nodes.Real.SyncAll = true + } + + // enable additional controllers required for scheduling with storage + if config.ControlPlane.VirtualScheduler.Enabled && config.Sync.ToHost.PersistentVolumeClaims.Enabled { + if !config.Sync.FromHost.CSINodes.Enabled { + klog.Warningf("persistentvolumeclaim syncing and scheduler enabled, but csiNodes syncing disabled. This may result in incorrect pod scheduling.") + } + if !config.Sync.FromHost.CSIStorageCapacities.Enabled { + klog.Warningf("persistentvolumeclaim syncing and scheduler enabled, but csiStorageCapacities syncing disabled. This may result in incorrect pod scheduling.") + } + if !config.Sync.FromHost.CSIDrivers.Enabled { + klog.Warningf("persistentvolumeclaim syncing and scheduler enabled, but csiDrivers syncing disabled. This may result in incorrect pod scheduling.") + } + if !config.Sync.FromHost.StorageClasses.Enabled && !config.Sync.ToHost.StorageClasses.Enabled { + return fmt.Errorf("persistentvolumeclaim syncing and scheduler enabled, but both sync.toHost.storageClasses and sync.fromHost.storageClasses syncers disabled") + } + } + + // check if nodes controller needs to be enabled + if config.ControlPlane.VirtualScheduler.Enabled && !config.Sync.FromHost.Nodes.Real.Enabled { + return fmt.Errorf("sync.fromHost.nodes.real.enabled is false, but required if using virtual scheduler") + } + + // check if storage classes and host storage classes are enabled at the same time + if config.Sync.FromHost.StorageClasses.Enabled && config.Sync.ToHost.StorageClasses.Enabled { + return fmt.Errorf("you cannot enable both sync.fromHost.storageClasses.enabled and sync.toHost.storageClasses.enabled at the same time. Choose only one of them") + } + + // validate central admission control err := validateCentralAdmissionControl(config) if err != nil { return err } + // validate generic sync config err = validateGenericSyncConfig(config.Experimental.GenericSync) if err != nil { return fmt.Errorf("validate experimental.genericSync") } + // validate distro + err = validateDistro(config) + if err != nil { + return err + } + return nil } -var ( - verbs = []string{"get", "list", "create", "update", "patch", "watch", "delete", "deletecollection"} -) +func validateDistro(config *VirtualClusterConfig) error { + enabledDistros := 0 + if config.Config.ControlPlane.Distro.K3S.Enabled { + enabledDistros++ + } + if config.Config.ControlPlane.Distro.K0S.Enabled { + enabledDistros++ + } + if config.Config.ControlPlane.Distro.K8S.Enabled { + enabledDistros++ + } + if config.Config.ControlPlane.Distro.EKS.Enabled { + enabledDistros++ + } + + if enabledDistros > 1 { + return fmt.Errorf("please only enable a single distribution") + } + return nil +} func validateGenericSyncConfig(config config.ExperimentalGenericSync) error { err := validateExportDuplicates(config.Exports) diff --git a/pkg/constants/controllers.go b/pkg/constants/controllers.go deleted file mode 100644 index 9e4c20536..000000000 --- a/pkg/constants/controllers.go +++ /dev/null @@ -1,50 +0,0 @@ -package constants - -import "k8s.io/apimachinery/pkg/util/sets" - -var ExistingControllers = sets.New( - "services", - "configmaps", - "secrets", - "endpoints", - "pods", - "events", - "fake-nodes", - "fake-persistentvolumes", - "persistentvolumeclaims", - "ingresses", - "ingressclasses", - "nodes", - "persistentvolumes", - "storageclasses", - "hoststorageclasses", - "priorityclasses", - "networkpolicies", - "volumesnapshots", - "poddisruptionbudgets", - "serviceaccounts", - "csinodes", - "csidrivers", - "csistoragecapacities", - "namespaces", -) - -var DefaultEnabledControllers = sets.New( - // helm charts need to be updated when changing this! - // values.yaml and template/_helpers.tpl reference these - "services", - "configmaps", - "secrets", - "endpoints", - "pods", - "events", - "persistentvolumeclaims", - "fake-nodes", - "fake-persistentvolumes", -) - -var SchedulerRequiredControllers = sets.New( - "csinodes", - "csidrivers", - "csistoragecapacities", -) diff --git a/pkg/constants/distro.go b/pkg/constants/distro.go deleted file mode 100644 index 824b616d4..000000000 --- a/pkg/constants/distro.go +++ /dev/null @@ -1,21 +0,0 @@ -package constants - -import "os" - -const ( - K3SDistro = "k3s" - K8SDistro = "k8s" - K0SDistro = "k0s" - EKSDistro = "eks" - Unknown = "unknown" -) - -func GetVClusterDistro() string { - distro := os.Getenv("VCLUSTER_DISTRO") - switch distro { - case K3SDistro, K8SDistro, K0SDistro, EKSDistro: - return distro - default: - return Unknown - } -} diff --git a/pkg/controllers/generic/patcher.go b/pkg/controllers/generic/patcher.go index 89e9c0162..07521f910 100644 --- a/pkg/controllers/generic/patcher.go +++ b/pkg/controllers/generic/patcher.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/loft-sh/vcluster/pkg/genericsyncconfig" + "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/log" "github.com/loft-sh/vcluster/pkg/patches" "github.com/pkg/errors" @@ -25,7 +25,7 @@ type patcher struct { statusIsSubresource bool } -func (s *patcher) ApplyPatches(ctx context.Context, fromObj, toObj client.Object, patchesConfig, reversePatchesConfig []*genericsyncconfig.Patch, translateMetadata func(vObj client.Object) (client.Object, error), nameResolver patches.NameResolver) (client.Object, error) { +func (s *patcher) ApplyPatches(ctx context.Context, fromObj, toObj client.Object, patchesConfig, reversePatchesConfig []*config.Patch, translateMetadata func(vObj client.Object) (client.Object, error), nameResolver patches.NameResolver) (client.Object, error) { translatedObject, err := translateMetadata(fromObj) if err != nil { return nil, errors.Wrap(err, "translate object") @@ -76,7 +76,7 @@ func (s *patcher) ApplyPatches(ctx context.Context, fromObj, toObj client.Object return outObject, nil } -func (s *patcher) ApplyReversePatches(ctx context.Context, fromObj, otherObj client.Object, reversePatchConfig []*genericsyncconfig.Patch, nameResolver patches.NameResolver) (controllerutil.OperationResult, error) { +func (s *patcher) ApplyReversePatches(ctx context.Context, fromObj, otherObj client.Object, reversePatchConfig []*config.Patch, nameResolver patches.NameResolver) (controllerutil.OperationResult, error) { originalUnstructured, err := toUnstructured(fromObj) if err != nil { return controllerutil.OperationResultNone, err diff --git a/pkg/controllers/register.go b/pkg/controllers/register.go index 008ed4e1e..792d69e57 100644 --- a/pkg/controllers/register.go +++ b/pkg/controllers/register.go @@ -8,7 +8,30 @@ import ( "time" "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd" + config2 "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/controllers/resources/configmaps" + "github.com/loft-sh/vcluster/pkg/controllers/resources/csidrivers" + "github.com/loft-sh/vcluster/pkg/controllers/resources/csinodes" + "github.com/loft-sh/vcluster/pkg/controllers/resources/csistoragecapacities" + "github.com/loft-sh/vcluster/pkg/controllers/resources/endpoints" + "github.com/loft-sh/vcluster/pkg/controllers/resources/events" + "github.com/loft-sh/vcluster/pkg/controllers/resources/ingressclasses" + "github.com/loft-sh/vcluster/pkg/controllers/resources/ingresses" + "github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces" + "github.com/loft-sh/vcluster/pkg/controllers/resources/networkpolicies" + "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" + "github.com/loft-sh/vcluster/pkg/controllers/resources/persistentvolumeclaims" + "github.com/loft-sh/vcluster/pkg/controllers/resources/persistentvolumes" + "github.com/loft-sh/vcluster/pkg/controllers/resources/poddisruptionbudgets" + "github.com/loft-sh/vcluster/pkg/controllers/resources/pods" + "github.com/loft-sh/vcluster/pkg/controllers/resources/priorityclasses" + "github.com/loft-sh/vcluster/pkg/controllers/resources/secrets" + "github.com/loft-sh/vcluster/pkg/controllers/resources/serviceaccounts" + "github.com/loft-sh/vcluster/pkg/controllers/resources/storageclasses" + "github.com/loft-sh/vcluster/pkg/controllers/resources/volumesnapshots/volumesnapshotclasses" + "github.com/loft-sh/vcluster/pkg/controllers/resources/volumesnapshots/volumesnapshotcontents" + "github.com/loft-sh/vcluster/pkg/controllers/resources/volumesnapshots/volumesnapshots" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "github.com/loft-sh/vcluster/pkg/util/translate" "k8s.io/apimachinery/pkg/util/wait" @@ -26,35 +49,12 @@ import ( ctrl "sigs.k8s.io/controller-runtime" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/loft-sh/vcluster/pkg/controllers/k8sdefaultendpoint" - "github.com/loft-sh/vcluster/pkg/controllers/manifests" - "github.com/loft-sh/vcluster/pkg/controllers/resources/csidrivers" - "github.com/loft-sh/vcluster/pkg/controllers/resources/csinodes" - "github.com/loft-sh/vcluster/pkg/controllers/resources/csistoragecapacities" - "github.com/loft-sh/vcluster/pkg/controllers/resources/ingressclasses" - "github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces" - "github.com/loft-sh/vcluster/pkg/controllers/resources/serviceaccounts" - "github.com/loft-sh/log" "github.com/loft-sh/vcluster/pkg/controllers/coredns" + "github.com/loft-sh/vcluster/pkg/controllers/k8sdefaultendpoint" + "github.com/loft-sh/vcluster/pkg/controllers/manifests" "github.com/loft-sh/vcluster/pkg/controllers/podsecurity" - "github.com/loft-sh/vcluster/pkg/controllers/resources/configmaps" - "github.com/loft-sh/vcluster/pkg/controllers/resources/endpoints" - "github.com/loft-sh/vcluster/pkg/controllers/resources/events" - "github.com/loft-sh/vcluster/pkg/controllers/resources/ingresses" - "github.com/loft-sh/vcluster/pkg/controllers/resources/networkpolicies" - "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" - "github.com/loft-sh/vcluster/pkg/controllers/resources/persistentvolumeclaims" - "github.com/loft-sh/vcluster/pkg/controllers/resources/persistentvolumes" - "github.com/loft-sh/vcluster/pkg/controllers/resources/poddisruptionbudgets" - "github.com/loft-sh/vcluster/pkg/controllers/resources/pods" - "github.com/loft-sh/vcluster/pkg/controllers/resources/priorityclasses" - "github.com/loft-sh/vcluster/pkg/controllers/resources/secrets" "github.com/loft-sh/vcluster/pkg/controllers/resources/services" - "github.com/loft-sh/vcluster/pkg/controllers/resources/storageclasses" - "github.com/loft-sh/vcluster/pkg/controllers/resources/volumesnapshots/volumesnapshotclasses" - "github.com/loft-sh/vcluster/pkg/controllers/resources/volumesnapshots/volumesnapshotcontents" - "github.com/loft-sh/vcluster/pkg/controllers/resources/volumesnapshots/volumesnapshots" synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" syncertypes "github.com/loft-sh/vcluster/pkg/types" "github.com/loft-sh/vcluster/pkg/util/loghelper" @@ -62,29 +62,42 @@ import ( "golang.org/x/sync/errgroup" ) -var ResourceControllers = map[string][]func(*synccontext.RegisterContext) (syncertypes.Object, error){ - "services": {services.New}, - "configmaps": {configmaps.New}, - "secrets": {secrets.New}, - "endpoints": {endpoints.New}, - "pods": {pods.New}, - "events": {events.New}, - "persistentvolumeclaims": {persistentvolumeclaims.New}, - "ingresses": {ingresses.New}, - "ingressclasses": {ingressclasses.New}, - "storageclasses": {storageclasses.New}, - "hoststorageclasses": {storageclasses.NewHostStorageClassSyncer}, - "priorityclasses": {priorityclasses.New}, - "nodes,fake-nodes": {nodes.New}, - "poddisruptionbudgets": {poddisruptionbudgets.New}, - "networkpolicies": {networkpolicies.New}, - "volumesnapshots": {volumesnapshotclasses.New, volumesnapshots.New, volumesnapshotcontents.New}, - "serviceaccounts": {serviceaccounts.New}, - "csinodes": {csinodes.New}, - "csidrivers": {csidrivers.New}, - "csistoragecapacities": {csistoragecapacities.New}, - "namespaces": {namespaces.New}, - "persistentvolumes,fake-persistentvolumes": {persistentvolumes.New}, +type initFunction func(*synccontext.RegisterContext) (syncertypes.Object, error) + +func getSyncers(ctx *config.ControllerContext) []initFunction { + return []initFunction{ + isEnabled(ctx.Config.Sync.ToHost.Services.Enabled, services.New), + isEnabled(ctx.Config.Sync.ToHost.ConfigMaps.Enabled, configmaps.New), + isEnabled(ctx.Config.Sync.ToHost.Secrets.Enabled, secrets.New), + isEnabled(ctx.Config.Sync.ToHost.Endpoints.Enabled, endpoints.New), + isEnabled(ctx.Config.Sync.ToHost.Pods.Enabled, pods.New), + isEnabled(ctx.Config.Sync.FromHost.Events.Enabled, events.New), + isEnabled(ctx.Config.Sync.ToHost.PersistentVolumeClaims.Enabled, persistentvolumeclaims.New), + isEnabled(ctx.Config.Sync.ToHost.Ingresses.Enabled, ingresses.New), + isEnabled(ctx.Config.Sync.FromHost.IngressClasses.Enabled, ingressclasses.New), + isEnabled(ctx.Config.Sync.ToHost.StorageClasses.Enabled, storageclasses.New), + isEnabled(ctx.Config.Sync.FromHost.StorageClasses.Enabled, storageclasses.NewHostStorageClassSyncer), + isEnabled(ctx.Config.Sync.ToHost.PriorityClasses.Enabled, priorityclasses.New), + isEnabled(ctx.Config.Sync.ToHost.PodDisruptionBudgets.Enabled, poddisruptionbudgets.New), + isEnabled(ctx.Config.Sync.ToHost.NetworkPolicies.Enabled, networkpolicies.New), + isEnabled(ctx.Config.Sync.ToHost.VolumeSnapshots.Enabled, volumesnapshotclasses.New), + isEnabled(ctx.Config.Sync.ToHost.VolumeSnapshots.Enabled, volumesnapshots.New), + isEnabled(ctx.Config.Sync.ToHost.VolumeSnapshots.Enabled, volumesnapshotcontents.New), + isEnabled(ctx.Config.Sync.ToHost.ServiceAccounts.Enabled, serviceaccounts.New), + isEnabled(ctx.Config.Sync.FromHost.CSINodes.Enabled, csinodes.New), + isEnabled(ctx.Config.Sync.FromHost.CSIDrivers.Enabled, csidrivers.New), + isEnabled(ctx.Config.Sync.FromHost.CSIStorageCapacities.Enabled, csistoragecapacities.New), + isEnabled(ctx.Config.Experimental.MultiNamespaceMode.Enabled, namespaces.New), + persistentvolumes.New, + nodes.New, + } +} + +func isEnabled(enabled bool, fn initFunction) initFunction { + if enabled { + return fn + } + return nil } func Create(ctx *config.ControllerContext) ([]syncertypes.Object, error) { @@ -92,22 +105,18 @@ func Create(ctx *config.ControllerContext) ([]syncertypes.Object, error) { // register controllers for resource synchronization syncers := []syncertypes.Object{} - for k, v := range ResourceControllers { - for _, controllerNew := range v { - controllers := strings.Split(k, ",") - for _, controller := range controllers { - if ctx.Controllers.Has(controller) { - loghelper.Infof("Start %s sync controller", controller) - createdController, err := controllerNew(registerContext) - if err != nil { - return nil, errors.Wrapf(err, "register %s controller", controller) - } - - syncers = append(syncers, createdController) - break - } - } + for _, newSyncer := range getSyncers(ctx) { + if newSyncer == nil { + continue + } + + createdController, err := newSyncer(registerContext) + if err != nil { + return nil, errors.Wrapf(err, "register %s controller", createdController.Name()) } + + loghelper.Infof("Start %s sync controller", createdController.Name()) + syncers = append(syncers, createdController) } return syncers, nil @@ -277,13 +286,13 @@ func RegisterInitManifestsController(controllerCtx *config.ControllerContext) er } func RegisterServiceSyncControllers(ctx *config.ControllerContext) error { - hostNamespace := ctx.Options.TargetNamespace + hostNamespace := ctx.Config.TargetNamespace if ctx.Config.Experimental.MultiNamespaceMode.Enabled { hostNamespace = ctx.CurrentNamespace } - if len(ctx.Options.MapHostServices) > 0 { - mapping, err := parseMapping(ctx.Options.MapHostServices, hostNamespace, "") + if len(ctx.Config.Networking.ReplicateServices.FromHost) > 0 { + mapping, err := parseMapping(ctx.Config.Networking.ReplicateServices.FromHost, hostNamespace, "") if err != nil { return errors.Wrap(err, "parse physical service mapping") } @@ -329,8 +338,8 @@ func RegisterServiceSyncControllers(ctx *config.ControllerContext) error { } } - if len(ctx.Options.MapVirtualServices) > 0 { - mapping, err := parseMapping(ctx.Options.MapVirtualServices, "", hostNamespace) + if len(ctx.Config.Networking.ReplicateServices.ToHost) > 0 { + mapping, err := parseMapping(ctx.Config.Networking.ReplicateServices.ToHost, "", hostNamespace) if err != nil { return errors.Wrap(err, "parse physical service mapping") } @@ -356,43 +365,39 @@ func RegisterServiceSyncControllers(ctx *config.ControllerContext) error { return nil } -func parseMapping(mappings []string, fromDefaultNamespace, toDefaultNamespace string) (map[string]types.NamespacedName, error) { +func parseMapping(mappings []config2.ServiceMapping, fromDefaultNamespace, toDefaultNamespace string) (map[string]types.NamespacedName, error) { ret := map[string]types.NamespacedName{} for _, m := range mappings { - splitted := strings.Split(m, "=") - if len(splitted) != 2 { - return nil, fmt.Errorf("invalid service mapping, please use namespace1/service1=service2") - } else if len(splitted[0]) == 0 || len(splitted[1]) == 0 { - return nil, fmt.Errorf("invalid service mapping, please use namespace1/service1=service2") - } + from := m.From + to := m.To - fromSplitted := strings.Split(splitted[0], "/") + fromSplitted := strings.Split(from, "/") if len(fromSplitted) == 1 { if fromDefaultNamespace == "" { return nil, fmt.Errorf("invalid service mapping, please use namespace1/service1=service2") } - splitted[0] = fromDefaultNamespace + "/" + splitted[0] + from = fromDefaultNamespace + "/" + from } else if len(fromSplitted) != 2 { return nil, fmt.Errorf("invalid service mapping, please use namespace1/service1=service2") } - toSplitted := strings.Split(splitted[1], "/") + toSplitted := strings.Split(to, "/") if len(toSplitted) == 1 { if toDefaultNamespace == "" { return nil, fmt.Errorf("invalid service mapping, please use namespace1/service1=namespace2/service2") } - ret[splitted[0]] = types.NamespacedName{ + ret[from] = types.NamespacedName{ Namespace: toDefaultNamespace, - Name: splitted[1], + Name: to, } } else if len(toSplitted) == 2 { if toDefaultNamespace != "" { return nil, fmt.Errorf("invalid service mapping, please use namespace1/service1=service2") } - ret[splitted[0]] = types.NamespacedName{ + ret[from] = types.NamespacedName{ Namespace: toSplitted[0], Name: toSplitted[1], } diff --git a/pkg/controllers/resources/configmaps/syncer.go b/pkg/controllers/resources/configmaps/syncer.go index ae71c0e29..cf38c5f4b 100644 --- a/pkg/controllers/resources/configmaps/syncer.go +++ b/pkg/controllers/resources/configmaps/syncer.go @@ -27,7 +27,7 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { return &configMapSyncer{ NamespacedTranslator: t, - syncAllConfigMaps: ctx.Options.SyncAllConfigMaps, + syncAllConfigMaps: ctx.Config.Sync.ToHost.ConfigMaps.All, }, nil } diff --git a/pkg/controllers/resources/csistoragecapacities/syncer.go b/pkg/controllers/resources/csistoragecapacities/syncer.go index 6255b66b1..c2c3bdcab 100644 --- a/pkg/controllers/resources/csistoragecapacities/syncer.go +++ b/pkg/controllers/resources/csistoragecapacities/syncer.go @@ -22,8 +22,8 @@ import ( func New(ctx *synccontext.RegisterContext) (syncertypes.Object, error) { return &csistoragecapacitySyncer{ - storageClassSyncEnabled: ctx.Controllers.Has("storageclasses"), - hostStorageClassSyncEnabled: ctx.Controllers.Has("hoststorageclasses"), + storageClassSyncEnabled: ctx.Config.Sync.ToHost.StorageClasses.Enabled, + hostStorageClassSyncEnabled: ctx.Config.Sync.FromHost.StorageClasses.Enabled, physicalClient: ctx.PhysicalManager.GetClient(), }, nil } diff --git a/pkg/controllers/resources/namespaces/syncer.go b/pkg/controllers/resources/namespaces/syncer.go index c7c9e8c59..f4a0d81df 100644 --- a/pkg/controllers/resources/namespaces/syncer.go +++ b/pkg/controllers/resources/namespaces/syncer.go @@ -1,9 +1,6 @@ package namespaces import ( - "fmt" - "strings" - "github.com/loft-sh/vcluster/pkg/constants" synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" "github.com/loft-sh/vcluster/pkg/controllers/syncer/translator" @@ -11,8 +8,6 @@ import ( "github.com/loft-sh/vcluster/pkg/util/translate" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/validation" - "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -31,17 +26,16 @@ const ( ) func New(ctx *synccontext.RegisterContext) (syncertypes.Object, error) { - namespaceLabels, err := parseNamespaceLabels(ctx.Options.NamespaceLabels) - if err != nil { - return nil, fmt.Errorf("invalid value of the namespace-labels flag: %w", err) + namespaceLabels := map[string]string{} + for k, v := range ctx.Config.Experimental.MultiNamespaceMode.NamespaceLabels { + namespaceLabels[k] = v } - - namespaceLabels[VClusterNameAnnotation] = ctx.Options.Name + namespaceLabels[VClusterNameAnnotation] = ctx.Config.Name namespaceLabels[VClusterNamespaceAnnotation] = ctx.CurrentNamespace return &namespaceSyncer{ Translator: translator.NewClusterTranslator(ctx, "namespace", &corev1.Namespace{}, NamespaceNameTranslator, excludedAnnotations...), - workloadServiceAccountName: ctx.Options.ServiceAccount, + workloadServiceAccountName: ctx.Config.ControlPlane.Advanced.WorkloadServiceAccount.Name, namespaceLabels: namespaceLabels, }, nil } @@ -106,20 +100,3 @@ func (s *namespaceSyncer) EnsureWorkloadServiceAccount(ctx *synccontext.SyncCont func NamespaceNameTranslator(vName string, _ client.Object) string { return translate.Default.PhysicalNamespace(vName) } - -func parseNamespaceLabels(labels []string) (map[string]string, error) { - out := map[string]string{} - for _, v := range labels { - parts := strings.SplitN(v, "=", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("incorrect format, expected: key=value got: %s", v) - } - out[parts[0]] = parts[1] - } - errs := validation.ValidateLabels(out, field.NewPath("namespace-labels")) - if len(errs) != 0 { - return nil, fmt.Errorf("invalid labels: %v", errs) - } - - return out, nil -} diff --git a/pkg/controllers/resources/nodes/fake_syncer.go b/pkg/controllers/resources/nodes/fake_syncer.go index 32dc0db10..21e0c83cc 100644 --- a/pkg/controllers/resources/nodes/fake_syncer.go +++ b/pkg/controllers/resources/nodes/fake_syncer.go @@ -35,7 +35,7 @@ var ( func NewFakeSyncer(ctx *synccontext.RegisterContext, nodeService nodeservice.Provider) (syncer.Object, error) { return &fakeNodeSyncer{ nodeServiceProvider: nodeService, - fakeKubeletIPs: ctx.Options.FakeKubeletIPs, + fakeKubeletIPs: ctx.Config.Networking.Advanced.ProxyKubelets.ByIP, }, nil } diff --git a/pkg/controllers/resources/nodes/register.go b/pkg/controllers/resources/nodes/register.go index 0696946ee..cca51c80c 100644 --- a/pkg/controllers/resources/nodes/register.go +++ b/pkg/controllers/resources/nodes/register.go @@ -16,8 +16,8 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { return nil, err } - nodeService := nodeservice.NewNodeServiceProvider(ctx.Options.ServiceName, ctx.CurrentNamespace, ctx.CurrentNamespaceClient, ctx.VirtualManager.GetClient(), uncachedVirtualClient) - if !ctx.Controllers.Has("nodes") { + nodeService := nodeservice.NewNodeServiceProvider(ctx.Config.ServiceName, ctx.CurrentNamespace, ctx.CurrentNamespaceClient, ctx.VirtualManager.GetClient(), uncachedVirtualClient) + if !ctx.Config.Sync.FromHost.Nodes.Real.Enabled { return NewFakeSyncer(ctx, nodeService) } diff --git a/pkg/controllers/resources/nodes/syncer.go b/pkg/controllers/resources/nodes/syncer.go index 8e61599f3..f6fa76d47 100644 --- a/pkg/controllers/resources/nodes/syncer.go +++ b/pkg/controllers/resources/nodes/syncer.go @@ -31,21 +31,17 @@ import ( ) func NewSyncer(ctx *synccontext.RegisterContext, nodeServiceProvider nodeservice.Provider) (syncertypes.Object, error) { - var err error var nodeSelector labels.Selector - if ctx.Options.SyncAllNodes { + if ctx.Config.Sync.FromHost.Nodes.Real.SyncAll { nodeSelector = labels.Everything() - } else if ctx.Options.NodeSelector != "" { - nodeSelector, err = labels.Parse(ctx.Options.NodeSelector) - if err != nil { - return nil, errors.Wrap(err, "parse node selector") - } + } else if len(ctx.Config.Sync.FromHost.Nodes.Real.Selector.Labels) > 0 { + nodeSelector = labels.Set(ctx.Config.Sync.FromHost.Nodes.Real.Selector.Labels).AsSelector() } // parse tolerations var tolerations []*corev1.Toleration - if len(ctx.Options.Tolerations) > 0 { - for _, t := range ctx.Options.Tolerations { + if len(ctx.Config.Sync.ToHost.Pods.EnforceTolerations) > 0 { + for _, t := range ctx.Config.Sync.ToHost.Pods.EnforceTolerations { tol, err := toleration.ParseToleration(t) if err == nil { tolerations = append(tolerations, &tol) @@ -54,13 +50,13 @@ func NewSyncer(ctx *synccontext.RegisterContext, nodeServiceProvider nodeservice } return &nodeSyncer{ - enableScheduler: ctx.Options.EnableScheduler, + enableScheduler: ctx.Config.ControlPlane.VirtualScheduler.Enabled, - enforceNodeSelector: ctx.Options.EnforceNodeSelector, + enforceNodeSelector: true, nodeSelector: nodeSelector, - clearImages: ctx.Options.ClearNodeImages, - useFakeKubelets: !ctx.Options.DisableFakeKubelets, - fakeKubeletIPs: ctx.Options.FakeKubeletIPs, + clearImages: ctx.Config.Sync.FromHost.Nodes.Real.ClearImageStatus, + useFakeKubelets: ctx.Config.Networking.Advanced.ProxyKubelets.ByHostname || ctx.Config.Networking.Advanced.ProxyKubelets.ByIP, + fakeKubeletIPs: ctx.Config.Networking.Advanced.ProxyKubelets.ByIP, physicalClient: ctx.PhysicalManager.GetClient(), virtualClient: ctx.VirtualManager.GetClient(), diff --git a/pkg/controllers/resources/persistentvolumeclaims/syncer.go b/pkg/controllers/resources/persistentvolumeclaims/syncer.go index e64bf534a..726100614 100644 --- a/pkg/controllers/resources/persistentvolumeclaims/syncer.go +++ b/pkg/controllers/resources/persistentvolumeclaims/syncer.go @@ -35,14 +35,14 @@ const ( ) func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { - storageClassesEnabled := ctx.Controllers.Has("storageclasses") + storageClassesEnabled := ctx.Config.Sync.ToHost.StorageClasses.Enabled excludedAnnotations := []string{bindCompletedAnnotation, boundByControllerAnnotation, storageProvisionerAnnotation} return &persistentVolumeClaimSyncer{ NamespacedTranslator: translator.NewNamespacedTranslator(ctx, "persistent-volume-claim", &corev1.PersistentVolumeClaim{}, excludedAnnotations...), storageClassesEnabled: storageClassesEnabled, - schedulerEnabled: ctx.Options.EnableScheduler, - useFakePersistentVolumes: !ctx.Controllers.Has("persistentvolumes"), + schedulerEnabled: ctx.Config.ControlPlane.VirtualScheduler.Enabled, + useFakePersistentVolumes: !ctx.Config.Sync.ToHost.PersistentVolumes.Enabled, }, nil } diff --git a/pkg/controllers/resources/persistentvolumes/register.go b/pkg/controllers/resources/persistentvolumes/register.go index 5928fa1fb..1afdb4c64 100644 --- a/pkg/controllers/resources/persistentvolumes/register.go +++ b/pkg/controllers/resources/persistentvolumes/register.go @@ -6,7 +6,7 @@ import ( ) func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { - if !ctx.Controllers.Has("persistentvolumes") { + if !ctx.Config.Sync.ToHost.PersistentVolumes.Enabled { return NewFakeSyncer(ctx) } diff --git a/pkg/controllers/resources/pods/syncer.go b/pkg/controllers/resources/pods/syncer.go index 79ac2b884..e2e59b7c3 100644 --- a/pkg/controllers/resources/pods/syncer.go +++ b/pkg/controllers/resources/pods/syncer.go @@ -49,21 +49,16 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { // parse node selector var nodeSelector *metav1.LabelSelector - if ctx.Options.EnforceNodeSelector && ctx.Options.NodeSelector != "" { - nodeSelector, err = metav1.ParseToLabelSelector(ctx.Options.NodeSelector) - if err != nil { - return nil, errors.Wrap(err, "parse node selector") - } else if len(nodeSelector.MatchExpressions) > 0 { - return nil, errors.New("match expressions in the node selector are not supported") - } else if len(nodeSelector.MatchLabels) == 0 { - return nil, errors.New("at least one label=value pair has to be defined in the label selector") + if ctx.Config.Sync.FromHost.Nodes.Real.Enabled && len(ctx.Config.Sync.FromHost.Nodes.Real.Selector.Labels) > 0 { + nodeSelector = &metav1.LabelSelector{ + MatchLabels: ctx.Config.Sync.FromHost.Nodes.Real.Selector.Labels, } } // parse tolerations var tolerations []*corev1.Toleration - if len(ctx.Options.Tolerations) > 0 { - for _, t := range ctx.Options.Tolerations { + if len(ctx.Config.Sync.ToHost.Pods.EnforceTolerations) > 0 { + for _, t := range ctx.Config.Sync.ToHost.Pods.EnforceTolerations { tol, err := toleration.ParseToleration(t) if err == nil { tolerations = append(tolerations, &tol) @@ -83,8 +78,8 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { return &podSyncer{ NamespacedTranslator: namespacedTranslator, - serviceName: ctx.Options.ServiceName, - enableScheduler: ctx.Options.EnableScheduler, + serviceName: ctx.Config.ServiceName, + enableScheduler: ctx.Config.ControlPlane.VirtualScheduler.Enabled, virtualClusterClient: virtualClusterClient, physicalClusterClient: physicalClusterClient, @@ -93,7 +88,7 @@ func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { nodeSelector: nodeSelector, tolerations: tolerations, - podSecurityStandard: ctx.Options.EnforcePodSecurityStandard, + podSecurityStandard: ctx.Config.Policies.PodSecurityStandard, }, nil } diff --git a/pkg/controllers/resources/pods/translate/hostpath.go b/pkg/controllers/resources/pods/translate/hostpath.go index 46401e0b3..d39d35161 100644 --- a/pkg/controllers/resources/pods/translate/hostpath.go +++ b/pkg/controllers/resources/pods/translate/hostpath.go @@ -128,9 +128,7 @@ func (t *translator) rewriteHostPaths(pPod *corev1.Pod) { } } - if t.hostpathMountPropagation { - t.ensureMountPropagation(pPod) - } + t.ensureMountPropagation(pPod) } } diff --git a/pkg/controllers/resources/pods/translate/translator.go b/pkg/controllers/resources/pods/translate/translator.go index 1c3e8b43b..9eeee060c 100644 --- a/pkg/controllers/resources/pods/translate/translator.go +++ b/pkg/controllers/resources/pods/translate/translator.go @@ -58,16 +58,12 @@ type Translator interface { } func NewTranslator(ctx *synccontext.RegisterContext, eventRecorder record.EventRecorder) (Translator, error) { - imageTranslator, err := NewImageTranslator(ctx.Options.TranslateImages) + imageTranslator, err := NewImageTranslator(ctx.Config.Sync.ToHost.Pods.TranslateImage) if err != nil { return nil, err } - name := ctx.Options.Name - if name == "" { - name = ctx.Options.ServiceName - } - + name := ctx.Config.Name virtualPath := fmt.Sprintf(VirtualPathTemplate, ctx.CurrentNamespace, name) virtualLogsPath := path.Join(virtualPath, "log") virtualKubeletPath := path.Join(virtualPath, "kubelet") @@ -81,23 +77,23 @@ func NewTranslator(ctx *synccontext.RegisterContext, eventRecorder record.EventR eventRecorder: eventRecorder, log: loghelper.New("pods-syncer-translator"), - defaultImageRegistry: ctx.Options.DefaultImageRegistry, - - serviceAccountSecretsEnabled: ctx.Options.ServiceAccountTokenSecrets, - clusterDomain: ctx.Options.ClusterDomain, - serviceAccount: ctx.Options.ServiceAccount, - overrideHosts: ctx.Options.OverrideHosts, - overrideHostsImage: ctx.Options.OverrideHostsContainerImage, - serviceAccountsEnabled: ctx.Controllers.Has("serviceaccounts"), - priorityClassesEnabled: ctx.Controllers.Has("priorityclasses"), - enableScheduler: ctx.Options.EnableScheduler, - syncedLabels: ctx.Options.SyncLabels, - - mountPhysicalHostPaths: ctx.Options.MountPhysicalHostPaths, - hostpathMountPropagation: true, - virtualLogsPath: virtualLogsPath, - virtualPodLogsPath: filepath.Join(virtualLogsPath, "pods"), - virtualKubeletPodPath: filepath.Join(virtualKubeletPath, "pods"), + defaultImageRegistry: ctx.Config.ControlPlane.Advanced.DefaultImageRegistry, + + serviceAccountSecretsEnabled: ctx.Config.Sync.ToHost.Pods.UseSecretsForSATokens, + clusterDomain: ctx.Config.Networking.Advanced.ClusterDomain, + serviceAccount: ctx.Config.ControlPlane.Advanced.WorkloadServiceAccount.Name, + overrideHosts: ctx.Config.Sync.ToHost.Pods.RewriteHosts.Enabled, + overrideHostsImage: ctx.Config.Sync.ToHost.Pods.RewriteHosts.InitContainerImage, + serviceAccountsEnabled: ctx.Config.Sync.ToHost.ServiceAccounts.Enabled, + priorityClassesEnabled: ctx.Config.Sync.ToHost.PriorityClasses.Enabled, + enableScheduler: ctx.Config.ControlPlane.VirtualScheduler.Enabled, + syncedLabels: ctx.Config.Experimental.SyncSettings.SyncLabels, + + mountPhysicalHostPaths: ctx.Config.ControlPlane.HostPathMapper.Enabled && !ctx.Config.ControlPlane.HostPathMapper.Central, + + virtualLogsPath: virtualLogsPath, + virtualPodLogsPath: filepath.Join(virtualLogsPath, "pods"), + virtualKubeletPodPath: filepath.Join(virtualKubeletPath, "pods"), }, nil } @@ -111,6 +107,9 @@ type translator struct { defaultImageRegistry string + // this is needed for host path mapper (legacy) + mountPhysicalHostPaths bool + serviceAccountsEnabled bool serviceAccountSecretsEnabled bool clusterDomain string @@ -121,11 +120,9 @@ type translator struct { enableScheduler bool syncedLabels []string - mountPhysicalHostPaths bool - hostpathMountPropagation bool - virtualLogsPath string - virtualPodLogsPath string - virtualKubeletPodPath string + virtualLogsPath string + virtualPodLogsPath string + virtualKubeletPodPath string } func (t *translator) Translate(ctx context.Context, vPod *corev1.Pod, services []*corev1.Service, dnsIP string, kubeIP string) (*corev1.Pod, error) { @@ -428,9 +425,7 @@ func (t *translator) translateVolumes(ctx context.Context, pPod *corev1.Pod, vPo } // rewrite host paths if enabled - if t.mountPhysicalHostPaths || t.hostpathMountPropagation { - t.rewriteHostPaths(pPod) - } + t.rewriteHostPaths(pPod) return nil } diff --git a/pkg/controllers/resources/secrets/syncer.go b/pkg/controllers/resources/secrets/syncer.go index 2c8bdf574..b36ff2442 100644 --- a/pkg/controllers/resources/secrets/syncer.go +++ b/pkg/controllers/resources/secrets/syncer.go @@ -41,9 +41,9 @@ func NewSyncer(ctx *synccontext.RegisterContext, useLegacy bool) (syncer.Object, NamespacedTranslator: translator.NewNamespacedTranslator(ctx, "secret", &corev1.Secret{}), useLegacyIngress: useLegacy, - includeIngresses: ctx.Controllers.Has("ingresses"), + includeIngresses: ctx.Config.Sync.ToHost.Ingresses.Enabled, - syncAllSecrets: ctx.Options.SyncAllSecrets, + syncAllSecrets: ctx.Config.Sync.ToHost.Secrets.All, }, nil } @@ -59,7 +59,7 @@ type secretSyncer struct { var _ syncer.IndicesRegisterer = &secretSyncer{} func (s *secretSyncer) RegisterIndices(ctx *synccontext.RegisterContext) error { - if ctx.Controllers.Has("ingresses") { + if ctx.Config.Sync.ToHost.Ingresses.Enabled { if s.useLegacyIngress { err := ctx.VirtualManager.GetFieldIndexer().IndexField(ctx.Context, &networkingv1beta1.Ingress{}, constants.IndexByIngressSecret, func(rawObj client.Object) []string { return legacy.SecretNamesFromIngress(rawObj.(*networkingv1beta1.Ingress)) diff --git a/pkg/controllers/resources/secrets/syncer_test.go b/pkg/controllers/resources/secrets/syncer_test.go index 26726475d..6381a3cf0 100644 --- a/pkg/controllers/resources/secrets/syncer_test.go +++ b/pkg/controllers/resources/secrets/syncer_test.go @@ -107,7 +107,7 @@ func TestSync(t *testing.T) { }, }, Sync: func(ctx *synccontext.RegisterContext) { - ctx.Options.SyncLabels = []string{testLabel} + ctx.Config.Experimental.SyncSettings.SyncLabels = []string{testLabel} syncContext, syncer := newFakeSyncer(t, ctx) _, err := syncer.(*secretSyncer).SyncToHost(syncContext, baseSecret) assert.NilError(t, err) @@ -128,7 +128,7 @@ func TestSync(t *testing.T) { }, }, Sync: func(ctx *synccontext.RegisterContext) { - ctx.Options.SyncLabels = []string{testLabel} + ctx.Config.Experimental.SyncSettings.SyncLabels = []string{testLabel} syncContext, syncer := newFakeSyncer(t, ctx) _, err := syncer.(*secretSyncer).Sync(syncContext, syncedSecret, updatedSecret) assert.NilError(t, err) diff --git a/pkg/controllers/resources/services/syncer.go b/pkg/controllers/resources/services/syncer.go index e01481968..26d92bc60 100644 --- a/pkg/controllers/resources/services/syncer.go +++ b/pkg/controllers/resources/services/syncer.go @@ -28,7 +28,7 @@ func New(ctx *synccontext.RegisterContext) (syncertypes.Object, error) { // overriding it, which would cause endless updates back and forth. NamespacedTranslator: translator.NewNamespacedTranslator(ctx, "service", &corev1.Service{}, "field.cattle.io/publicEndpoints"), - serviceName: ctx.Options.ServiceName, + serviceName: ctx.Config.ServiceName, }, nil } diff --git a/pkg/controllers/syncer/testing/context.go b/pkg/controllers/syncer/testing/context.go index 5c626ec1f..2ffab02fa 100644 --- a/pkg/controllers/syncer/testing/context.go +++ b/pkg/controllers/syncer/testing/context.go @@ -2,10 +2,11 @@ package testing import ( "context" + "os" + "path/filepath" "testing" - "github.com/loft-sh/vcluster/pkg/constants" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/loft-sh/vcluster/pkg/util/log" @@ -25,8 +26,8 @@ import ( const ( DefaultTestTargetNamespace = "test" DefaultTestCurrentNamespace = "vcluster" - DefaultTestVclusterName = "vcluster" - DefaultTestVclusterServiceName = "vcluster" + DefaultTestVClusterName = "vcluster" + DefaultTestVClusterServiceName = "vcluster" ) func FakeStartSyncer(t *testing.T, ctx *synccontext.RegisterContext, create func(ctx *synccontext.RegisterContext) (syncer.Object, error)) (*synccontext.SyncContext, syncer.Object) { @@ -48,13 +49,8 @@ func FakeStartSyncer(t *testing.T, ctx *synccontext.RegisterContext, create func func NewFakeRegisterContext(pClient *testingutil.FakeIndexClient, vClient *testingutil.FakeIndexClient) *synccontext.RegisterContext { translate.Default = translate.NewSingleNamespaceTranslator(DefaultTestTargetNamespace) return &synccontext.RegisterContext{ - Context: context.Background(), - Options: &options.VirtualClusterOptions{ - Name: DefaultTestVclusterName, - ServiceName: DefaultTestVclusterServiceName, - TargetNamespace: DefaultTestTargetNamespace, - }, - Controllers: constants.ExistingControllers.Clone(), + Context: context.Background(), + Config: NewFakeConfig(), CurrentNamespace: DefaultTestCurrentNamespace, CurrentNamespaceClient: pClient, VirtualManager: newFakeManager(vClient), @@ -62,6 +58,39 @@ func NewFakeRegisterContext(pClient *testingutil.FakeIndexClient, vClient *testi } } +func NewFakeConfig() *config.VirtualClusterConfig { + // find vCluster config + workDir, err := os.Getwd() + if err != nil { + panic("current workDir: " + err.Error()) + } + + // find base dir + configPath := "" + for { + configPath = filepath.Join(workDir, "chart", "values.yaml") + _, err = os.Stat(configPath) + if err == nil { + break + } else if workDir == "/" { + panic("couldn't find chart/values.yaml") + } + + workDir = filepath.Dir(workDir) + } + + // parse config + vConfig, err := config.ParseConfig(configPath, DefaultTestVClusterName) + if err != nil { + panic("load test config: " + workDir + " - " + err.Error()) + } + + vConfig.Name = DefaultTestVClusterName + vConfig.ServiceName = DefaultTestVClusterServiceName + vConfig.TargetNamespace = DefaultTestTargetNamespace + return vConfig +} + type fakeEventBroadcaster struct{} func (f *fakeEventBroadcaster) StartEventWatcher(_ func(*corev1.Event)) watch.Interface { diff --git a/pkg/controllers/syncer/translator/cluster_translator.go b/pkg/controllers/syncer/translator/cluster_translator.go index 191e3049c..a6cfe006e 100644 --- a/pkg/controllers/syncer/translator/cluster_translator.go +++ b/pkg/controllers/syncer/translator/cluster_translator.go @@ -19,7 +19,7 @@ func NewClusterTranslator(ctx *context.RegisterContext, name string, obj client. virtualClient: ctx.VirtualManager.GetClient(), obj: obj, nameTranslator: nameTranslator, - syncedLabels: ctx.Options.SyncLabels, + syncedLabels: ctx.Config.Experimental.SyncSettings.SyncLabels, } } diff --git a/pkg/controllers/syncer/translator/namespaced_translator.go b/pkg/controllers/syncer/translator/namespaced_translator.go index 936ee777b..de6d817d4 100644 --- a/pkg/controllers/syncer/translator/namespaced_translator.go +++ b/pkg/controllers/syncer/translator/namespaced_translator.go @@ -21,7 +21,7 @@ func NewNamespacedTranslator(ctx *context.RegisterContext, name string, obj clie return &namespacedTranslator{ name: name, - syncedLabels: ctx.Options.SyncLabels, + syncedLabels: ctx.Config.Experimental.SyncSettings.SyncLabels, excludedAnnotations: excludedAnnotations, virtualClient: ctx.VirtualManager.GetClient(), diff --git a/pkg/k0s/k0s.go b/pkg/k0s/k0s.go index 3847c0640..8a05d83db 100644 --- a/pkg/k0s/k0s.go +++ b/pkg/k0s/k0s.go @@ -1,30 +1,67 @@ package k0s import ( + "bytes" "context" "fmt" "os" "os/exec" "path/filepath" "strings" + "text/template" - "github.com/ghodss/yaml" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/commandwriter" "k8s.io/klog/v2" ) -const ( - VClusterCommandEnv = "VCLUSTER_COMMAND" -) - -type k0sCommand struct { - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` -} - const runDir = "/run/k0s" +const cidrPlaceholder = "CIDR_PLACEHOLDER" -func StartK0S(ctx context.Context, cancel context.CancelFunc) error { +var k0sConfig = `apiVersion: k0s.k0sproject.io/v1beta1 + kind: Cluster + metadata: + name: k0s + spec: + api: + port: 6443 + k0sApiPort: 9443 + extraArgs: + bind-address: 127.0.0.1 + enable-admission-plugins: NodeRestriction + endpoint-reconciler-type: none + network: + {{- if .Values.serviceCIDR }} + serviceCIDR: {{ .Values.serviceCIDR }} + {{- else }} + # Will be replaced automatically by the syncer container on first startup + serviceCIDR: CIDR_PLACEHOLDER + {{- end }} + provider: custom + {{- if .Values.networking.advanced.clusterDomain }} + clusterDomain: {{ .Values.networking.advanced.clusterDomain }} + {{- end}} + controllerManager: + extraArgs: + {{- if not .Values.controlPlane.virtualScheduler.enabled }} + controllers: '*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' + {{- else }} + controllers: '*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl' + node-monitor-grace-period: 1h + node-monitor-period: 1h + {{- end }} + {{- if .Values.controlPlane.backingStore.embeddedEtcd.enabled }} + storage: + etcd: + externalCluster: + endpoints: ["127.0.0.1:2379"] + caFile: /data/k0s/pki/etcd/ca.crt + etcdPrefix: "/registry" + clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt + clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key + {{- end }}` + +func StartK0S(ctx context.Context, cancel context.CancelFunc, vConfig *config.VirtualClusterConfig) error { // this is not really useful but go isn't happy if we don't cancel the context // everywhere defer cancel() @@ -35,14 +72,25 @@ func StartK0S(ctx context.Context, cancel context.CancelFunc) error { _ = os.RemoveAll(filepath.Join(runDir, entry.Name())) } - // create command - command := &k0sCommand{} - err := yaml.Unmarshal([]byte(os.Getenv(VClusterCommandEnv)), command) - if err != nil { - return fmt.Errorf("parsing k0s command %s: %w", os.Getenv(VClusterCommandEnv), err) + // build args + args := []string{} + if len(vConfig.ControlPlane.Distro.K0S.Command) > 0 { + args = append(args, vConfig.ControlPlane.Distro.K0S.Command...) + } else { + args = append(args, "/binaries/k0s") + args = append(args, "controller") + args = append(args, "--config=/tmp/k0s-config.yaml") + args = append(args, "--data-dir=/data/k0s") + args = append(args, "--status-socket=/run/k0s/status.sock") + if vConfig.ControlPlane.VirtualScheduler.Enabled { + args = append(args, "--disable-components=konnectivity-server,kube-scheduler,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,worker-config") + } else { + args = append(args, "--disable-components=konnectivity-server,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,worker-config") + } } - args := append(command.Command, command.Args...) + // add extra args + args = append(args, vConfig.ControlPlane.Distro.K0S.ExtraArgs...) // check what writer we should use writer, err := commandwriter.NewCommandWriter("k0s") @@ -56,6 +104,7 @@ func StartK0S(ctx context.Context, cancel context.CancelFunc) error { cmd := exec.CommandContext(ctx, args[0], args[1:]...) cmd.Stdout = writer.Writer() cmd.Stderr = writer.Writer() + cmd.Env = append(os.Environ(), "ETCD_UNSUPPORTED_ARCH=arm64") err = cmd.Run() // make sure we wait for scanner to be done @@ -67,3 +116,50 @@ func StartK0S(ctx context.Context, cancel context.CancelFunc) error { } return nil } + +func WriteK0sConfig( + serviceCIDR string, + vConfig *config.VirtualClusterConfig, +) error { + + // choose config + configTemplate := k0sConfig + if vConfig.Config.ControlPlane.Distro.K0S.Config != "" { + configTemplate = vConfig.Config.ControlPlane.Distro.K0S.Config + } + + // exec template + outBytes, err := execTemplate(configTemplate, map[string]interface{}{ + "Values": vConfig.Config, + }) + if err != nil { + return fmt.Errorf("exec k0s config template: %w", err) + } + + // apply changes + updatedConfig := []byte(strings.ReplaceAll(string(outBytes), cidrPlaceholder, serviceCIDR)) + + // write the config to file + err = os.WriteFile("/tmp/k0s-config.yaml", updatedConfig, 0640) + if err != nil { + klog.Errorf("error while write k0s config to file: %s", err.Error()) + return err + } + + return nil +} + +func execTemplate(templateContents string, values map[string]interface{}) ([]byte, error) { + t, err := template.New("").Parse(templateContents) + if err != nil { + return nil, err + } + + b := &bytes.Buffer{} + err = t.Execute(b, values) + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} diff --git a/pkg/k3s/k3s.go b/pkg/k3s/k3s.go index ac4ba3549..e4fd8700d 100644 --- a/pkg/k3s/k3s.go +++ b/pkg/k3s/k3s.go @@ -7,7 +7,7 @@ import ( "os/exec" "strings" - "github.com/ghodss/yaml" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/util/commandwriter" "github.com/loft-sh/vcluster/pkg/util/random" corev1 "k8s.io/api/core/v1" @@ -19,27 +19,45 @@ import ( var tokenPath = "/data/server/token" -const VClusterCommandEnv = "VCLUSTER_COMMAND" - -type k3sCommand struct { - Command []string `json:"command,omitempty"` - Args []string `json:"args,omitempty"` -} - -func StartK3S(ctx context.Context, serviceCIDR, k3sToken string) error { - command := &k3sCommand{} - err := yaml.Unmarshal([]byte(os.Getenv(VClusterCommandEnv)), command) - if err != nil { - return fmt.Errorf("parsing k3s command %s: %w", os.Getenv(VClusterCommandEnv), err) +func StartK3S(ctx context.Context, vConfig *config.VirtualClusterConfig, serviceCIDR, k3sToken string) error { + // build args + args := []string{} + if len(vConfig.ControlPlane.Distro.K3S.Command) > 0 { + args = append(args, vConfig.ControlPlane.Distro.K3S.Command...) + } else { + args = append(args, "/binaries/k3s") + args = append(args, "server") + args = append(args, "--write-kubeconfig=/data/k3s-config/kube-config.yaml") + args = append(args, "--data-dir=/data") + args = append(args, "--service-cidr="+serviceCIDR) + args = append(args, "--token="+strings.TrimSpace(k3sToken)) + args = append(args, "--disable=traefik,servicelb,metrics-server,local-storage,coredns") + args = append(args, "--disable-network-policy") + args = append(args, "--disable-agent") + args = append(args, "--disable-cloud-controller") + args = append(args, "--egress-selector-mode=disabled") + args = append(args, "--flannel-backend=none") + args = append(args, "--kube-apiserver-arg=bind-address=127.0.0.1") + if vConfig.ControlPlane.VirtualScheduler.Enabled { + args = append(args, "--kube-controller-manager-arg=controllers=*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl") + args = append(args, "--kube-apiserver-arg=endpoint-reconciler-type=none") + args = append(args, "--kube-controller-manager-arg=node-monitor-grace-period=1h") + args = append(args, "--kube-controller-manager-arg=node-monitor-period=1h") + } else { + args = append(args, "--disable-scheduler") + args = append(args, "--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl") + args = append(args, "--kube-apiserver-arg=endpoint-reconciler-type=none") + } + if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { + args = append(args, "--datastore-endpoint=https://localhost:2379") + args = append(args, "--datastore-cafile=/data/pki/etcd/ca.crt") + args = append(args, "--datastore-certfile=/data/pki/apiserver-etcd-client.crt") + args = append(args, "--datastore-keyfile=/data/pki/apiserver-etcd-client.key") + } } - // add service cidr and k3s token - command.Args = append( - command.Args, - "--service-cidr", serviceCIDR, - "--token", strings.TrimSpace(k3sToken), - ) - args := append(command.Command, command.Args...) + // add extra args + args = append(args, vConfig.ControlPlane.Distro.K3S.ExtraArgs...) // check what writer we should use writer, err := commandwriter.NewCommandWriter("k3s") @@ -65,7 +83,12 @@ func StartK3S(ctx context.Context, serviceCIDR, k3sToken string) error { return nil } -func EnsureK3SToken(ctx context.Context, currentNamespaceClient kubernetes.Interface, currentNamespace, vClusterName string) (string, error) { +func EnsureK3SToken(ctx context.Context, currentNamespaceClient kubernetes.Interface, currentNamespace, vClusterName string, options *config.VirtualClusterConfig) (string, error) { + // check if token is set externally + if options.ControlPlane.Distro.K3S.Token != "" { + return options.ControlPlane.Distro.K3S.Token, nil + } + // check if secret exists secretName := fmt.Sprintf("vc-k3s-%s", vClusterName) secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, secretName, metav1.GetOptions{}) @@ -92,7 +115,6 @@ func EnsureK3SToken(ctx context.Context, currentNamespaceClient kubernetes.Inter }, Type: corev1.SecretTypeOpaque, }, metav1.CreateOptions{}) - if kerrors.IsAlreadyExists(err) { // retrieve k3s secret again secret, err = currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, secretName, metav1.GetOptions{}) diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index a69a360d9..fac642ddd 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -6,43 +6,75 @@ import ( "errors" "fmt" "net/http" - "os" "os/exec" "strings" "time" - "github.com/ghodss/yaml" + config2 "github.com/loft-sh/vcluster/config" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/etcd" "github.com/loft-sh/vcluster/pkg/util/commandwriter" "golang.org/x/sync/errgroup" "k8s.io/klog/v2" ) -const apiServerCmd = "APISERVER_COMMAND" -const schedulerCmd = "SCHEDULER_COMMAND" -const controllerCmd = "CONTROLLER_COMMAND" - -type command struct { - Command []string `json:"command,omitempty"` -} - -func StartK8S(ctx context.Context, serviceCIDR string) error { +func StartK8S( + ctx context.Context, + serviceCIDR string, + apiServer config2.DistroContainerDisabled, + controllerManager config2.DistroContainerDisabled, + scheduler config2.DistroContainer, + vConfig *config.VirtualClusterConfig, +) error { serviceCIDRArg := fmt.Sprintf("--service-cluster-ip-range=%s", serviceCIDR) eg := &errgroup.Group{} // start api server first - apiEnv, ok := os.LookupEnv(apiServerCmd) - if ok { - apiCommand := &command{} - err := yaml.Unmarshal([]byte(apiEnv), apiCommand) - if err != nil { - return fmt.Errorf("parsing apiserver command %s: %w", apiEnv, err) - } - - apiCommand.Command = append(apiCommand.Command, serviceCIDRArg) + if !apiServer.Disabled { eg.Go(func() error { + // build flags + args := []string{} + if len(apiServer.Command) > 0 { + args = append(args, apiServer.Command...) + } else { + args = append(args, "/binaries/kube-apiserver") + args = append(args, "--advertise-address=127.0.0.1") + args = append(args, serviceCIDRArg) + args = append(args, "--bind-address=127.0.0.1") + args = append(args, "--allow-privileged=true") + args = append(args, "--authorization-mode=RBAC") + args = append(args, "--client-ca-file="+vConfig.VirtualClusterKubeConfig().ClientCACert) + args = append(args, "--enable-bootstrap-token-auth=true") + args = append(args, "--etcd-cafile=/pki/etcd/ca.crt") + args = append(args, "--etcd-certfile=/pki/apiserver-etcd-client.crt") + args = append(args, "--etcd-keyfile=/pki/apiserver-etcd-client.key") + if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { + args = append(args, "--etcd-servers=https://127.0.0.1:2379") + } else { + args = append(args, "--etcd-servers=https://"+vConfig.Name+"-etcd:2379") + } + args = append(args, "--proxy-client-cert-file=/pki/front-proxy-client.crt") + args = append(args, "--proxy-client-key-file=/pki/front-proxy-client.key") + args = append(args, "--requestheader-allowed-names=front-proxy-client") + args = append(args, "--requestheader-client-ca-file=/pki/front-proxy-ca.crt") + args = append(args, "--requestheader-extra-headers-prefix=X-Remote-Extra-") + args = append(args, "--requestheader-group-headers=X-Remote-Group") + args = append(args, "--requestheader-username-headers=X-Remote-User") + args = append(args, "--secure-port=6443") + args = append(args, "--service-account-issuer=https://kubernetes.default.svc.cluster.local") + args = append(args, "--service-account-key-file=/pki/sa.pub") + args = append(args, "--service-account-signing-key-file=/pki/sa.key") + args = append(args, "--tls-cert-file=/pki/apiserver.crt") + args = append(args, "--tls-private-key-file=/pki/apiserver.key") + args = append(args, "--watch-cache=false") + args = append(args, "--endpoint-reconciler-type=none") + } + + // add extra args + args = append(args, apiServer.ExtraArgs...) + // get etcd endpoints and certificates from flags - endpoints, certificates, err := etcd.EndpointsAndCertificatesFromFlags(apiCommand.Command) + endpoints, certificates, err := etcd.EndpointsAndCertificatesFromFlags(args) if err != nil { return fmt.Errorf("get etcd certificates and endpoint: %w", err) } @@ -54,7 +86,7 @@ func StartK8S(ctx context.Context, serviceCIDR string) error { } // now start the api server - return RunCommand(ctx, apiCommand, "apiserver") + return RunCommand(ctx, args, "apiserver") }) } @@ -65,31 +97,74 @@ func StartK8S(ctx context.Context, serviceCIDR string) error { } // start controller command - controllerEnv, ok := os.LookupEnv(controllerCmd) - if ok { - controllerCommand := &command{} - err := yaml.Unmarshal([]byte(controllerEnv), controllerCommand) - if err != nil { - return fmt.Errorf("parsing controller command %s: %w", controllerEnv, err) - } - - controllerCommand.Command = append(controllerCommand.Command, serviceCIDRArg) + if !controllerManager.Disabled { eg.Go(func() error { - return RunCommand(ctx, controllerCommand, "controller") + // build flags + args := []string{} + if len(controllerManager.Command) > 0 { + args = append(args, controllerManager.Command...) + } else { + args = append(args, "/binaries/kube-controller-manager") + args = append(args, serviceCIDRArg) + args = append(args, "--authentication-kubeconfig=/pki/controller-manager.conf") + args = append(args, "--authorization-kubeconfig=/pki/controller-manager.conf") + args = append(args, "--bind-address=127.0.0.1") + args = append(args, "--client-ca-file=/pki/ca.crt") + args = append(args, "--cluster-name=kubernetes") + args = append(args, "--cluster-signing-cert-file=/pki/ca.crt") + args = append(args, "--cluster-signing-key-file=/pki/ca.key") + args = append(args, "--horizontal-pod-autoscaler-sync-period=60s") + args = append(args, "--kubeconfig=/pki/controller-manager.conf") + args = append(args, "--node-monitor-grace-period=180s") + args = append(args, "--node-monitor-period=30s") + args = append(args, "--pvclaimbinder-sync-period=60s") + args = append(args, "--requestheader-client-ca-file=/pki/front-proxy-ca.crt") + args = append(args, "--root-ca-file=/pki/ca.crt") + args = append(args, "--service-account-private-key-file=/pki/sa.key") + args = append(args, "--use-service-account-credentials=true") + if vConfig.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 { + args = append(args, "--leader-elect=true") + } else { + args = append(args, "--leader-elect=false") + } + if vConfig.ControlPlane.VirtualScheduler.Enabled { + args = append(args, "--controllers=*,-nodeipam,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl") + args = append(args, "--node-monitor-grace-period=1h") + args = append(args, "--node-monitor-period=1h") + } else { + args = append(args, "--controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl") + } + } + + // add extra args + args = append(args, controllerManager.ExtraArgs...) + return RunCommand(ctx, args, "controller-manager") }) } // start scheduler command - schedulerEnv, ok := os.LookupEnv(schedulerCmd) - if ok { - schedulerCommand := &command{} - err := yaml.Unmarshal([]byte(schedulerEnv), schedulerCommand) - if err != nil { - return fmt.Errorf("parsing scheduler command %s: %w", schedulerEnv, err) - } - + if vConfig.ControlPlane.VirtualScheduler.Enabled { eg.Go(func() error { - return RunCommand(ctx, schedulerCommand, "scheduler") + // build flags + args := []string{} + if len(scheduler.Command) > 0 { + args = append(args, scheduler.Command...) + } else { + args = append(args, "/binaries/kube-scheduler") + args = append(args, "--authentication-kubeconfig=/pki/scheduler.conf") + args = append(args, "--authorization-kubeconfig=/pki/scheduler.conf") + args = append(args, "--bind-address=127.0.0.1") + args = append(args, "--kubeconfig=/pki/scheduler.conf") + if vConfig.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 { + args = append(args, "--leader-elect=true") + } else { + args = append(args, "--leader-elect=false") + } + } + + // add extra args + args = append(args, scheduler.ExtraArgs...) + return RunCommand(ctx, args, "scheduler") }) } @@ -103,7 +178,7 @@ func StartK8S(ctx context.Context, serviceCIDR string) error { return err } -func RunCommand(ctx context.Context, command *command, component string) error { +func RunCommand(ctx context.Context, command []string, component string) error { writer, err := commandwriter.NewCommandWriter(component) if err != nil { return err @@ -111,8 +186,8 @@ func RunCommand(ctx context.Context, command *command, component string) error { defer writer.Writer() // start the command - klog.InfoS("Starting "+component, "args", strings.Join(command.Command, " ")) - cmd := exec.CommandContext(ctx, command.Command[0], command.Command[1:]...) + klog.InfoS("Starting "+component, "args", strings.Join(command, " ")) + cmd := exec.CommandContext(ctx, command[0], command[1:]...) cmd.Stdout = writer.Writer() cmd.Stderr = writer.Writer() err = cmd.Run() diff --git a/pkg/metricsapiservice/register.go b/pkg/metricsapiservice/register.go index 25e3ba9ab..e5827c5a6 100644 --- a/pkg/metricsapiservice/register.go +++ b/pkg/metricsapiservice/register.go @@ -5,7 +5,7 @@ import ( "math" "time" - "github.com/loft-sh/vcluster/pkg/options" + "github.com/loft-sh/vcluster/pkg/config" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -53,7 +53,7 @@ func applyOperation(ctx context.Context, operationFunc wait.ConditionWithContext }, operationFunc) } -func deleteOperation(ctrlCtx *options.ControllerContext) wait.ConditionWithContextFunc { +func deleteOperation(ctrlCtx *config.ControllerContext) wait.ConditionWithContextFunc { return func(ctx context.Context) (bool, error) { err := ctrlCtx.VirtualManager.GetClient().Delete(ctx, &apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{ @@ -73,7 +73,7 @@ func deleteOperation(ctrlCtx *options.ControllerContext) wait.ConditionWithConte } } -func createOperation(ctrlCtx *options.ControllerContext) wait.ConditionWithContextFunc { +func createOperation(ctrlCtx *config.ControllerContext) wait.ConditionWithContextFunc { return func(ctx context.Context) (bool, error) { service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -134,12 +134,12 @@ func createOperation(ctrlCtx *options.ControllerContext) wait.ConditionWithConte } } -func RegisterOrDeregisterAPIService(ctx *options.ControllerContext) error { +func RegisterOrDeregisterAPIService(ctx *config.ControllerContext) error { // check if the api service should get created exists := checkExistingAPIService(ctx.Context, ctx.VirtualManager.GetClient()) - if ctx.Options.ProxyMetricsServer { + if ctx.Config.Observability.Metrics.Proxy.Nodes.Enabled || ctx.Config.Observability.Metrics.Proxy.Pods.Enabled { return applyOperation(ctx.Context, createOperation(ctx)) - } else if !ctx.Options.ProxyMetricsServer && exists { + } else if exists { return applyOperation(ctx.Context, deleteOperation(ctx)) } diff --git a/pkg/patches/conditions.go b/pkg/patches/conditions.go index 0272fced6..1f322af81 100644 --- a/pkg/patches/conditions.go +++ b/pkg/patches/conditions.go @@ -1,13 +1,13 @@ package patches import ( - "github.com/loft-sh/vcluster/pkg/genericsyncconfig" + "github.com/loft-sh/vcluster/config" "github.com/pkg/errors" "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" yaml "gopkg.in/yaml.v3" ) -func ValidateAllConditions(obj *yaml.Node, match *yaml.Node, conditions []*genericsyncconfig.PatchCondition) (bool, error) { +func ValidateAllConditions(obj *yaml.Node, match *yaml.Node, conditions []*config.PatchCondition) (bool, error) { for _, condition := range conditions { matched, err := ValidateCondition(obj, match, condition) if err != nil { @@ -20,7 +20,7 @@ func ValidateAllConditions(obj *yaml.Node, match *yaml.Node, conditions []*gener return true, nil } -func ValidateCondition(obj *yaml.Node, match *yaml.Node, condition *genericsyncconfig.PatchCondition) (bool, error) { +func ValidateCondition(obj *yaml.Node, match *yaml.Node, condition *config.PatchCondition) (bool, error) { if condition == nil { return true, nil } diff --git a/pkg/patches/patch.go b/pkg/patches/patch.go index 8b20dbae2..865e5512f 100644 --- a/pkg/patches/patch.go +++ b/pkg/patches/patch.go @@ -5,10 +5,10 @@ import ( "fmt" "regexp" + config2 "github.com/loft-sh/vcluster/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" jsonyaml "github.com/ghodss/yaml" - "github.com/loft-sh/vcluster/pkg/genericsyncconfig" "github.com/pkg/errors" yaml "gopkg.in/yaml.v3" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,7 +23,7 @@ type NameResolver interface { TranslateNamespaceRef(namespace string) (string, error) } -func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*genericsyncconfig.Patch, reversePatchesConf []*genericsyncconfig.Patch, nameResolver NameResolver) error { +func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*config2.Patch, reversePatchesConf []*config2.Patch, nameResolver NameResolver) error { node1, err := NewJSONNode(destObj) if err != nil { return errors.Wrap(err, "new json yaml node") @@ -50,8 +50,8 @@ func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*genericsyncco continue } - err := applyPatch(node1, node2, &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRemove, + err := applyPatch(node1, node2, &config2.Patch{ + Operation: config2.PatchTypeRemove, Path: p.Path, }, nameResolver) if err != nil { @@ -72,23 +72,23 @@ func ApplyPatches(destObj, sourceObj client.Object, patchesConf []*genericsyncco return nil } -func applyPatch(obj1, obj2 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func applyPatch(obj1, obj2 *yaml.Node, patch *config2.Patch, resolver NameResolver) error { switch patch.Operation { - case genericsyncconfig.PatchTypeRewriteName: + case config2.PatchTypeRewriteName: return RewriteName(obj1, patch, resolver) - case genericsyncconfig.PatchTypeRewriteLabelKey: + case config2.PatchTypeRewriteLabelKey: return RewriteLabelKey(obj1, patch, resolver) - case genericsyncconfig.PatchTypeRewriteLabelExpressionsSelector: + case config2.PatchTypeRewriteLabelExpressionsSelector: return RewriteLabelExpressionsSelector(obj1, patch, resolver) - case genericsyncconfig.PatchTypeRewriteLabelSelector: + case config2.PatchTypeRewriteLabelSelector: return RewriteLabelSelector(obj1, patch, resolver) - case genericsyncconfig.PatchTypeReplace: + case config2.PatchTypeReplace: return Replace(obj1, patch) - case genericsyncconfig.PatchTypeRemove: + case config2.PatchTypeRemove: return Remove(obj1, patch) - case genericsyncconfig.PatchTypeAdd: + case config2.PatchTypeAdd: return Add(obj1, patch) - case genericsyncconfig.PatchTypeCopyFromObject: + case config2.PatchTypeCopyFromObject: return CopyFromObject(obj1, obj2, patch) } diff --git a/pkg/patches/patch_types.go b/pkg/patches/patch_types.go index fca1f8b57..aaec4d451 100644 --- a/pkg/patches/patch_types.go +++ b/pkg/patches/patch_types.go @@ -4,15 +4,15 @@ import ( "fmt" "strconv" + "github.com/loft-sh/vcluster/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/loft-sh/vcluster/pkg/genericsyncconfig" "github.com/pkg/errors" yaml "gopkg.in/yaml.v3" k8syaml "sigs.k8s.io/yaml" ) -func CopyFromObject(obj1, obj2 *yaml.Node, patch *genericsyncconfig.Patch) error { +func CopyFromObject(obj1, obj2 *yaml.Node, patch *config.Patch) error { if obj2 == nil { return nil } @@ -68,7 +68,7 @@ func CopyFromObject(obj1, obj2 *yaml.Node, patch *genericsyncconfig.Patch) error return nil } -func Remove(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { +func Remove(obj1 *yaml.Node, patch *config.Patch) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -95,7 +95,7 @@ func Remove(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { return nil } -func Add(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { +func Add(obj1 *yaml.Node, patch *config.Patch) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -134,7 +134,7 @@ func Add(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { return nil } -func Replace(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { +func Replace(obj1 *yaml.Node, patch *config.Patch) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -159,7 +159,7 @@ func Replace(obj1 *yaml.Node, patch *genericsyncconfig.Patch) error { return nil } -func RewriteName(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func RewriteName(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -190,7 +190,7 @@ func RewriteName(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameR return nil } -func ProcessRewrite(obj *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func ProcessRewrite(obj *yaml.Node, patch *config.Patch, resolver NameResolver) error { var namespace string var err error @@ -240,7 +240,7 @@ func ProcessRewrite(obj *yaml.Node, patch *genericsyncconfig.Patch, resolver Nam return nil } -func ValidateAndTranslateName(obj *yaml.Node, match *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver, namespace string) error { +func ValidateAndTranslateName(obj *yaml.Node, match *yaml.Node, patch *config.Patch, resolver NameResolver, namespace string) error { validated, err := ValidateAllConditions(obj, match, patch.Conditions) if err != nil { return errors.Wrap(err, "validate conditions") @@ -270,7 +270,7 @@ func ValidateAndTranslateName(obj *yaml.Node, match *yaml.Node, patch *genericsy return nil } -func ValidateAndTranslateNamespace(obj *yaml.Node, match *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func ValidateAndTranslateNamespace(obj *yaml.Node, match *yaml.Node, patch *config.Patch, resolver NameResolver) error { validated, err := ValidateAllConditions(obj, match, patch.Conditions) if err != nil { return errors.Wrap(err, "validate conditions") @@ -293,7 +293,7 @@ func ValidateAndTranslateNamespace(obj *yaml.Node, match *yaml.Node, patch *gene return nil } -func GetNamespace(obj *yaml.Node, patch *genericsyncconfig.Patch) (string, error) { +func GetNamespace(obj *yaml.Node, patch *config.Patch) (string, error) { var namespace string matches, err := FindMatches(obj, patch.NamespacePath) @@ -312,7 +312,7 @@ func GetNamespace(obj *yaml.Node, patch *genericsyncconfig.Patch) (string, error return namespace, nil } -func RewriteLabelKey(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func RewriteLabelKey(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -350,7 +350,7 @@ func RewriteLabelKey(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver N return nil } -func RewriteLabelSelector(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func RewriteLabelSelector(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") @@ -396,7 +396,7 @@ func RewriteLabelSelector(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resol return nil } -func RewriteLabelExpressionsSelector(obj1 *yaml.Node, patch *genericsyncconfig.Patch, resolver NameResolver) error { +func RewriteLabelExpressionsSelector(obj1 *yaml.Node, patch *config.Patch, resolver NameResolver) error { matches, err := FindMatches(obj1, patch.Path) if err != nil { return errors.Wrap(err, "find matches") diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index ce23bd7be..287685188 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -36,7 +36,12 @@ func (m *manager) Start( syncerConfig *clientcmdapi.Config, vConfig *config.VirtualClusterConfig, ) error { - err := m.legacyManager.Start(ctx, currentNamespace, targetNamespace, virtualKubeConfig, physicalKubeConfig, syncerConfig, vConfig.LegacyOptions) + legacyOptions, err := vConfig.LegacyOptions() + if err != nil { + return fmt.Errorf("build legacy options: %w", err) + } + + err = m.legacyManager.Start(ctx, currentNamespace, targetNamespace, virtualKubeConfig, physicalKubeConfig, syncerConfig, legacyOptions) if err != nil { return fmt.Errorf("start legacy plugins: %w", err) } diff --git a/pkg/plugin/v2/plugin.go b/pkg/plugin/v2/plugin.go index 160a0cb90..832ee868a 100644 --- a/pkg/plugin/v2/plugin.go +++ b/pkg/plugin/v2/plugin.go @@ -248,8 +248,14 @@ func (m *Manager) buildInitRequest( return nil, fmt.Errorf("encode config: %w", err) } + // convert to legacy options + legacyOptions, err := vConfig.LegacyOptions() + if err != nil { + return nil, fmt.Errorf("get legacy options: %w", err) + } + // We need this for downward compatibility - encodedLegacyOptions, err := json.Marshal(vConfig.LegacyOptions) + encodedLegacyOptions, err := json.Marshal(legacyOptions) if err != nil { return nil, fmt.Errorf("marshal options: %w", err) } diff --git a/pkg/server/cert/syncer.go b/pkg/server/cert/syncer.go index 8c6d97f9a..b2dd2387a 100644 --- a/pkg/server/cert/syncer.go +++ b/pkg/server/cert/syncer.go @@ -9,10 +9,9 @@ import ( "sync" "time" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes/nodeservice" - "github.com/loft-sh/vcluster/pkg/options" - "github.com/loft-sh/vcluster/pkg/util/translate" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,16 +33,16 @@ type Syncer interface { dynamiccertificates.CertKeyContentProvider } -func NewSyncer(_ context.Context, currentNamespace string, currentNamespaceClient client.Client, options *options.VirtualClusterOptions) (Syncer, error) { +func NewSyncer(_ context.Context, currentNamespace string, currentNamespaceClient client.Client, options *config.VirtualClusterConfig) (Syncer, error) { return &syncer{ - clusterDomain: options.ClusterDomain, + clusterDomain: options.Networking.Advanced.ClusterDomain, - serverCaKey: options.ServerCaKey, - serverCaCert: options.ServerCaCert, + serverCaKey: options.VirtualClusterKubeConfig().ServerCAKey, + serverCaCert: options.VirtualClusterKubeConfig().ServerCACert, - fakeKubeletIPs: options.FakeKubeletIPs, + fakeKubeletIPs: options.Networking.Advanced.ProxyKubelets.ByIP, - addSANs: options.TLSSANs, + addSANs: options.ControlPlane.Proxy.ExtraSANs, listeners: []dynamiccertificates.Listener{}, serviceName: options.ServiceName, diff --git a/pkg/server/server.go b/pkg/server/server.go index a12031701..0733d6d3e 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -14,10 +14,10 @@ import ( "github.com/loft-sh/vcluster/pkg/authorization/delegatingauthorizer" "github.com/loft-sh/vcluster/pkg/authorization/impersonationauthorizer" "github.com/loft-sh/vcluster/pkg/authorization/kubeletauthorizer" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes/nodeservice" - "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/server/cert" "github.com/loft-sh/vcluster/pkg/server/filters" "github.com/loft-sh/vcluster/pkg/server/handler" @@ -74,7 +74,7 @@ type Server struct { // NewServer creates and installs a new Server. // 'filter', if non-nil, protects requests to the api only. -func NewServer(ctx *options.ControllerContext, requestHeaderCaFile, clientCaFile string) (*Server, error) { +func NewServer(ctx *config.ControllerContext, requestHeaderCaFile, clientCaFile string) (*Server, error) { localConfig := ctx.LocalManager.GetConfig() virtualConfig := ctx.VirtualManager.GetConfig() uncachedLocalClient, err := client.New(localConfig, client.Options{ @@ -93,7 +93,7 @@ func NewServer(ctx *options.ControllerContext, requestHeaderCaFile, clientCaFile } cachedLocalClient, err := createCachedClient(ctx.Context, localConfig, ctx.CurrentNamespace, uncachedLocalClient.RESTMapper(), uncachedLocalClient.Scheme(), func(cache cache.Cache) error { - if ctx.Options.FakeKubeletIPs { + if ctx.Config.Networking.Advanced.ProxyKubelets.ByIP { err := cache.IndexField(ctx.Context, &corev1.Service{}, constants.IndexByClusterIP, func(object client.Object) []string { svc := object.(*corev1.Service) if len(svc.Labels) == 0 || svc.Labels[nodeservice.ServiceClusterLabel] != translate.VClusterName { @@ -145,7 +145,7 @@ func NewServer(ctx *options.ControllerContext, requestHeaderCaFile, clientCaFile uncachedLocalClient = pluginhookclient.WrapPhysicalClient(uncachedLocalClient) cachedLocalClient = pluginhookclient.WrapPhysicalClient(cachedLocalClient) - certSyncer, err := cert.NewSyncer(ctx.Context, ctx.CurrentNamespace, cachedLocalClient, ctx.Options) + certSyncer, err := cert.NewSyncer(ctx.Context, ctx.CurrentNamespace, cachedLocalClient, ctx.Config) if err != nil { return nil, errors.Wrap(err, "create cert syncer") } @@ -156,7 +156,7 @@ func NewServer(ctx *options.ControllerContext, requestHeaderCaFile, clientCaFile certSyncer: certSyncer, handler: http.NewServeMux(), - fakeKubeletIPs: ctx.Options.FakeKubeletIPs, + fakeKubeletIPs: ctx.Config.Networking.Advanced.ProxyKubelets.ByIP, currentNamespace: ctx.CurrentNamespace, currentNamespaceClient: cachedLocalClient, @@ -199,24 +199,24 @@ func NewServer(ctx *options.ControllerContext, requestHeaderCaFile, clientCaFile } h := handler.ImpersonatingHandler("", virtualConfig) - h = filters.WithServiceCreateRedirect(h, uncachedLocalClient, uncachedVirtualClient, virtualConfig, ctx.Options.SyncLabels) + h = filters.WithServiceCreateRedirect(h, uncachedLocalClient, uncachedVirtualClient, virtualConfig, ctx.Config.Experimental.SyncSettings.SyncLabels) h = filters.WithRedirect(h, localConfig, uncachedLocalClient.Scheme(), uncachedVirtualClient, admissionHandler, s.redirectResources) h = filters.WithMetricsProxy(h, localConfig, cachedVirtualClient) // is metrics proxy enabled? - if ctx.Options.ProxyMetricsServer { + if ctx.Config.Observability.Metrics.Proxy.Nodes.Enabled || ctx.Config.Observability.Metrics.Proxy.Pods.Enabled { h = filters.WithMetricsServerProxy( h, - ctx.Options.TargetNamespace, + ctx.Config.TargetNamespace, cachedLocalClient, cachedVirtualClient, localConfig, virtualConfig, - ctx.Options.MultiNamespaceMode, + ctx.Config.Experimental.MultiNamespaceMode.Enabled, ) } - if ctx.Options.DeprecatedSyncNodeChanges { + if ctx.Config.Sync.FromHost.Nodes.Real.Enabled && ctx.Config.Sync.FromHost.Nodes.Real.SyncLabelsTaints { h = filters.WithNodeChanges(ctx.Context, h, uncachedLocalClient, uncachedVirtualClient, virtualConfig) } h = filters.WithFakeKubelet(h, localConfig, cachedVirtualClient) diff --git a/pkg/setup/controller_context.go b/pkg/setup/controller_context.go index 62a441363..c153f9317 100644 --- a/pkg/setup/controller_context.go +++ b/pkg/setup/controller_context.go @@ -8,12 +8,10 @@ import ( "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" - "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/plugin" "github.com/loft-sh/vcluster/pkg/pro" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" - "github.com/loft-sh/vcluster/pkg/util/toleration" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,26 +29,14 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) -var allowedPodSecurityStandards = map[string]bool{ - "privileged": true, - "baseline": true, - "restricted": true, -} - // NewControllerContext builds the controller context we can use to start the syncer func NewControllerContext( ctx context.Context, - options *options.VirtualClusterOptions, + options *config.VirtualClusterConfig, currentNamespace string, inClusterConfig *rest.Config, scheme *runtime.Scheme, ) (*config.ControllerContext, error) { - // validate options - err := ValidateOptions(options) - if err != nil { - return nil, err - } - // create controller context controllerContext, err := InitManagers( ctx, @@ -76,13 +62,13 @@ func NewControllerContext( func InitManagers( ctx context.Context, - options *options.VirtualClusterOptions, + options *config.VirtualClusterConfig, currentNamespace string, inClusterConfig *rest.Config, scheme *runtime.Scheme, newPhysicalClient client.NewClientFunc, newVirtualClient client.NewClientFunc, -) (*options.ControllerContext, error) { +) (*config.ControllerContext, error) { // load virtual config virtualConfig, virtualRawConfig, err := LoadVirtualConfig(ctx, options) if err != nil { @@ -91,7 +77,7 @@ func InitManagers( // is multi namespace mode? var defaultNamespaces map[string]cache.Config - if options.MultiNamespaceMode { + if options.Experimental.MultiNamespaceMode.Enabled { // set options.TargetNamespace to empty because it will later be used in Manager options.TargetNamespace = "" translate.Default = translate.NewMultiNamespaceTranslator(currentNamespace) @@ -114,7 +100,7 @@ func InitManagers( klog.Info("Using physical cluster at " + inClusterConfig.Host) localManager, err := ctrl.NewManager(inClusterConfig, ctrl.Options{ Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: options.HostMetricsBindAddress}, + Metrics: metricsserver.Options{BindAddress: "0"}, LeaderElection: false, Cache: cache.Options{DefaultNamespaces: defaultNamespaces}, NewClient: newPhysicalClient, @@ -126,7 +112,7 @@ func InitManagers( // create virtual manager virtualClusterManager, err := ctrl.NewManager(virtualConfig, ctrl.Options{ Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: options.VirtualMetricsBindAddress}, + Metrics: metricsserver.Options{BindAddress: "0"}, LeaderElection: false, NewClient: newVirtualClient, }) @@ -144,26 +130,23 @@ func StartPlugins( inClusterConfig, virtualConfig *rest.Config, virtualRawConfig *clientcmdapi.Config, - options *options.VirtualClusterOptions, + options *config.VirtualClusterConfig, ) error { - // start plugins only if they are not disabled - if !options.DisablePlugins { - klog.Infof("Start Plugins Manager...") - syncerConfig, err := CreateVClusterKubeConfig(virtualRawConfig, options) - if err != nil { - return err - } + klog.Infof("Start Plugins Manager...") + syncerConfig, err := CreateVClusterKubeConfig(virtualRawConfig, options) + if err != nil { + return err + } - err = plugin.DefaultManager.Start(ctx, currentNamespace, options.TargetNamespace, virtualConfig, inClusterConfig, syncerConfig, options) - if err != nil { - return err - } + err = plugin.DefaultManager.Start(ctx, currentNamespace, options.TargetNamespace, virtualConfig, inClusterConfig, syncerConfig, options) + if err != nil { + return err } return nil } -func LoadVirtualConfig(ctx context.Context, options *options.VirtualClusterOptions) (*rest.Config, *clientcmdapi.Config, error) { +func LoadVirtualConfig(ctx context.Context, options *config.VirtualClusterConfig) (*rest.Config, *clientcmdapi.Config, error) { // wait for client config clientConfig, err := WaitForClientConfig(ctx, options) if err != nil { @@ -189,38 +172,11 @@ func LoadVirtualConfig(ctx context.Context, options *options.VirtualClusterOptio return virtualClusterConfig, &rawConfig, nil } -func ValidateOptions(options *options.VirtualClusterOptions) error { - // check the value of pod security standard - if options.EnforcePodSecurityStandard != "" && !allowedPodSecurityStandards[options.EnforcePodSecurityStandard] { - return fmt.Errorf("invalid argument enforce-pod-security-standard=%s, must be one of: privileged, baseline, restricted", options.EnforcePodSecurityStandard) - } - - // parse tolerations - for _, t := range options.Tolerations { - _, err := toleration.ParseToleration(t) - if err != nil { - return err - } - } - - // check if enable scheduler works correctly - if options.EnableScheduler && !options.SyncAllNodes && len(options.NodeSelector) == 0 { - options.SyncAllNodes = true - } - - // migrate fake kubelet flag - if !options.DeprecatedUseFakeKubelets { - options.DisableFakeKubelets = true - } - - return nil -} - -func WaitForClientConfig(ctx context.Context, options *options.VirtualClusterOptions) (clientcmd.ClientConfig, error) { +func WaitForClientConfig(ctx context.Context, options *config.VirtualClusterConfig) (clientcmd.ClientConfig, error) { // wait until kube config is available var clientConfig clientcmd.ClientConfig err := wait.PollUntilContextTimeout(ctx, time.Second, time.Hour, true, func(ctx context.Context) (bool, error) { - out, err := os.ReadFile(options.KubeConfigPath) + out, err := os.ReadFile(options.VirtualClusterKubeConfig().KubeConfig) if err != nil { if os.IsNotExist(err) { klog.Info("couldn't find virtual cluster kube-config, will retry in 1 seconds") @@ -267,7 +223,7 @@ func WaitForClientConfig(ctx context.Context, options *options.VirtualClusterOpt return clientConfig, nil } -func CreateVClusterKubeConfig(config *clientcmdapi.Config, options *options.VirtualClusterOptions) (*clientcmdapi.Config, error) { +func CreateVClusterKubeConfig(config *clientcmdapi.Config, options *config.VirtualClusterConfig) (*clientcmdapi.Config, error) { config = config.DeepCopy() // exchange kube config server & resolve certificate @@ -283,10 +239,10 @@ func CreateVClusterKubeConfig(config *clientcmdapi.Config, options *options.Virt config.Clusters[i].CertificateAuthorityData = o } - if options.KubeConfigServer != "" { - config.Clusters[i].Server = options.KubeConfigServer + if options.Config.ExportKubeConfig.Server != "" { + config.Clusters[i].Server = options.Config.ExportKubeConfig.Server } else { - config.Clusters[i].Server = fmt.Sprintf("https://localhost:%d", options.Port) + config.Clusters[i].Server = fmt.Sprintf("https://localhost:%d", options.Config.ControlPlane.Proxy.Port) } } @@ -322,8 +278,8 @@ func InitControllerContext( localManager, virtualManager ctrl.Manager, virtualRawConfig *clientcmdapi.Config, - vClusterOptions *options.VirtualClusterOptions, -) (*options.ControllerContext, error) { + vClusterOptions *config.VirtualClusterConfig, +) (*config.ControllerContext, error) { stopChan := make(<-chan struct{}) // get virtual cluster version @@ -344,25 +300,18 @@ func InitControllerContext( return nil, err } - // parse enabled controllers - controllers, err := ParseControllers(vClusterOptions) - if err != nil { - return nil, err - } - localDiscoveryClient, err := discovery.NewDiscoveryClientForConfig(localManager.GetConfig()) if err != nil { return nil, err } - controllers, err = DisableMissingAPIs(localDiscoveryClient, controllers) + err = vClusterOptions.DisableMissingAPIs(localDiscoveryClient) if err != nil { return nil, err } - return &options.ControllerContext{ + return &config.ControllerContext{ Context: ctx, - Controllers: controllers, LocalManager: localManager, VirtualManager: virtualManager, VirtualRawConfig: virtualRawConfig, @@ -372,11 +321,11 @@ func InitControllerContext( CurrentNamespaceClient: currentNamespaceClient, StopChan: stopChan, - Options: vClusterOptions, + Config: vClusterOptions, }, nil } -func NewCurrentNamespaceClient(ctx context.Context, currentNamespace string, localManager ctrl.Manager, options *options.VirtualClusterOptions) (client.Client, error) { +func NewCurrentNamespaceClient(ctx context.Context, currentNamespace string, localManager ctrl.Manager, options *config.VirtualClusterConfig) (client.Client, error) { var err error // currentNamespaceCache is needed for tasks such as finding out fake kubelet ips diff --git a/pkg/setup/controllers.go b/pkg/setup/controllers.go index 44515e7a8..0f666dd64 100644 --- a/pkg/setup/controllers.go +++ b/pkg/setup/controllers.go @@ -6,12 +6,12 @@ import ( "math" "time" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/controllers" "github.com/loft-sh/vcluster/pkg/controllers/resources/services" synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" "github.com/loft-sh/vcluster/pkg/coredns" "github.com/loft-sh/vcluster/pkg/metricsapiservice" - "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/plugin" "github.com/loft-sh/vcluster/pkg/pro" "github.com/loft-sh/vcluster/pkg/specialservices" @@ -33,13 +33,11 @@ import ( ) func StartControllers( - controllerContext *options.ControllerContext, + controllerContext *config.ControllerContext, controlPlaneNamespace, controlPlaneService string, controlPlaneConfig *rest.Config, ) error { - proOptions := controllerContext.Options.ProOptions - // exchange control plane client controlPlaneClient, err := pro.ExchangeControlPlaneClient(controllerContext, controlPlaneNamespace, controlPlaneConfig) if err != nil { @@ -48,7 +46,7 @@ func StartControllers( // start coredns & create syncers var syncers []syncertypes.Object - if !proOptions.NoopSyncer { + if !controllerContext.Config.Experimental.SyncSettings.DisableSync { // setup CoreDNS according to the manifest file // skip this if both integrated and dedicated coredns // deployments are explicitly disabled @@ -57,7 +55,7 @@ func StartControllers( ApplyCoreDNS(controllerContext) // delete coredns deployment if integrated core dns - if proOptions.IntegratedCoredns { + if controllerContext.Config.ControlPlane.CoreDNS.Embedded { err := controllerContext.VirtualManager.GetClient().Delete(controllerContext.Context, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "coredns", @@ -84,7 +82,7 @@ func StartControllers( } // sync remote Endpoints - if proOptions.RemoteKubeConfig != "" { + if controllerContext.Config.Experimental.IsolatedControlPlane.KubeConfig != "" { err := pro.SyncRemoteEndpoints( controllerContext.Context, types.NamespacedName{ @@ -94,7 +92,7 @@ func StartControllers( controlPlaneClient, types.NamespacedName{ Namespace: controllerContext.CurrentNamespace, - Name: controllerContext.Options.ServiceName, + Name: controllerContext.Config.ServiceName, }, controllerContext.CurrentNamespaceClient, ) @@ -104,7 +102,7 @@ func StartControllers( } // sync endpoints for noop syncer - if proOptions.NoopSyncer && proOptions.SyncKubernetesService { + if controllerContext.Config.Experimental.SyncSettings.DisableSync && controllerContext.Config.Experimental.SyncSettings.RewriteKubernetesService { err := pro.SyncNoopSyncerEndpoints( controllerContext, types.NamespacedName{ @@ -124,7 +122,7 @@ func StartControllers( } // if not noop syncer - if !proOptions.NoopSyncer { + if !controllerContext.Config.Experimental.SyncSettings.DisableSync { // make sure the kubernetes service is synced err = SyncKubernetesService(controllerContext) if err != nil { @@ -141,7 +139,7 @@ func StartControllers( // write the kube config to secret go func() { wait.Until(func() { - err := WriteKubeConfigToSecret(controllerContext.Context, controlPlaneNamespace, controlPlaneClient, controllerContext.Options, controllerContext.VirtualRawConfig, proOptions.RemoteKubeConfig != "") + err := WriteKubeConfigToSecret(controllerContext.Context, controlPlaneNamespace, controlPlaneClient, controllerContext.Config, controllerContext.VirtualRawConfig) if err != nil { klog.Errorf("Error writing kube config to secret: %v", err) } @@ -149,19 +147,17 @@ func StartControllers( }() // set leader - if !controllerContext.Options.DisablePlugins { - err = plugin.DefaultManager.SetLeader(controllerContext.Context) - if err != nil { - return fmt.Errorf("plugin set leader: %w", err) - } + err = plugin.DefaultManager.SetLeader(controllerContext.Context) + if err != nil { + return fmt.Errorf("plugin set leader: %w", err) } return nil } -func ApplyCoreDNS(controllerContext *options.ControllerContext) { +func ApplyCoreDNS(controllerContext *config.ControllerContext) { _ = wait.ExponentialBackoffWithContext(controllerContext.Context, wait.Backoff{Duration: time.Second, Factor: 1.5, Cap: time.Minute, Steps: math.MaxInt32}, func(ctx context.Context) (bool, error) { - err := coredns.ApplyManifest(ctx, controllerContext.Options.DefaultImageRegistry, controllerContext.VirtualManager.GetConfig(), controllerContext.VirtualClusterVersion) + err := coredns.ApplyManifest(ctx, controllerContext.Config.ControlPlane.Advanced.DefaultImageRegistry, controllerContext.VirtualManager.GetConfig(), controllerContext.VirtualClusterVersion) if err != nil { if errors.Is(err, coredns.ErrNoCoreDNSManifests) { klog.Infof("No CoreDNS manifests found, skipping CoreDNS configuration") @@ -198,7 +194,7 @@ func SetGlobalOwner(ctx context.Context, currentNamespaceClient client.Client, c return nil } -func SyncKubernetesService(ctx *options.ControllerContext) error { +func SyncKubernetesService(ctx *config.ControllerContext) error { err := specialservices.SyncKubernetesService( &synccontext.SyncContext{ Context: ctx.Context, @@ -209,7 +205,7 @@ func SyncKubernetesService(ctx *options.ControllerContext) error { CurrentNamespaceClient: ctx.CurrentNamespaceClient, }, ctx.CurrentNamespace, - ctx.Options.ServiceName, + ctx.Config.ServiceName, types.NamespacedName{ Name: specialservices.DefaultKubernetesSVCName, Namespace: specialservices.DefaultKubernetesSVCNamespace, @@ -227,7 +223,7 @@ func SyncKubernetesService(ctx *options.ControllerContext) error { return nil } -func StartManagers(controllerContext *options.ControllerContext, syncers []syncertypes.Object) error { +func StartManagers(controllerContext *config.ControllerContext, syncers []syncertypes.Object) error { // execute controller initializers to setup prereqs, etc. err := controllers.ExecuteInitializers(controllerContext, syncers) if err != nil { @@ -270,9 +266,9 @@ func StartManagers(controllerContext *options.ControllerContext, syncers []synce controllerContext.Context, controllerContext.CurrentNamespaceClient, controllerContext.CurrentNamespace, - controllerContext.Options.TargetNamespace, - controllerContext.Options.SetOwner, - controllerContext.Options.ServiceName, + controllerContext.Config.TargetNamespace, + controllerContext.Config.Experimental.SyncSettings.SetOwner, + controllerContext.Config.ServiceName, ) if err != nil { return errors.Wrap(err, "finding vcluster pod owner") @@ -281,67 +277,67 @@ func StartManagers(controllerContext *options.ControllerContext, syncers []synce return nil } -func RegisterOrDeregisterAPIService(ctx *options.ControllerContext) { +func RegisterOrDeregisterAPIService(ctx *config.ControllerContext) { err := metricsapiservice.RegisterOrDeregisterAPIService(ctx) if err != nil { klog.Errorf("Error registering metrics apiservice: %v", err) } } -func WriteKubeConfigToSecret(ctx context.Context, currentNamespace string, currentNamespaceClient client.Client, options *options.VirtualClusterOptions, config *clientcmdapi.Config, isRemote bool) error { - config, err := CreateVClusterKubeConfig(config, options) +func WriteKubeConfigToSecret(ctx context.Context, currentNamespace string, currentNamespaceClient client.Client, options *config.VirtualClusterConfig, syncerConfig *clientcmdapi.Config) error { + syncerConfig, err := CreateVClusterKubeConfig(syncerConfig, options) if err != nil { return err } - if options.KubeConfigContextName != "" { - config.CurrentContext = options.KubeConfigContextName + if options.Config.ExportKubeConfig.Context != "" { + syncerConfig.CurrentContext = options.Config.ExportKubeConfig.Context // update authInfo - for k := range config.AuthInfos { - config.AuthInfos[options.KubeConfigContextName] = config.AuthInfos[k] - if k != options.KubeConfigContextName { - delete(config.AuthInfos, k) + for k := range syncerConfig.AuthInfos { + syncerConfig.AuthInfos[syncerConfig.CurrentContext] = syncerConfig.AuthInfos[k] + if k != syncerConfig.CurrentContext { + delete(syncerConfig.AuthInfos, k) } break } // update cluster - for k := range config.Clusters { - config.Clusters[options.KubeConfigContextName] = config.Clusters[k] - if k != options.KubeConfigContextName { - delete(config.Clusters, k) + for k := range syncerConfig.Clusters { + syncerConfig.Clusters[syncerConfig.CurrentContext] = syncerConfig.Clusters[k] + if k != syncerConfig.CurrentContext { + delete(syncerConfig.Clusters, k) } break } // update context - for k := range config.Contexts { - tmpCtx := config.Contexts[k] - tmpCtx.Cluster = options.KubeConfigContextName - tmpCtx.AuthInfo = options.KubeConfigContextName - config.Contexts[options.KubeConfigContextName] = tmpCtx - if k != options.KubeConfigContextName { - delete(config.Contexts, k) + for k := range syncerConfig.Contexts { + tmpCtx := syncerConfig.Contexts[k] + tmpCtx.Cluster = syncerConfig.CurrentContext + tmpCtx.AuthInfo = syncerConfig.CurrentContext + syncerConfig.Contexts[syncerConfig.CurrentContext] = tmpCtx + if k != syncerConfig.CurrentContext { + delete(syncerConfig.Contexts, k) } break } } // check if we need to write the kubeconfig secrete to the default location as well - if options.KubeConfigSecret != "" { + if options.Config.ExportKubeConfig.Secret.Name != "" { // which namespace should we create the additional secret in? - secretNamespace := options.KubeConfigSecretNamespace + secretNamespace := options.Config.ExportKubeConfig.Secret.Namespace if secretNamespace == "" { secretNamespace = currentNamespace } // write the extra secret - err = kubeconfig.WriteKubeConfig(ctx, currentNamespaceClient, options.KubeConfigSecret, secretNamespace, config, isRemote) + err = kubeconfig.WriteKubeConfig(ctx, currentNamespaceClient, options.Config.ExportKubeConfig.Secret.Name, secretNamespace, syncerConfig, options.Config.Experimental.IsolatedControlPlane.KubeConfig != "") if err != nil { - return fmt.Errorf("creating %s secret in the %s ns failed: %w", options.KubeConfigSecret, secretNamespace, err) + return fmt.Errorf("creating %s secret in the %s ns failed: %w", options.Config.ExportKubeConfig.Secret.Name, secretNamespace, err) } } // write the default Secret - return kubeconfig.WriteKubeConfig(ctx, currentNamespaceClient, kubeconfig.GetDefaultSecretName(translate.VClusterName), currentNamespace, config, isRemote) + return kubeconfig.WriteKubeConfig(ctx, currentNamespaceClient, kubeconfig.GetDefaultSecretName(translate.VClusterName), currentNamespace, syncerConfig, options.Config.Experimental.IsolatedControlPlane.KubeConfig != "") } diff --git a/pkg/setup/enable_controllers.go b/pkg/setup/enable_controllers.go deleted file mode 100644 index d8b4fc598..000000000 --- a/pkg/setup/enable_controllers.go +++ /dev/null @@ -1,140 +0,0 @@ -package setup - -import ( - "fmt" - "strings" - - "github.com/loft-sh/vcluster/pkg/constants" - "github.com/loft-sh/vcluster/pkg/options" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/discovery" - "k8s.io/klog/v2" -) - -const ( - storageV1GroupVersion = "storage.k8s.io/v1" -) - -// map from groupversion to list of resources in that groupversion -// the syncers will be disabled unless that resource is advertised in that groupversion -var possibleMissing = map[string][]string{ - storageV1GroupVersion: constants.SchedulerRequiredControllers.UnsortedList(), -} - -func ParseControllers(options *options.VirtualClusterOptions) (sets.Set[string], error) { - enabledControllers := constants.DefaultEnabledControllers.Clone() - disabledControllers := sets.New[string]() - - // migrate deprecated flags - if len(options.DeprecatedDisableSyncResources) > 0 { - disabledControllers.Insert(strings.Split(options.DeprecatedDisableSyncResources, ",")...) - } - if options.DeprecatedEnablePriorityClasses { - enabledControllers.Insert("priorityclasses") - } - if !options.DeprecatedUseFakePersistentVolumes { - enabledControllers.Insert("persistentvolumes") - } - if !options.DeprecatedUseFakeNodes { - enabledControllers.Insert("nodes") - } - if options.DeprecatedEnableStorageClasses { - enabledControllers.Insert("storageclasses") - } - - for _, c := range options.Controllers { - controller := strings.TrimSpace(c) - if len(controller) == 0 { - return nil, fmt.Errorf("unrecognized controller %s, available controllers: %s", c, availableControllers()) - } - - if controller[0] == '-' { - controller = controller[1:] - disabledControllers.Insert(controller) - } else { - enabledControllers.Insert(controller) - } - - if !constants.ExistingControllers.Has(controller) { - return nil, fmt.Errorf("unrecognized controller %s, available controllers: %s", controller, availableControllers()) - } - } - // enable ingressclasses if ingress syncing is enabled and incressclasses not explicitly disabled - if enabledControllers.Has("ingresses") && !disabledControllers.Has("ingressesclasses") { - enabledControllers.Insert("ingressclasses") - } - - // enable namespaces controller in MultiNamespaceMode - if options.MultiNamespaceMode { - enabledControllers.Insert("namespaces") - } - - // do validations on dynamically added controllers here (to take into acount disabledControllers): - - // enable additional controllers required for scheduling with storage - if options.EnableScheduler && enabledControllers.Has("persistentvolumeclaims") { - klog.Infof("persistentvolumeclaim syncing and scheduler enabled, enabling required controllers: %q", constants.SchedulerRequiredControllers) - enabledControllers = enabledControllers.Union(constants.SchedulerRequiredControllers) - requiredButDisabled := disabledControllers.Intersection(constants.SchedulerRequiredControllers) - if requiredButDisabled.Len() > 0 { - klog.Warningf("persistentvolumeclaim syncing and scheduler enabled, but required syncers explicitly disabled: %q. This may result in incorrect pod scheduling.", sets.List(requiredButDisabled)) - } - if !enabledControllers.Has("storageclasses") { - klog.Info("persistentvolumeclaim syncing and scheduler enabled, but storageclass sync not enabled. Syncing host storageclasses to vcluster(hoststorageclasses)") - enabledControllers.Insert("hoststorageclasses") - if disabledControllers.HasAll("storageclasses", "hoststorageclasses") { - return nil, fmt.Errorf("persistentvolumeclaim syncing and scheduler enabled, but both storageclasses and hoststorageclasses syncers disabled") - } - } - } - - // remove explicitly disabled controllers - enabledControllers = enabledControllers.Difference(disabledControllers) - - // do validations on user configured controllers here (on just enabledControllers): - - // check if nodes controller needs to be enabled - if (options.SyncAllNodes || options.EnableScheduler) && !enabledControllers.Has("nodes") { - return nil, fmt.Errorf("node sync needs to be enabled when using --sync-all-nodes OR --enable-scheduler flags") - } - - // check if storage classes and host storage classes are enabled at the same time - if enabledControllers.HasAll("storageclasses", "hoststorageclasses") { - return nil, fmt.Errorf("you cannot sync storageclasses and hoststorageclasses at the same time. Choose only one of them") - } - - return enabledControllers, nil -} - -func availableControllers() string { - return strings.Join(sets.List(constants.ExistingControllers), ", ") -} - -// DisableMissingAPIs checks if the apis are enabled, if any are missing, disable the syncer and print a log -func DisableMissingAPIs(discoveryClient discovery.DiscoveryInterface, controllers sets.Set[string]) (sets.Set[string], error) { - enabledControllers := controllers.Clone() - for groupVersion, resourceList := range possibleMissing { - resources, err := discoveryClient.ServerResourcesForGroupVersion(groupVersion) - if err != nil && !kerrors.IsNotFound(err) { - return nil, err - } - for _, resourcePlural := range resourceList { - found := false - // search the resourses for a match - if resources != nil { - for _, r := range resources.APIResources { - if r.Name == resourcePlural { - found = true - break - } - } - } - if !found { - enabledControllers.Delete(resourcePlural) - klog.Warningf("host kubernetes apiserver not advertising resource %q in GroupVersion %q, disabling the syncer", resourcePlural, storageV1GroupVersion) - } - } - } - return enabledControllers, nil -} diff --git a/pkg/setup/enable_controllers_test.go b/pkg/setup/enable_controllers_test.go deleted file mode 100644 index bf971a64e..000000000 --- a/pkg/setup/enable_controllers_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package setup - -import ( - "testing" - - "github.com/loft-sh/vcluster/pkg/constants" - "github.com/loft-sh/vcluster/pkg/options" - "github.com/spf13/pflag" - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - fakeDiscovery "k8s.io/client-go/discovery/fake" - clientTesting "k8s.io/client-go/testing" -) - -func TestEnableControllers(t *testing.T) { - testTable := []struct { - optsModifier func(*options.VirtualClusterOptions) - desc string - errSubString string - expectEnabled []string - expectDisabled []string - expectError bool - pause bool - }{ - { - desc: "default case", - optsModifier: func(_ *options.VirtualClusterOptions) {}, - expectEnabled: sets.List(constants.DefaultEnabledControllers), - expectDisabled: sets.List(constants.ExistingControllers.Difference(constants.DefaultEnabledControllers)), - expectError: false, - }, - { - desc: "scheduler with pvc enabled, nodes not enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"persistentvolumeclaims"} - v.EnableScheduler = true - }, - expectError: true, - }, - { - desc: "scheduler with pvc enabled, storageclasses not enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"persistentvolumeclaims", "nodes"} - v.EnableScheduler = true - }, - expectEnabled: append([]string{"hoststorageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), - expectDisabled: []string{"storageclasses"}, - expectError: false, - }, - { - desc: "scheduler with pvc enabled, storageclasses enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"persistentvolumeclaims", "nodes", "storageclasses"} - v.EnableScheduler = true - }, - expectEnabled: append([]string{"storageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), - expectDisabled: []string{"hoststorageclasses"}, - expectError: false, - }, - { - desc: "scheduler with pvc enabled, hoststorageclasses enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"persistentvolumeclaims", "nodes"} - v.EnableScheduler = true - }, - expectEnabled: append([]string{"hoststorageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), - expectDisabled: []string{"storageclasses"}, - expectError: false, - }, - { - desc: "scheduler disabled, storageclasses not enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"persistentvolumeclaims"} - }, - expectEnabled: []string{}, - expectDisabled: append([]string{"storageclasses", "hoststorageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), - expectError: false, - }, - { - desc: "storageclasses and hoststorageclasses enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"storageclasses", "hoststorageclasses"} - }, - expectEnabled: []string{}, - expectDisabled: []string{}, - expectError: true, - }, - { - desc: "syncAllNodes true, nodes not enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{} - v.SyncAllNodes = true - }, - expectEnabled: []string{}, - expectDisabled: []string{}, - expectError: true, - pause: true, - }, - { - desc: "syncAllNodes true, nodes enabled", - optsModifier: func(v *options.VirtualClusterOptions) { - v.Controllers = []string{"nodes"} - v.SyncAllNodes = true - }, - expectEnabled: []string{}, - expectDisabled: []string{}, - expectError: false, - }, - } - - for _, tc := range testTable { - if tc.pause { - t.Log("you can put a breakpoint here") - } - var opts options.VirtualClusterOptions - options.AddFlags(pflag.NewFlagSet("test", pflag.PanicOnError), &opts) - t.Logf("test case: %q", tc.desc) - if tc.optsModifier != nil { - tc.optsModifier(&opts) - } - - foundControllers, err := ParseControllers(&opts) - if tc.expectError { - assert.ErrorContains(t, err, tc.errSubString, "should have failed validation") - } else { - assert.NilError(t, err, "should have passed validation") - } - - expectedNotFound := sets.New(tc.expectEnabled...).Difference(foundControllers) - assert.Assert(t, is.Len(sets.List(expectedNotFound), 0), "should be enabled, but not enabled") - - disabledFound := sets.New(tc.expectDisabled...).Intersection(foundControllers) - assert.Assert(t, is.Len(sets.List(disabledFound), 0), "should be disabled, but found enabled") - } -} - -var csiStorageCapacityV1 = metav1.APIResource{ - Name: "csistoragecapacities", - SingularName: "csistoragecapacity", - Namespaced: false, - Group: "storage.k8s.io", - Version: "v1", - Kind: "CSIStorageCapacity", -} - -var csiDriverV1 = metav1.APIResource{ - Name: "csidrivers", - SingularName: "csidriver", - Namespaced: false, - Group: "storage.k8s.io", - Version: "v1", - Kind: "CSIDriver", -} - -var csiNodeV1 = metav1.APIResource{ - Name: "csinodes", - SingularName: "csinode", - Namespaced: false, - Group: "storage.k8s.io", - Version: "v1", - Kind: "CSINode", -} - -func TestDisableMissingAPIs(t *testing.T) { - tests := []struct { - apis map[string][]metav1.APIResource - expectedNotFound sets.Set[string] - expectedFound sets.Set[string] - name string - }{ - { - name: "K8s 1.21 or lower", - apis: map[string][]metav1.APIResource{}, - expectedNotFound: constants.SchedulerRequiredControllers, - expectedFound: sets.New[string](), - }, - { - name: "K8s 1.23 or lower", - apis: map[string][]metav1.APIResource{ - storageV1GroupVersion: {csiNodeV1, csiDriverV1}, - }, - expectedNotFound: sets.New("csistoragecapacities"), - expectedFound: sets.New("csinodes", "csidrivers"), - }, - { - name: "K8s 1.24 or higher", - apis: map[string][]metav1.APIResource{ - storageV1GroupVersion: {csiNodeV1, csiDriverV1, csiStorageCapacityV1}, - }, - expectedNotFound: sets.New[string](), - expectedFound: sets.New("csistoragecapacities", "csinodes", "csidrivers"), - }, - } - - for i, testCase := range tests { - t.Logf("running test #%d: %q", i, testCase.name) - // initialize mocked discovery - resourceLists := []*metav1.APIResourceList{} - for groupVersion, resourceList := range testCase.apis { - resourceLists = append(resourceLists, &metav1.APIResourceList{GroupVersion: groupVersion, APIResources: resourceList}) - } - fakeDisoveryClient := &fakeDiscovery.FakeDiscovery{Fake: &clientTesting.Fake{Resources: resourceLists}} - - // run function - actualControllers, err := DisableMissingAPIs(fakeDisoveryClient, constants.ExistingControllers.Clone()) - assert.NilError(t, err) - - // unexpectedly not disabled - notDisabled := actualControllers.Intersection(testCase.expectedNotFound).UnsortedList() - assert.Assert(t, is.Len(notDisabled, 0), "expected %q to be disabled", testCase.expectedNotFound.UnsortedList()) - - // should be enabled - missing := testCase.expectedFound.Difference(actualControllers).UnsortedList() - assert.Assert(t, is.Len(missing, 0), "expected %q to be found, but found only: %q", testCase.expectedFound.UnsortedList(), actualControllers.Intersection(testCase.expectedFound).UnsortedList()) - } -} diff --git a/pkg/setup/initialize.go b/pkg/setup/initialize.go index 7d3768c71..f999aded7 100644 --- a/pkg/setup/initialize.go +++ b/pkg/setup/initialize.go @@ -8,14 +8,14 @@ import ( "path/filepath" "reflect" "strconv" + "strings" "time" "github.com/loft-sh/vcluster/pkg/certs" - "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/k0s" "github.com/loft-sh/vcluster/pkg/k3s" "github.com/loft-sh/vcluster/pkg/k8s" - "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/pro" "github.com/loft-sh/vcluster/pkg/specialservices" "github.com/loft-sh/vcluster/pkg/telemetry" @@ -31,21 +31,17 @@ import ( // Initialize creates the required secrets and configmaps for the control plane to start func Initialize( ctx context.Context, - workspaceNamespaceClient, currentNamespaceClient kubernetes.Interface, - workspaceNamespace, currentNamespace, vClusterName string, - options *options.VirtualClusterOptions, + options *config.VirtualClusterConfig, ) error { // Ensure that service CIDR range is written into the expected location err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(waitCtx context.Context) (bool, error) { err := initialize( waitCtx, ctx, - workspaceNamespaceClient, currentNamespaceClient, - workspaceNamespace, currentNamespace, vClusterName, options, @@ -70,50 +66,53 @@ func Initialize( func initialize( ctx context.Context, parentCtx context.Context, - workspaceNamespaceClient, currentNamespaceClient kubernetes.Interface, - workspaceNamespace, currentNamespace, vClusterName string, - options *options.VirtualClusterOptions, + options *config.VirtualClusterConfig, ) error { - distro := constants.GetVClusterDistro() + distro := options.Distro() + + // migrate from + migrateFrom := "" + if options.ControlPlane.BackingStore.EmbeddedEtcd.Enabled && options.ControlPlane.BackingStore.EmbeddedEtcd.MigrateFromExternalEtcd { + migrateFrom = "https://" + options.Name + "-etcd:2379" + } // retrieve service cidr - var serviceCIDR string - if distro != constants.K0SDistro { - var warning string - serviceCIDR, warning = servicecidr.GetServiceCIDR(ctx, currentNamespaceClient, currentNamespace) - if warning != "" { - klog.Warning(warning) - } + serviceCIDR, warning := servicecidr.GetServiceCIDR(ctx, currentNamespaceClient, currentNamespace) + if warning != "" { + klog.Warning(warning) } // check what distro are we running switch distro { - case constants.K0SDistro: + case config.K0SDistro: + // only return the first cidr, because k0s don't accept coma separated ones + serviceCIDR = strings.Split(serviceCIDR, ",")[0] + // ensure service cidr - serviceCIDR, err := servicecidr.EnsureServiceCIDRInK0sSecret(ctx, workspaceNamespaceClient, currentNamespaceClient, workspaceNamespace, currentNamespace, vClusterName) + err := k0s.WriteK0sConfig(serviceCIDR, options) if err != nil { return err } // create certificates if they are not there yet certificatesDir := "/data/k0s/pki" - err = GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.ClusterDomain) + err = GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } // should start embedded etcd? - if options.ProOptions.EtcdEmbedded && options.ProOptions.EtcdReplicas > 0 { + if options.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { err = pro.StartEmbeddedEtcd( parentCtx, vClusterName, currentNamespace, certificatesDir, - options.ProOptions.EtcdReplicas, - options.ProOptions.MigrateFrom, + int(options.ControlPlane.StatefulSet.HighAvailability.Replicas), + migrateFrom, ) if err != nil { return fmt.Errorf("start embedded etcd: %w", err) @@ -125,7 +124,7 @@ func initialize( go func() { // we need to run this with the parent ctx as otherwise this context will be cancelled by the wait // loop in Initialize - err := k0s.StartK0S(parentCtxWithCancel, cancel) + err := k0s.StartK0S(parentCtxWithCancel, cancel, options) if err != nil { klog.Fatalf("Error running k0s: %v", err) } @@ -137,18 +136,18 @@ func initialize( cancel() return err } - case constants.K3SDistro: + case config.K3SDistro: // its k3s, let's create the token secret - k3sToken, err := k3s.EnsureK3SToken(ctx, currentNamespaceClient, currentNamespace, vClusterName) + k3sToken, err := k3s.EnsureK3SToken(ctx, currentNamespaceClient, currentNamespace, vClusterName, options) if err != nil { return err } // should start embedded etcd? - if options.ProOptions.EtcdEmbedded && options.ProOptions.EtcdReplicas > 0 { + if options.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { // generate certificates certificatesDir := "/data/pki" - err := GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.ClusterDomain) + err := GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } @@ -160,8 +159,8 @@ func initialize( vClusterName, currentNamespace, certificatesDir, - options.ProOptions.EtcdReplicas, - options.ProOptions.MigrateFrom, + int(options.ControlPlane.StatefulSet.HighAvailability.Replicas), + migrateFrom, ) if err != nil { return fmt.Errorf("start embedded etcd: %w", err) @@ -172,31 +171,31 @@ func initialize( go func() { // we need to run this with the parent ctx as otherwise this context will be cancelled by the wait // loop in Initialize - err := k3s.StartK3S(parentCtx, serviceCIDR, k3sToken) + err := k3s.StartK3S(parentCtx, options, serviceCIDR, k3sToken) if err != nil { klog.Fatalf("Error running k3s: %v", err) } }() - case constants.K8SDistro, constants.EKSDistro: + case config.K8SDistro, config.EKSDistro: // try to generate k8s certificates - certificatesDir := filepath.Dir(options.ServerCaCert) + certificatesDir := filepath.Dir(options.VirtualClusterKubeConfig().ServerCACert) if certificatesDir == "/pki" { - err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.ClusterDomain) + err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } } // should start embedded etcd? - if options.ProOptions.EtcdEmbedded && options.ProOptions.EtcdReplicas > 0 { + if options.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { // start embedded etcd err := pro.StartEmbeddedEtcd( parentCtx, vClusterName, currentNamespace, certificatesDir, - options.ProOptions.EtcdReplicas, - options.ProOptions.MigrateFrom, + int(options.ControlPlane.StatefulSet.HighAvailability.Replicas), + migrateFrom, ) if err != nil { return fmt.Errorf("start embedded etcd: %w", err) @@ -207,16 +206,35 @@ func initialize( go func() { // we need to run this with the parent ctx as otherwise this context will be cancelled by the wait // loop in Initialize - err := k8s.StartK8S(parentCtx, serviceCIDR) + var err error + if distro == config.K8SDistro { + err = k8s.StartK8S( + parentCtx, + serviceCIDR, + options.ControlPlane.Distro.K8S.APIServer, + options.ControlPlane.Distro.K8S.ControllerManager, + options.ControlPlane.Distro.K8S.Scheduler, + options, + ) + } else if distro == config.EKSDistro { + err = k8s.StartK8S( + parentCtx, + serviceCIDR, + options.ControlPlane.Distro.EKS.APIServer, + options.ControlPlane.Distro.EKS.ControllerManager, + options.ControlPlane.Distro.EKS.Scheduler, + options, + ) + } if err != nil { klog.Fatalf("Error running k8s: %v", err) } }() - case constants.Unknown: - certificatesDir := filepath.Dir(options.ServerCaCert) + case config.Unknown: + certificatesDir := filepath.Dir(options.VirtualClusterKubeConfig().ServerCACert) if certificatesDir == "/pki" { // generate k8s certificates - err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.ClusterDomain) + err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } diff --git a/pkg/setup/proxy.go b/pkg/setup/proxy.go index 5a03b6d32..fc2cdcc94 100644 --- a/pkg/setup/proxy.go +++ b/pkg/setup/proxy.go @@ -16,7 +16,7 @@ func StartProxy(ctx *config.ControllerContext, controlPlaneNamespace, controlPla } // start the proxy - proxyServer, err := server.NewServer(ctx, ctx.Options.RequestHeaderCaCert, ctx.Options.ClientCaCert) + proxyServer, err := server.NewServer(ctx, ctx.Config.VirtualClusterKubeConfig().RequestHeaderCACert, ctx.Config.VirtualClusterKubeConfig().ClientCACert) if err != nil { return err } diff --git a/pkg/util/servicecidr/servicecidr.go b/pkg/util/servicecidr/servicecidr.go index 28bd3555c..2caab4e82 100644 --- a/pkg/util/servicecidr/servicecidr.go +++ b/pkg/util/servicecidr/servicecidr.go @@ -4,69 +4,18 @@ import ( "context" "fmt" "net" - "os" "strings" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" ) const ( - CIDRConfigMapPrefix = "vc-cidr-" - CIDRConfigMapKey = "cidr" - K0sConfigKey = "config.yaml" - K0sCIDRPlaceHolder = "CIDR_PLACEHOLDER" - ErrorMessageFind = "The range of valid IPs is " FallbackCIDR = "10.96.0.0/12" ) -func GetK0sSecretName(vClusterName string) string { - return fmt.Sprintf("vc-%s-config", vClusterName) -} - -func EnsureServiceCIDRInK0sSecret( - ctx context.Context, - workspaceNamespaceClient, - currentNamespaceClient kubernetes.Interface, - workspaceNamespace, - currentNamespace string, - vClusterName string, -) (string, error) { - secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, GetK0sSecretName(vClusterName), metav1.GetOptions{}) - if err != nil { - return "", fmt.Errorf("could not read k0s configuration secret %s/%s: %w", currentNamespace, GetK0sSecretName(vClusterName), err) - } - - // verify secret - configData, ok := secret.Data[K0sConfigKey] - if !ok { - return "", fmt.Errorf("k0s configuration secret %s/%s does not contain the expected key - %s", secret.Namespace, secret.Name, K0sConfigKey) - } - - // find out correct cidr - serviceCIDR, warning := GetServiceCIDR(ctx, workspaceNamespaceClient, workspaceNamespace) - if warning != "" { - klog.Info(warning) - } - // only return the first cidr, because k0s don't accept coma separated ones - serviceCIDR = strings.Split(serviceCIDR, ",")[0] - - // apply changes - updatedConfig := []byte(strings.ReplaceAll(string(configData), K0sCIDRPlaceHolder, serviceCIDR)) - - // write the config to file - err = os.WriteFile("/tmp/k0s-config.yaml", updatedConfig, 0640) - if err != nil { - klog.Errorf("error while write k0s config to file: %s", err.Error()) - return "", err - } - - return serviceCIDR, nil -} - func GetServiceCIDR(ctx context.Context, client kubernetes.Interface, namespace string) (string, string) { ipv4CIDR, ipv4Err := getServiceCIDR(ctx, client, namespace, false) ipv6CIDR, ipv6Err := getServiceCIDR(ctx, client, namespace, true) From 2e39c671093ce0072960e14177a07d1ff2a8355b Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 11 Mar 2024 18:48:08 +0100 Subject: [PATCH 3/5] fix: unit tests --- chart/values.yaml | 4 +- pkg/patches/patch_test.go | 90 +++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index 5de8abd51..eca8f96d7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -9,10 +9,10 @@ sync: enabled: true configMaps: enabled: true - all: true + all: false secrets: enabled: true - all: true + all: false pods: enabled: true translateImage: {} diff --git a/pkg/patches/patch_test.go b/pkg/patches/patch_test.go index 6bf79e34e..03f88210e 100644 --- a/pkg/patches/patch_test.go +++ b/pkg/patches/patch_test.go @@ -7,10 +7,10 @@ import ( "strings" "testing" + "github.com/loft-sh/vcluster/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/loft-sh/vcluster/pkg/genericsyncconfig" patchesregex "github.com/loft-sh/vcluster/pkg/patches/regex" "github.com/loft-sh/vcluster/pkg/util/translate" yaml "gopkg.in/yaml.v3" @@ -19,7 +19,7 @@ import ( type patchTestCase struct { name string - patch *genericsyncconfig.Patch + patch *config.Patch obj1 string obj2 string @@ -37,8 +37,8 @@ func TestPatch(t *testing.T) { testCases := []*patchTestCase{ { name: "copy merge", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeCopyFromObject, + patch: &config.Patch{ + Operation: config.PatchTypeCopyFromObject, FromPath: "status.test", Path: "test", }, @@ -52,8 +52,8 @@ test: test`, }, { name: "copy", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeCopyFromObject, + patch: &config.Patch{ + Operation: config.PatchTypeCopyFromObject, FromPath: "status", Path: "status", }, @@ -66,8 +66,8 @@ status: }, { name: "simple", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeReplace, + patch: &config.Patch{ + Operation: config.PatchTypeReplace, Path: "test.test2", Value: "abc", }, @@ -78,8 +78,8 @@ status: }, { name: "insert", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeAdd, + patch: &config.Patch{ + Operation: config.PatchTypeAdd, Path: "test.test2[0].test3", Value: "abc", }, @@ -94,8 +94,8 @@ test2: {}`, }, { name: "insert slice", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeAdd, + patch: &config.Patch{ + Operation: config.PatchTypeAdd, Path: "test.test2", Value: "abc", }, @@ -109,8 +109,8 @@ test2: {}`, }, { name: "insert slice", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeReplace, + patch: &config.Patch{ + Operation: config.PatchTypeReplace, Path: "test..abc", Value: "def", }, @@ -125,11 +125,11 @@ test2: {}`, }, { name: "condition", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeReplace, + patch: &config.Patch{ + Operation: config.PatchTypeReplace, Path: "test.abc", Value: "def", - Conditions: []*genericsyncconfig.PatchCondition{ + Conditions: []*config.PatchCondition{ { Path: "test.status", Empty: &True, @@ -143,11 +143,11 @@ test2: {}`, }, { name: "condition equal", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeReplace, + patch: &config.Patch{ + Operation: config.PatchTypeReplace, Path: "test.abc", Value: "def", - Conditions: []*genericsyncconfig.PatchCondition{ + Conditions: []*config.PatchCondition{ { Path: "test.status", Equal: map[string]interface{}{ @@ -167,11 +167,11 @@ test2: {}`, }, { name: "condition equal", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeReplace, + patch: &config.Patch{ + Operation: config.PatchTypeReplace, Path: "test.abc", Value: "def", - Conditions: []*genericsyncconfig.PatchCondition{ + Conditions: []*config.PatchCondition{ { Path: "test.status", Equal: map[string]interface{}{ @@ -191,8 +191,8 @@ test2: {}`, }, { name: "resolve label selector", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteLabelSelector, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteLabelSelector, Path: "test.abc", }, nameResolver: &fakeNameResolver{}, @@ -204,8 +204,8 @@ test2: {}`, }, { name: "resolve empty label selector", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteLabelSelector, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteLabelSelector, Path: "test.abc", }, nameResolver: &fakeNameResolver{}, @@ -216,8 +216,8 @@ test2: {}`, }, { name: "resolve filled label selector", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteLabelSelector, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteLabelSelector, Path: "test.abc", }, nameResolver: &fakeNameResolver{}, @@ -231,8 +231,8 @@ test2: {}`, }, { name: "rewrite name", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "name", }, nameResolver: &fakeVirtualToHostNameResolver{ @@ -244,8 +244,8 @@ test2: {}`, }, { name: "rewrite name - invalid object", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "name{]", }, obj1: `name: {}`, @@ -253,8 +253,8 @@ test2: {}`, }, { name: "rewrite name - namespace based", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "root.list", NamePath: "nm", NamespacePath: "ns", @@ -278,8 +278,8 @@ test2: {}`, }, { name: "rewrite name - multiple - no namespace", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "root.list", NamePath: "nm", }, @@ -302,8 +302,8 @@ test2: {}`, }, { name: "rewrite name - multiple name matches", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "root.includes", NamePath: "names..nm", NamespacePath: "namespace", @@ -327,8 +327,8 @@ test2: {}`, }, { name: "rewrite name - single name match - non array", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "root.includes", NamePath: "nm", NamespacePath: "namespace", @@ -348,8 +348,8 @@ test2: {}`, }, { name: "rewrite name - multiple name matches - multiple namespace references", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "root.includes", NamePath: "..names..nm", NamespacePath: "..namespaces..ns", @@ -398,8 +398,8 @@ test2: {}`, // }, { name: "rewrite name should not panic when match is not scalar", - patch: &genericsyncconfig.Patch{ - Operation: genericsyncconfig.PatchTypeRewriteName, + patch: &config.Patch{ + Operation: config.PatchTypeRewriteName, Path: "test.endpoints[*]", }, nameResolver: &fakeVirtualToHostNameResolver{}, From 1fea51c0139984fc94c52df00d23e1d0ae1f2729 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Fri, 15 Mar 2024 09:28:37 +0100 Subject: [PATCH 4/5] refactor: improvements --- .github/workflows/release.yaml | 5 +- .golangci.yml | 1 + chart/templates/_rbac.tpl | 2 +- chart/templates/clusterrole.yaml | 4 +- chart/templates/role.yaml | 2 +- chart/templates/service-monitor.yaml | 8 +- chart/tests/clusterrole_test.yaml | 18 ++ chart/tests/role_test.yaml | 22 ++ chart/tests/service-monitor_test.yaml | 5 +- chart/values.schema.json | 80 +++---- chart/values.yaml | 22 +- cmd/vcluster/cmd/start.go | 4 + config/config.go | 226 +++++++++--------- pkg/config/config.go | 8 +- pkg/config/parse.go | 2 +- pkg/config/validation.go | 12 +- pkg/controllers/resources/nodes/syncer.go | 2 +- .../resources/nodes/syncer_test.go | 4 +- pkg/metricsapiservice/register.go | 2 +- pkg/server/server.go | 4 +- test/e2e_metrics_proxy/values.yaml | 6 +- test/e2e_node/values.yaml | 3 +- test/e2e_scheduler/values.yaml | 3 +- 23 files changed, 233 insertions(+), 212 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1fef6ef1b..a01c01305 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -90,10 +90,7 @@ jobs: RELEASE_VERSION=$(echo $GITHUB_REF | sed -nE 's!refs/tags/v!!p') helm plugin install https://github.com/chartmuseum/helm-push.git helm repo add chartmuseum $CHART_MUSEUM_URL --username $CHART_MUSEUM_USER --password $CHART_MUSEUM_PASSWORD - helm cm-push --force --version="$RELEASE_VERSION" --app-version="$RELEASE_VERSION" charts/k3s/ chartmuseum - helm cm-push --force --version="$RELEASE_VERSION" --app-version="$RELEASE_VERSION" charts/k0s/ chartmuseum - helm cm-push --force --version="$RELEASE_VERSION" --app-version="$RELEASE_VERSION" charts/k8s/ chartmuseum - helm cm-push --force --version="$RELEASE_VERSION" --app-version="$RELEASE_VERSION" charts/eks/ chartmuseum + helm cm-push --force --version="$RELEASE_VERSION" --app-version="$RELEASE_VERSION" chart chartmuseum env: CHART_MUSEUM_URL: "https://charts.loft.sh/" CHART_MUSEUM_USER: ${{ secrets.CHART_MUSEUM_USER }} diff --git a/.golangci.yml b/.golangci.yml index c0d5f179e..79637e252 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,6 +5,7 @@ linters: disable-all: true enable: - asasalint + - tagalign - asciicheck - bidichk - decorder diff --git a/chart/templates/_rbac.tpl b/chart/templates/_rbac.tpl index dd32e3f7c..edfe0eea6 100644 --- a/chart/templates/_rbac.tpl +++ b/chart/templates/_rbac.tpl @@ -26,7 +26,7 @@ .Values.sync.fromHost.ingressClasses.enabled .Values.sync.fromHost.storageClasses.enabled .Values.sync.fromHost.nodes.enabled - .Values.observability.metrics.proxy.nodes.enabled + .Values.observability.metrics.proxy.nodes .Values.experimental.multiNamespaceMode.enabled .Values.networking.resolveServices -}} {{- true -}} diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index c2ad62124..358ac708c 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -31,7 +31,7 @@ rules: resources: [ "pods"] verbs: ["get", "watch", "list"] {{- end }} - {{- if and .Values.sync.fromHost.nodes.enabled .Values.sync.fromHost.nodes.syncLabelsTaints }} + {{- if and .Values.sync.fromHost.nodes.enabled .Values.sync.fromHost.nodes.syncBackChanges }} - apiGroups: [""] resources: ["nodes", "nodes/status"] verbs: ["update", "patch"] @@ -84,7 +84,7 @@ rules: resources: ["namespaces", "serviceaccounts"] verbs: ["create", "delete", "patch", "update", "get", "watch", "list"] {{- end }} - {{- if .Values.observability.metrics.proxy.nodes.enabled }} + {{- if .Values.observability.metrics.proxy.nodes }} - apiGroups: ["metrics.k8s.io"] resources: ["nodes"] verbs: ["get", "list"] diff --git a/chart/templates/role.yaml b/chart/templates/role.yaml index cde2b6c49..37ed162be 100644 --- a/chart/templates/role.yaml +++ b/chart/templates/role.yaml @@ -47,7 +47,7 @@ rules: resources: ["leases"] verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] {{- end }} - {{- if .Values.observability.metrics.proxy.pods.enabled }} + {{- if .Values.observability.metrics.proxy.pods }} - apiGroups: ["metrics.k8s.io"] resources: ["pods"] verbs: ["get", "list"] diff --git a/chart/templates/service-monitor.yaml b/chart/templates/service-monitor.yaml index ee172c4a4..a92e166fe 100644 --- a/chart/templates/service-monitor.yaml +++ b/chart/templates/service-monitor.yaml @@ -1,4 +1,4 @@ -{{- if .Values.controlPlane.observability.serviceMonitor.enabled }} +{{- if .Values.controlPlane.serviceMonitor.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -9,10 +9,10 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" - {{- if .Values.controlPlane.observability.serviceMonitor.labels }} -{{ toYaml .Values.controlPlane.observability.serviceMonitor.labels | indent 4 }} + {{- if .Values.controlPlane.serviceMonitor.labels }} +{{ toYaml .Values.controlPlane.serviceMonitor.labels | indent 4 }} {{- end }} - {{- $annotations := merge dict .Values.controlPlane.observability.serviceMonitor.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} + {{- $annotations := merge dict .Values.controlPlane.serviceMonitor.annotations .Values.controlPlane.advanced.globalMetadata.annotations }} {{- if $annotations }} annotations: {{ toYaml $annotations | indent 4 }} diff --git a/chart/tests/clusterrole_test.yaml b/chart/tests/clusterrole_test.yaml index 4552f71d4..837eed9f8 100644 --- a/chart/tests/clusterrole_test.yaml +++ b/chart/tests/clusterrole_test.yaml @@ -193,3 +193,21 @@ tests: resources: [ "features", "virtualclusters" ] verbs: [ "get", "list", "watch" ] + - it: metrics proxy + set: + observability: + metrics: + proxy: + nodes: true + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - contains: + path: rules + content: + apiGroups: [ "metrics.k8s.io" ] + resources: [ "nodes" ] + verbs: [ "get", "list" ] diff --git a/chart/tests/role_test.yaml b/chart/tests/role_test.yaml index e12bde988..615075fd3 100644 --- a/chart/tests/role_test.yaml +++ b/chart/tests/role_test.yaml @@ -152,3 +152,25 @@ tests: - equal: path: metadata.name value: vc-mn-my-release-v-my-namespace + + - it: metrics proxy + set: + observability: + metrics: + proxy: + pods: true + release: + name: my-release + namespace: my-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: kind + value: Role + - contains: + path: rules + content: + apiGroups: [ "metrics.k8s.io" ] + resources: [ "pods" ] + verbs: [ "get", "list" ] diff --git a/chart/tests/service-monitor_test.yaml b/chart/tests/service-monitor_test.yaml index 122b0d81e..c05c0e04d 100644 --- a/chart/tests/service-monitor_test.yaml +++ b/chart/tests/service-monitor_test.yaml @@ -14,9 +14,8 @@ tests: namespace: my-namespace set: controlPlane: - observability: - serviceMonitor: - enabled: true + serviceMonitor: + enabled: true asserts: - hasDocuments: count: 1 diff --git a/chart/values.schema.json b/chart/values.schema.json index 8adf88ad9..ccee3d0b7 100755 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -66,9 +66,9 @@ "$ref": "#/$defs/HostPathMapper", "description": "HostPathMapper defines if vCluster should rewrite host paths." }, - "observability": { - "$ref": "#/$defs/ControlPlaneObservability", - "description": "Observability defines if a service monitor should get deployed by helm." + "serviceMonitor": { + "$ref": "#/$defs/ServiceMonitor", + "description": "ServiceMonitor can be used to automatically create a service monitor for vCluster deployment itself." }, "advanced": { "$ref": "#/$defs/ControlPlaneAdvanced", @@ -199,16 +199,6 @@ "additionalProperties": false, "type": "object" }, - "ControlPlaneObservability": { - "properties": { - "serviceMonitor": { - "$ref": "#/$defs/ServiceMonitor", - "description": "ServiceMonitor can be used to automatically create a service monitor for vCluster deployment itself." - } - }, - "additionalProperties": false, - "type": "object" - }, "ControlPlanePersistence": { "properties": { "volumeClaim": { @@ -1510,11 +1500,11 @@ "MetricsProxy": { "properties": { "nodes": { - "$ref": "#/$defs/EnableSwitch", + "type": "boolean", "description": "Nodes defines if metrics-server nodes api should get proxied from host to vCluster." }, "pods": { - "$ref": "#/$defs/EnableSwitch", + "type": "boolean", "description": "Pods defines if metrics-server pods api should get proxied from host to vCluster." } }, @@ -1571,12 +1561,12 @@ "$ref": "#/$defs/ReplicateServices", "description": "ReplicateServices allows replicating services from the host within the vCluster or the other way around." }, - "resolveServices": { + "resolveDNS": { "items": { - "$ref": "#/$defs/ResolveServices" + "$ref": "#/$defs/ResolveDNS" }, "type": "array", - "description": "ResolveServices allows to define extra DNS rules. This only works if embedded coredns is configured." + "description": "ResolveDNS allows to define extra DNS rules. This only works if embedded coredns is configured." }, "advanced": { "$ref": "#/$defs/NetworkingAdvanced", @@ -2094,46 +2084,40 @@ "additionalProperties": false, "type": "object" }, - "ResolveServiceHostname": { + "ResolveDNS": { "properties": { "hostname": { "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, - "ResolveServiceService": { - "properties": { + }, "service": { "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, - "ResolveServiceTarget": { - "properties": { - "vcluster": { - "$ref": "#/$defs/ResolveServiceService" }, - "host": { - "$ref": "#/$defs/ResolveServiceService" + "namespace": { + "type": "string" }, - "external": { - "$ref": "#/$defs/ResolveServiceHostname" + "target": { + "$ref": "#/$defs/ResolveDNSTarget" } }, "additionalProperties": false, "type": "object" }, - "ResolveServices": { + "ResolveDNSTarget": { "properties": { + "hostname": { + "type": "string" + }, + "vcluster": { + "type": "string" + }, "service": { "type": "string" }, - "target": { - "$ref": "#/$defs/ResolveServiceTarget" + "namespace": { + "type": "string" + }, + "mode": { + "type": "string" } }, "additionalProperties": false, @@ -2319,6 +2303,10 @@ }, "SyncNodeSelector": { "properties": { + "all": { + "type": "boolean", + "description": "All specifies if all nodes should get synced by vCluster from the host to the vCluster or only the ones where pods are assigned to." + }, "labels": { "additionalProperties": { "type": "string" @@ -2336,13 +2324,9 @@ "type": "boolean", "description": "Enabled specifies if syncing real nodes should be enabled. If this is disabled, vCluster will create fake nodes instead." }, - "syncAll": { - "type": "boolean", - "description": "SyncAll specifies if all nodes should get synced by vCluster from the host to the vCluster or only the ones where pods are assigned to." - }, - "syncLabelsTaints": { + "syncBackChanges": { "type": "boolean", - "description": "SyncLabelsTaints enables syncing labels and taints from the vCluster to the host cluster. If this is enabled someone within the vCluster will be able to change the labels and taints of the host cluster node." + "description": "SyncBackChanges enables syncing labels and taints from the vCluster to the host cluster. If this is enabled someone within the vCluster will be able to change the labels and taints of the host cluster node." }, "clearImageStatus": { "type": "boolean", diff --git a/chart/values.yaml b/chart/values.yaml index f814bbf65..4fe0efe60 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -53,10 +53,10 @@ sync: enabled: false nodes: enabled: false - syncAll: false - syncLabelsTaints: false + syncBackChanges: false clearImageStatus: false selector: + all: false labels: {} # Control Plane Options @@ -224,6 +224,7 @@ controlPlane: proxy: bindAddress: "0.0.0.0" port: 8443 + extraSANs: [] coredns: enabled: true @@ -342,11 +343,10 @@ controlPlane: startupProbe: enabled: true - observability: - serviceMonitor: - enabled: false - labels: {} - annotations: {} + serviceMonitor: + enabled: false + labels: {} + annotations: {} advanced: defaultImageRegistry: "" @@ -388,17 +388,15 @@ rbac: observability: metrics: proxy: - nodes: - enabled: false - pods: - enabled: false + nodes: false + pods: false networking: # Embedded CoreDNS plugin config replicateServices: toHost: [] fromHost: [] - resolveServices: [] + resolveDNS: [] advanced: clusterDomain: "cluster.local" fallbackHostCluster: true diff --git a/cmd/vcluster/cmd/start.go b/cmd/vcluster/cmd/start.go index a1c831a54..2f566422b 100644 --- a/cmd/vcluster/cmd/start.go +++ b/cmd/vcluster/cmd/start.go @@ -59,7 +59,11 @@ func NewStartCommand() *cobra.Command { } cmd.Flags().StringVar(&startOptions.Config, "config", "/var/vcluster/config.yaml", "The path where to find the vCluster config to load") + + // Should only used for development cmd.Flags().StringArrayVar(&startOptions.SetValues, "set", []string{}, "Set values for the config. E.g. --set 'exportKubeConfig.secret.name=my-name'") + + _ = cmd.Flags().MarkHidden("set") return cmd } diff --git a/config/config.go b/config/config.go index 3189da64c..933c165af 100644 --- a/config/config.go +++ b/config/config.go @@ -7,36 +7,36 @@ type Config struct { // ExportKubeConfig describes how vCluster should export the vCluster kube config ExportKubeConfig ExportKubeConfig `json:"exportKubeConfig,omitempty"` - // ControlPlane holds options how to configure the vCluster control-plane - ControlPlane ControlPlane `json:"controlPlane,omitempty"` - // Sync describes how to sync resources from the vCluster to host cluster and back Sync Sync `json:"sync,omitempty"` - // Observability holds options to proxy metrics from the host cluster into the vCluster - Observability Observability `json:"observability,omitempty"` - // Networking are networking options related to the vCluster Networking Networking `json:"networking,omitempty"` - // Plugins define what vCluster plugins to load. - Plugins map[string]Plugins `json:"plugins,omitempty"` - // Policies defines policies to enforce for the vCluster deployment as well as within the vCluster Policies Policies `json:"policies,omitempty"` + // Observability holds options to proxy metrics from the host cluster into the vCluster + Observability Observability `json:"observability,omitempty"` + + // ControlPlane holds options how to configure the vCluster control-plane + ControlPlane ControlPlane `json:"controlPlane,omitempty"` + // RBAC are role based access control options for the vCluster RBAC RBAC `json:"rbac,omitempty"` + // Plugins define what vCluster plugins to load. + Plugins map[string]Plugins `json:"plugins,omitempty"` + + // Platform holds options how vCluster should connect to vCluster platform. + Platform Platform `json:"platform,omitempty"` + // Experimental are alpha features for vCluster. Configuration here might change, so be careful with this. Experimental Experimental `json:"experimental,omitempty"` // Telemetry is the configuration related to telemetry gathered about vCluster usage. Telemetry Telemetry `json:"telemetry,omitempty"` - // Platform holds options how vCluster should connect to vCluster platform. - Platform Platform `json:"platform,omitempty"` - // ServiceCIDR holds the service cidr for the vCluster. Please do not use that option anymore. ServiceCIDR string `json:"serviceCIDR,omitempty"` @@ -83,51 +83,51 @@ type Sync struct { } type SyncToHost struct { + // Pods defines if pods created within the vCluster should get synced to the host cluster. + Pods SyncPods `json:"pods,omitempty"` + // Secrets defines if secrets created within the vCluster should get synced to the host cluster. + Secrets SyncAllResource `json:"secrets,omitempty"` + // ConfigMaps defines if config maps created within the vCluster should get synced to the host cluster. + ConfigMaps SyncAllResource `json:"configMaps,omitempty"` + // Ingresses defines if ingresses created within the vCluster should get synced to the host cluster. + Ingresses EnableSwitch `json:"ingresses,omitempty"` // Services defines if services created within the vCluster should get synced to the host cluster. Services EnableSwitch `json:"services,omitempty"` // Endpoints defines if endpoints created within the vCluster should get synced to the host cluster. Endpoints EnableSwitch `json:"endpoints,omitempty"` - // Ingresses defines if ingresses created within the vCluster should get synced to the host cluster. - Ingresses EnableSwitch `json:"ingresses,omitempty"` - // PriorityClasses defines if priority classes created within the vCluster should get synced to the host cluster. - PriorityClasses EnableSwitch `json:"priorityClasses,omitempty"` // NetworkPolicies defines if network policies created within the vCluster should get synced to the host cluster. NetworkPolicies EnableSwitch `json:"networkPolicies,omitempty"` + // PersistentVolumeClaims defines if persistent volume claims created within the vCluster should get synced to the host cluster. + PersistentVolumeClaims EnableSwitch `json:"persistentVolumeClaims,omitempty"` + // PersistentVolumes defines if persistent volumes created within the vCluster should get synced to the host cluster. + PersistentVolumes EnableSwitch `json:"persistentVolumes,omitempty"` // VolumeSnapshots defines if volume snapshots created within the vCluster should get synced to the host cluster. VolumeSnapshots EnableSwitch `json:"volumeSnapshots,omitempty"` - // PodDisruptionBudgets defines if pod disruption budgets created within the vCluster should get synced to the host cluster. - PodDisruptionBudgets EnableSwitch `json:"podDisruptionBudgets,omitempty"` - // ServiceAccounts defines if service accounts created within the vCluster should get synced to the host cluster. - ServiceAccounts EnableSwitch `json:"serviceAccounts,omitempty"` // StorageClasses defines if storage classes created within the vCluster should get synced to the host cluster. StorageClasses EnableSwitch `json:"storageClasses,omitempty"` - // PersistentVolumes defines if persistent volumes created within the vCluster should get synced to the host cluster. - PersistentVolumes EnableSwitch `json:"persistentVolumes,omitempty"` - // PersistentVolumeClaims defines if persistent volume claims created within the vCluster should get synced to the host cluster. - PersistentVolumeClaims EnableSwitch `json:"persistentVolumeClaims,omitempty"` - // ConfigMaps defines if config maps created within the vCluster should get synced to the host cluster. - ConfigMaps SyncAllResource `json:"configMaps,omitempty"` - // Secrets defines if secrets created within the vCluster should get synced to the host cluster. - Secrets SyncAllResource `json:"secrets,omitempty"` - // Pods defines if pods created within the vCluster should get synced to the host cluster. - Pods SyncPods `json:"pods,omitempty"` + // ServiceAccounts defines if service accounts created within the vCluster should get synced to the host cluster. + ServiceAccounts EnableSwitch `json:"serviceAccounts,omitempty"` + // PodDisruptionBudgets defines if pod disruption budgets created within the vCluster should get synced to the host cluster. + PodDisruptionBudgets EnableSwitch `json:"podDisruptionBudgets,omitempty"` + // PriorityClasses defines if priority classes created within the vCluster should get synced to the host cluster. + PriorityClasses EnableSwitch `json:"priorityClasses,omitempty"` } type SyncFromHost struct { - // CSIDrivers defines if csi drivers should get synced from the host cluster to the vCluster, but not back. - CSIDrivers EnableSwitch `json:"csiDrivers,omitempty"` - // CSINodes defines if csi nodes should get synced from the host cluster to the vCluster, but not back. - CSINodes EnableSwitch `json:"csiNodes,omitempty"` - // CSIStorageCapacities defines if csi storage capacities should get synced from the host cluster to the vCluster, but not back. - CSIStorageCapacities EnableSwitch `json:"csiStorageCapacities,omitempty"` - // IngressClasses defines if ingress classes should get synced from the host cluster to the vCluster, but not back. - IngressClasses EnableSwitch `json:"ingressClasses,omitempty"` + // Nodes defines if nodes should get synced from the host cluster to the vCluster, but not back. + Nodes SyncNodes `json:"nodes,omitempty"` // Events defines if events should get synced from the host cluster to the vCluster, but not back. Events EnableSwitch `json:"events,omitempty"` + // IngressClasses defines if ingress classes should get synced from the host cluster to the vCluster, but not back. + IngressClasses EnableSwitch `json:"ingressClasses,omitempty"` // StorageClasses defines if storage classes should get synced from the host cluster to the vCluster, but not back. StorageClasses EnableSwitch `json:"storageClasses,omitempty"` - // Nodes defines if nodes should get synced from the host cluster to the vCluster, but not back. - Nodes SyncNodes `json:"nodes,omitempty"` + // CSINodes defines if csi nodes should get synced from the host cluster to the vCluster, but not back. + CSINodes EnableSwitch `json:"csiNodes,omitempty"` + // CSIDrivers defines if csi drivers should get synced from the host cluster to the vCluster, but not back. + CSIDrivers EnableSwitch `json:"csiDrivers,omitempty"` + // CSIStorageCapacities defines if csi storage capacities should get synced from the host cluster to the vCluster, but not back. + CSIStorageCapacities EnableSwitch `json:"csiStorageCapacities,omitempty"` } type EnableSwitch struct { @@ -176,11 +176,8 @@ type SyncNodes struct { // Enabled specifies if syncing real nodes should be enabled. If this is disabled, vCluster will create fake nodes instead. Enabled bool `json:"enabled,omitempty"` - // SyncAll specifies if all nodes should get synced by vCluster from the host to the vCluster or only the ones where pods are assigned to. - SyncAll bool `json:"syncAll,omitempty"` - - // SyncLabelsTaints enables syncing labels and taints from the vCluster to the host cluster. If this is enabled someone within the vCluster will be able to change the labels and taints of the host cluster node. - SyncLabelsTaints bool `json:"syncLabelsTaints,omitempty"` + // SyncBackChanges enables syncing labels and taints from the vCluster to the host cluster. If this is enabled someone within the vCluster will be able to change the labels and taints of the host cluster node. + SyncBackChanges bool `json:"syncBackChanges,omitempty"` // ClearImageStatus will erase the image status when syncing a node. This allows to hide images that are pulled by the node. ClearImageStatus bool `json:"clearImageStatus,omitempty"` @@ -190,6 +187,9 @@ type SyncNodes struct { } type SyncNodeSelector struct { + // All specifies if all nodes should get synced by vCluster from the host to the vCluster or only the ones where pods are assigned to. + All bool `json:"all,omitempty"` + // Labels are the node labels used to sync nodes from host cluster to vCluster. This will also set the node selector when syncing a pod from vCluster to host cluster to the same value. Labels map[string]string `json:"labels,omitempty"` } @@ -199,11 +199,6 @@ type Observability struct { Metrics ObservabilityMetrics `json:"metrics,omitempty"` } -type ControlPlaneObservability struct { - // ServiceMonitor can be used to automatically create a service monitor for vCluster deployment itself. - ServiceMonitor ServiceMonitor `json:"serviceMonitor,omitempty"` -} - type ServiceMonitor struct { // Enabled configures if helm should create the service monitor. Enabled bool `json:"enabled,omitempty"` @@ -222,18 +217,18 @@ type ObservabilityMetrics struct { type MetricsProxy struct { // Nodes defines if metrics-server nodes api should get proxied from host to vCluster. - Nodes EnableSwitch `json:"nodes,omitempty"` + Nodes bool `json:"nodes,omitempty"` // Pods defines if metrics-server pods api should get proxied from host to vCluster. - Pods EnableSwitch `json:"pods,omitempty"` + Pods bool `json:"pods,omitempty"` } type Networking struct { // ReplicateServices allows replicating services from the host within the vCluster or the other way around. ReplicateServices ReplicateServices `json:"replicateServices,omitempty"` - // ResolveServices allows to define extra DNS rules. This only works if embedded coredns is configured. - ResolveServices []ResolveServices `json:"resolveServices,omitempty"` + // ResolveDNS allows to define extra DNS rules. This only works if embedded coredns is configured. + ResolveDNS []ResolveDNS `json:"resolveDNS,omitempty"` // Advanced holds advanced network options. Advanced NetworkingAdvanced `json:"advanced,omitempty"` @@ -256,23 +251,19 @@ type ServiceMapping struct { To string `json:"to,omitempty"` } -type ResolveServices struct { - Service string `json:"service,omitempty"` - Target ResolveServiceTarget `json:"target,omitempty"` +type ResolveDNS struct { + Hostname string `json:"hostname"` + Service string `json:"service"` + Namespace string `json:"namespace"` + Target ResolveDNSTarget `json:"target,omitempty"` } -type ResolveServiceTarget struct { - VCluster ResolveServiceService `json:"vcluster,omitempty"` - Host ResolveServiceService `json:"host,omitempty"` - External ResolveServiceHostname `json:"external,omitempty"` -} - -type ResolveServiceService struct { - Service string `json:"service,omitempty"` -} - -type ResolveServiceHostname struct { - Hostname string `json:"hostname,omitempty"` +type ResolveDNSTarget struct { + Hostname string `json:"hostname,omitempty"` + VCluster string `json:"vcluster"` + Service string `json:"service"` + Namespace string `json:"namespace"` + Mode string `json:"mode"` } type NetworkingAdvanced struct { @@ -316,24 +307,25 @@ type Plugin struct { type Plugins struct { // Name is the name of the init-container and NOT the plugin name Name string `json:"name,omitempty"` - // Command is the command that should be used for the init container - Command []string `json:"command,omitempty"` - // Args are the arguments that should be used for the init container - Args []string `json:"args,omitempty"` // Image is the container image that should be used for the plugin Image string `json:"image,omitempty"` // ImagePullPolicy is the pull policy to use for the container image ImagePullPolicy string `json:"imagePullPolicy,omitempty"` // Config is the plugin config to use. This can be arbitrary config used for the plugin. Config map[string]interface{} `json:"config,omitempty"` + // RBAC holds additional rbac configuration for the plugin + RBAC PluginsRBAC `json:"rbac,omitempty"` + + // Command is the command that should be used for the init container + Command []string `json:"command,omitempty"` + // Args are the arguments that should be used for the init container + Args []string `json:"args,omitempty"` // SecurityContext is the container security context used for the init container SecurityContext map[string]interface{} `json:"securityContext,omitempty"` // Resources are the container resources used for the init container Resources map[string]interface{} `json:"resources,omitempty"` // VolumeMounts are extra volume mounts for the init container VolumeMounts []interface{} `json:"volumeMounts,omitempty"` - // RBAC holds additional rbac configuration for the plugin - RBAC PluginsRBAC `json:"rbac,omitempty"` } type PluginsRBAC struct { @@ -375,23 +367,39 @@ type ControlPlane struct { CoreDNS CoreDNS `json:"coredns,omitempty"` // Proxy defines options for the vCluster control plane proxy that is used to do authentication and intercept requests. Proxy ControlPlaneProxy `json:"proxy,omitempty"` - // Service defines options for the vCluster service deployed by helm. - Service ControlPlaneService `json:"service,omitempty"` + // HostPathMapper defines if vCluster should rewrite host paths. + HostPathMapper HostPathMapper `json:"hostPathMapper,omitempty"` // Ingress defines options for the vCluster ingress deployed by helm. Ingress ControlPlaneIngress `json:"ingress,omitempty"` + // Service defines options for the vCluster service deployed by helm. + Service ControlPlaneService `json:"service,omitempty"` // StatefulSet defines options for the vCluster statefulSet deployed by helm. StatefulSet ControlPlaneStatefulSet `json:"statefulSet,omitempty"` - // HostPathMapper defines if vCluster should rewrite host paths. - HostPathMapper HostPathMapper `json:"hostPathMapper,omitempty"` - // Observability defines if a service monitor should get deployed by helm. - Observability ControlPlaneObservability `json:"observability,omitempty"` + // ServiceMonitor can be used to automatically create a service monitor for vCluster deployment itself. + ServiceMonitor ServiceMonitor `json:"serviceMonitor,omitempty"` // Advanced holds additional configuration for the vCluster control plane. Advanced ControlPlaneAdvanced `json:"advanced,omitempty"` } type ControlPlaneStatefulSet struct { + // HighAvailability holds options related to high availability. + HighAvailability ControlPlaneHighAvailability `json:"highAvailability,omitempty"` + // Resources are the resource requests and limits for the statefulSet container. + Resources Resources `json:"resources,omitempty"` + // Scheduling holds options related to scheduling. + Scheduling ControlPlaneScheduling `json:"scheduling,omitempty"` + // Security defines pod or container security context. + Security ControlPlaneSecurity `json:"security,omitempty"` + // Probes enables or disables the main container probes. + Probes ControlPlaneProbes `json:"probes,omitempty"` + // Persistence defines options around persistence for the statefulSet. + Persistence ControlPlanePersistence `json:"persistence,omitempty"` + LabelsAndAnnotations `json:",inline"` + // Pods are additional labels or annotations for the statefulSet pod. + Pods LabelsAndAnnotations `json:"pods,omitempty"` + // Image is the image for the controlPlane statefulSet container Image Image `json:"image,omitempty"` // ImagePullPolicy is the policy how to pull the image. @@ -404,21 +412,6 @@ type ControlPlaneStatefulSet struct { Args []string `json:"args,omitempty"` // Env are additional environment variables for the statefulSet container. Env []map[string]interface{} `json:"env,omitempty"` - // Pods are additional labels or annotations for the statefulSet pod. - Pods LabelsAndAnnotations `json:"pods,omitempty"` - - // Probes enables or disables the main container probes. - Probes ControlPlaneProbes `json:"probes,omitempty"` - // Security defines pod or container security context. - Security ControlPlaneSecurity `json:"security,omitempty"` - // Persistence defines options around persistence for the statefulSet. - Persistence ControlPlanePersistence `json:"persistence,omitempty"` - // Scheduling holds options related to scheduling. - Scheduling ControlPlaneScheduling `json:"scheduling,omitempty"` - // HighAvailability holds options related to high availability. - HighAvailability ControlPlaneHighAvailability `json:"highAvailability,omitempty"` - // Resources are the resource requests and limits for the statefulSet container. - Resources Resources `json:"resources,omitempty"` } type Distro struct { @@ -469,10 +462,10 @@ type DistroK0s struct { type DistroCommon struct { // Env are extra environment variables to use for the main container. Env []map[string]interface{} `json:"env,omitempty"` - // SecurityContext can be used for the distro init container - SecurityContext map[string]interface{} `json:"securityContext,omitempty"` // Resources are the resources for the distro init container Resources map[string]interface{} `json:"resources,omitempty"` + // SecurityContext can be used for the distro init container + SecurityContext map[string]interface{} `json:"securityContext,omitempty"` } type DistroContainer struct { @@ -664,12 +657,13 @@ type ControlPlaneProxy struct { type ControlPlaneService struct { // Enabled defines if the control plane service should be enabled Enabled bool `json:"enabled,omitempty"` + + // Spec allows you to configure extra service options. + Spec map[string]interface{} `json:"spec,omitempty"` // KubeletNodePort is the node port where the fake kubelet is exposed. Defaults to 0. KubeletNodePort int `json:"kubeletNodePort,omitempty"` // HTTPSNodePort is the node port where https is exposed. Defaults to 0. HTTPSNodePort int `json:"httpsNodePort,omitempty"` - // Spec allows you to configure extra service options. - Spec map[string]interface{} `json:"spec,omitempty"` LabelsAndAnnotations `json:",inline"` } @@ -850,14 +844,14 @@ type LabelsAndAnnotations struct { } type Policies struct { + // NetworkPolicy specifies network policy options. + NetworkPolicy NetworkPolicy `json:"networkPolicy,omitempty"` // PodSecurityStandard that can be enforced can be one of: empty (""), baseline, restricted or privileged PodSecurityStandard string `json:"podSecurityStandard,omitempty"` // ResourceQuota specifies resource quota options. ResourceQuota ResourceQuota `json:"resourceQuota,omitempty"` // LimitRange specifies limit range options. LimitRange LimitRange `json:"limitRange,omitempty"` - // NetworkPolicy specifies network policy options. - NetworkPolicy NetworkPolicy `json:"networkPolicy,omitempty"` // CentralAdmission defines what validating or mutating webhooks should be enforced within the vCluster. CentralAdmission CentralAdmission `json:"centralAdmission,omitempty" product:"pro"` } @@ -950,42 +944,46 @@ type RBAC struct { type RBACClusterRole struct { // Disabled defines if the cluster role should be disabled. Otherwise, its automatically determined if vCluster requires a cluster role. Disabled bool `json:"disabled,omitempty"` - // OverwriteRules will overwrite the cluster role rules completely. - OverwriteRules []map[string]interface{} `json:"overwriteRules,omitempty"` // ExtraRules will add rules to the cluster role. ExtraRules []map[string]interface{} `json:"extraRules,omitempty"` + // OverwriteRules will overwrite the cluster role rules completely. + OverwriteRules []map[string]interface{} `json:"overwriteRules,omitempty"` } type RBACRole struct { // Enabled Enabled bool `json:"enabled,omitempty"` - // OverwriteRules will overwrite the role rules completely. - OverwriteRules []map[string]interface{} `json:"overwriteRules,omitempty"` // ExtraRules will add rules to the role. ExtraRules []map[string]interface{} `json:"extraRules,omitempty"` + // OverwriteRules will overwrite the role rules completely. + OverwriteRules []map[string]interface{} `json:"overwriteRules,omitempty"` } type Telemetry struct { // Disabled specifies that the telemetry for vCluster control plane should be disabled. Disabled bool `json:"disabled,omitempty"` InstanceCreator string `json:"instanceCreator,omitempty"` + MachineID string `json:"machineID,omitempty"` PlatformUserID string `json:"platformUserID,omitempty"` PlatformInstanceID string `json:"platformInstanceID,omitempty"` - MachineID string `json:"machineID,omitempty"` } type Experimental struct { - // IsolatedControlPlane is a feature to run the vCluster control plane in a different Kubernetes cluster than the workloads themselves. - IsolatedControlPlane ExperimentalIsolatedControlPlane `json:"isolatedControlPlane,omitempty"` + // Deploy allows you to configure manifests and helm charts to deploy within the vCluster. + Deploy ExperimentalDeploy `json:"deploy,omitempty"` + // SyncSettings are advanced settings for the syncer controller. SyncSettings ExperimentalSyncSettings `json:"syncSettings,omitempty"` + // GenericSync holds options to generically sync resources from vCluster to host. GenericSync ExperimentalGenericSync `json:"genericSync,omitempty"` - // Deploy allows you to configure manifests and helm charts to deploy within the vCluster. - Deploy ExperimentalDeploy `json:"deploy,omitempty"` + // MultiNamespaceMode tells vCluster to sync to multiple namespaces instead of a single one. This will map each vCluster namespace to a single namespace in the host cluster. MultiNamespaceMode ExperimentalMultiNamespaceMode `json:"multiNamespaceMode,omitempty"` + // IsolatedControlPlane is a feature to run the vCluster control plane in a different Kubernetes cluster than the workloads themselves. + IsolatedControlPlane ExperimentalIsolatedControlPlane `json:"isolatedControlPlane,omitempty"` + // VirtualClusterKubeConfig allows you to override distro specifics and specify where vCluster will find the required certificates and vCluster config. VirtualClusterKubeConfig VirtualClusterKubeConfig `json:"virtualClusterKubeConfig,omitempty"` } @@ -1064,6 +1062,9 @@ type ExperimentalDeployHelmChart struct { } type Platform struct { + // APIKey defines how vCluster can find the api key used for the platform. + APIKey PlatformAPIKey `json:"apiKey,omitempty"` + // Name is the name of the vCluster instance in the vCluster platform Name string `json:"name,omitempty"` @@ -1072,9 +1073,6 @@ type Platform struct { // Project is the project within the platform where the vCluster should connect to. Project string `json:"project,omitempty"` - - // APIKey defines how vCluster can find the api key used for the platform. - APIKey PlatformAPIKey `json:"apiKey,omitempty"` } type PlatformOwner struct { diff --git a/pkg/config/config.go b/pkg/config/config.go index d01121f1c..2a1a319b5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,7 +16,7 @@ type VirtualClusterConfig struct { config.Config `json:",inline"` // Name is the name of the vCluster - Name string `json:"name,omitempty"` + Name string `json:"name"` // ServiceName is the name of the service of the vCluster ServiceName string `json:"serviceName,omitempty"` @@ -137,7 +137,7 @@ func (v VirtualClusterConfig) LegacyOptions() (*LegacyVirtualClusterOptions, err TargetNamespace: v.TargetNamespace, ServiceName: v.ServiceName, SetOwner: v.Experimental.SyncSettings.SetOwner, - SyncAllNodes: v.Sync.FromHost.Nodes.SyncAll, + SyncAllNodes: v.Sync.FromHost.Nodes.Selector.All, EnableScheduler: v.ControlPlane.Advanced.VirtualScheduler.Enabled, DisableFakeKubelets: !v.Networking.Advanced.ProxyKubelets.ByIP && !v.Networking.Advanced.ProxyKubelets.ByHostname, FakeKubeletIPs: v.Networking.Advanced.ProxyKubelets.ByIP, @@ -164,9 +164,9 @@ func (v VirtualClusterConfig) LegacyOptions() (*LegacyVirtualClusterOptions, err MultiNamespaceMode: v.Experimental.MultiNamespaceMode.Enabled, SyncAllSecrets: v.Sync.ToHost.Secrets.All, SyncAllConfigMaps: v.Sync.ToHost.ConfigMaps.All, - ProxyMetricsServer: v.Observability.Metrics.Proxy.Nodes.Enabled || v.Observability.Metrics.Proxy.Pods.Enabled, + ProxyMetricsServer: v.Observability.Metrics.Proxy.Nodes || v.Observability.Metrics.Proxy.Pods, - DeprecatedSyncNodeChanges: v.Sync.FromHost.Nodes.SyncLabelsTaints, + DeprecatedSyncNodeChanges: v.Sync.FromHost.Nodes.SyncBackChanges, }, nil } diff --git a/pkg/config/parse.go b/pkg/config/parse.go index bbed80980..393e5608a 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -47,7 +47,7 @@ func ParseConfig(path, name string, setValues []string) (*VirtualClusterConfig, } // validate config - err = ValidateConfig(retConfig) + err = ValidateConfigAndSetDefaults(retConfig) if err != nil { return nil, err } diff --git a/pkg/config/validation.go b/pkg/config/validation.go index fd7df4871..6974e3353 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -5,11 +5,11 @@ import ( "errors" "fmt" "net/url" + "slices" "github.com/ghodss/yaml" "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/util/toleration" - "github.com/samber/lo" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" ) @@ -23,7 +23,7 @@ var ( verbs = []string{"get", "list", "create", "update", "patch", "watch", "delete", "deletecollection"} ) -func ValidateConfig(config *VirtualClusterConfig) error { +func ValidateConfigAndSetDefaults(config *VirtualClusterConfig) error { // check the value of pod security standard if config.Policies.PodSecurityStandard != "" && !allowedPodSecurityStandards[config.Policies.PodSecurityStandard] { return fmt.Errorf("invalid argument enforce-pod-security-standard=%s, must be one of: privileged, baseline, restricted", config.Policies.PodSecurityStandard) @@ -38,8 +38,8 @@ func ValidateConfig(config *VirtualClusterConfig) error { } // check if enable scheduler works correctly - if config.ControlPlane.Advanced.VirtualScheduler.Enabled && !config.Sync.FromHost.Nodes.SyncAll && len(config.Sync.FromHost.Nodes.Selector.Labels) == 0 { - config.Sync.FromHost.Nodes.SyncAll = true + if config.ControlPlane.Advanced.VirtualScheduler.Enabled && !config.Sync.FromHost.Nodes.Selector.All && len(config.Sync.FromHost.Nodes.Selector.Labels) == 0 { + config.Sync.FromHost.Nodes.Selector.All = true } // enable additional controllers required for scheduling with storage @@ -99,7 +99,7 @@ func validateDistro(config *VirtualClusterConfig) error { } if enabledDistros > 1 { - return fmt.Errorf("please only enable a single distribution") + return fmt.Errorf("only one distribution can be enabled") } return nil } @@ -228,7 +228,7 @@ func validatePatch(patch *config.Patch) error { } func validateVerb(verb string) error { - if !lo.Contains(verbs, verb) { + if !slices.Contains(verbs, verb) { return fmt.Errorf("invalid verb \"%s\"; expected on of %q", verb, verbs) } diff --git a/pkg/controllers/resources/nodes/syncer.go b/pkg/controllers/resources/nodes/syncer.go index 175c3b13c..0f59f8d29 100644 --- a/pkg/controllers/resources/nodes/syncer.go +++ b/pkg/controllers/resources/nodes/syncer.go @@ -32,7 +32,7 @@ import ( func NewSyncer(ctx *synccontext.RegisterContext, nodeServiceProvider nodeservice.Provider) (syncertypes.Object, error) { var nodeSelector labels.Selector - if ctx.Config.Sync.FromHost.Nodes.SyncAll { + if ctx.Config.Sync.FromHost.Nodes.Selector.All { nodeSelector = labels.Everything() } else if len(ctx.Config.Sync.FromHost.Nodes.Selector.Labels) > 0 { nodeSelector = labels.Set(ctx.Config.Sync.FromHost.Nodes.Selector.Labels).AsSelector() diff --git a/pkg/controllers/resources/nodes/syncer_test.go b/pkg/controllers/resources/nodes/syncer_test.go index 4179b7155..a97d8054a 100644 --- a/pkg/controllers/resources/nodes/syncer_test.go +++ b/pkg/controllers/resources/nodes/syncer_test.go @@ -520,7 +520,7 @@ func TestSync(t *testing.T) { }, Sync: func(ctx *synccontext.RegisterContext) { ctx.Config.Networking.Advanced.ProxyKubelets.ByIP = false - ctx.Config.Sync.FromHost.Nodes.SyncAll = true + ctx.Config.Sync.FromHost.Nodes.Selector.All = true ctx.Config.Sync.FromHost.Nodes.ClearImageStatus = true syncCtx, syncerSvc := newFakeSyncer(t, ctx) _, err := syncerSvc.Sync(syncCtx, baseNode, baseNode) @@ -572,7 +572,7 @@ func TestSync(t *testing.T) { }, Sync: func(ctx *synccontext.RegisterContext) { ctx.Config.Networking.Advanced.ProxyKubelets.ByIP = false - ctx.Config.Sync.FromHost.Nodes.SyncAll = true + ctx.Config.Sync.FromHost.Nodes.Selector.All = true syncCtx, syncerSvc := newFakeSyncer(t, ctx) _, err := syncerSvc.Sync(syncCtx, baseNode, baseNode) assert.NilError(t, err) diff --git a/pkg/metricsapiservice/register.go b/pkg/metricsapiservice/register.go index e5827c5a6..cc50e1a5f 100644 --- a/pkg/metricsapiservice/register.go +++ b/pkg/metricsapiservice/register.go @@ -137,7 +137,7 @@ func createOperation(ctrlCtx *config.ControllerContext) wait.ConditionWithContex func RegisterOrDeregisterAPIService(ctx *config.ControllerContext) error { // check if the api service should get created exists := checkExistingAPIService(ctx.Context, ctx.VirtualManager.GetClient()) - if ctx.Config.Observability.Metrics.Proxy.Nodes.Enabled || ctx.Config.Observability.Metrics.Proxy.Pods.Enabled { + if ctx.Config.Observability.Metrics.Proxy.Nodes || ctx.Config.Observability.Metrics.Proxy.Pods { return applyOperation(ctx.Context, createOperation(ctx)) } else if exists { return applyOperation(ctx.Context, deleteOperation(ctx)) diff --git a/pkg/server/server.go b/pkg/server/server.go index 20f52aff1..860462e11 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -204,7 +204,7 @@ func NewServer(ctx *config.ControllerContext, requestHeaderCaFile, clientCaFile h = filters.WithMetricsProxy(h, localConfig, cachedVirtualClient) // is metrics proxy enabled? - if ctx.Config.Observability.Metrics.Proxy.Nodes.Enabled || ctx.Config.Observability.Metrics.Proxy.Pods.Enabled { + if ctx.Config.Observability.Metrics.Proxy.Nodes || ctx.Config.Observability.Metrics.Proxy.Pods { h = filters.WithMetricsServerProxy( h, ctx.Config.TargetNamespace, @@ -216,7 +216,7 @@ func NewServer(ctx *config.ControllerContext, requestHeaderCaFile, clientCaFile ) } - if ctx.Config.Sync.FromHost.Nodes.Enabled && ctx.Config.Sync.FromHost.Nodes.SyncLabelsTaints { + if ctx.Config.Sync.FromHost.Nodes.Enabled && ctx.Config.Sync.FromHost.Nodes.SyncBackChanges { h = filters.WithNodeChanges(ctx.Context, h, uncachedLocalClient, uncachedVirtualClient, virtualConfig) } h = filters.WithFakeKubelet(h, localConfig, cachedVirtualClient) diff --git a/test/e2e_metrics_proxy/values.yaml b/test/e2e_metrics_proxy/values.yaml index 45f70f5d2..811f44fa5 100644 --- a/test/e2e_metrics_proxy/values.yaml +++ b/test/e2e_metrics_proxy/values.yaml @@ -1,7 +1,5 @@ observability: metrics: proxy: - nodes: - enabled: true - pods: - enabled: true + nodes: true + pods: true diff --git a/test/e2e_node/values.yaml b/test/e2e_node/values.yaml index 429896f2c..d0ba4bf48 100644 --- a/test/e2e_node/values.yaml +++ b/test/e2e_node/values.yaml @@ -8,7 +8,8 @@ sync: nodes: enabled: true # Either syncAllNodes or nodeSelector is required - syncAll: true + selector: + all: true experimental: deploy: diff --git a/test/e2e_scheduler/values.yaml b/test/e2e_scheduler/values.yaml index f132f237e..70ad522b9 100644 --- a/test/e2e_scheduler/values.yaml +++ b/test/e2e_scheduler/values.yaml @@ -3,7 +3,8 @@ sync: nodes: enabled: true # Either syncAllNodes or nodeSelector is required - syncAll: true + selector: + all: true controlPlane: advanced: From f1e599e2e8e441f59924bca654362f3d74afe2cb Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Fri, 15 Mar 2024 10:52:22 +0100 Subject: [PATCH 5/5] refactor: changes for vCluster.pro --- chart/templates/_coredns.tpl | 6 +- chart/templates/_rbac.tpl | 3 +- chart/templates/clusterrole.yaml | 5 -- cmd/vcluster/cmd/start.go | 2 +- cmd/vclusterctl/cmd/connect.go | 2 +- config/config.go | 104 +++++++++++++++++++++--- pkg/config/coredns_plugin_validation.go | 83 +++++++++++++++++++ pkg/config/validation.go | 88 ++++++++++++++++++++ pkg/pro/license.go | 3 +- pkg/server/cert/syncer.go | 16 ++-- pkg/server/server.go | 4 +- pkg/server/types/types.go | 8 +- pkg/setup/controllers.go | 2 +- pkg/util/translate/lb_service_name.go | 9 -- 14 files changed, 284 insertions(+), 51 deletions(-) create mode 100644 pkg/config/coredns_plugin_validation.go delete mode 100644 pkg/util/translate/lb_service_name.go diff --git a/chart/templates/_coredns.tpl b/chart/templates/_coredns.tpl index 41101bf1c..dd7518ec9 100644 --- a/chart/templates/_coredns.tpl +++ b/chart/templates/_coredns.tpl @@ -16,14 +16,14 @@ Corefile: |- kubeconfig /data/vcluster/admin.conf {{- end }} pods insecure - {{- if or .Values.networking.advanced.fallbackHostCluster (and .Values.controlPlane.coredns.embedded .Values.networking.resolveServices) }} + {{- if or .Values.networking.advanced.fallbackHostCluster (and .Values.controlPlane.coredns.embedded .Values.networking.resolveDNS) }} fallthrough cluster.local in-addr.arpa ip6.arpa {{- else }} fallthrough in-addr.arpa ip6.arpa {{- end }} } - {{- if and .Values.controlPlane.coredns.embedded .Values.networking.resolveServices }} - vcluster {{ toYaml .Values.networking.resolveServices | b64enc }} + {{- if and .Values.controlPlane.coredns.embedded .Values.networking.resolveDNS }} + vcluster {{- end }} hosts /etc/NodeHosts { ttl 60 diff --git a/chart/templates/_rbac.tpl b/chart/templates/_rbac.tpl index edfe0eea6..50de084ee 100644 --- a/chart/templates/_rbac.tpl +++ b/chart/templates/_rbac.tpl @@ -27,8 +27,7 @@ .Values.sync.fromHost.storageClasses.enabled .Values.sync.fromHost.nodes.enabled .Values.observability.metrics.proxy.nodes - .Values.experimental.multiNamespaceMode.enabled - .Values.networking.resolveServices -}} + .Values.experimental.multiNamespaceMode.enabled -}} {{- true -}} {{- end -}} {{- end -}} diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index 358ac708c..5ba424991 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -26,11 +26,6 @@ rules: resources: ["pods", "nodes", "nodes/status", "nodes/metrics", "nodes/stats", "nodes/proxy"] verbs: ["get", "watch", "list"] {{- end }} - {{- if .Values.networking.resolveServices }} - - apiGroups: [""] - resources: [ "pods"] - verbs: ["get", "watch", "list"] - {{- end }} {{- if and .Values.sync.fromHost.nodes.enabled .Values.sync.fromHost.nodes.syncBackChanges }} - apiGroups: [""] resources: ["nodes", "nodes/status"] diff --git a/cmd/vcluster/cmd/start.go b/cmd/vcluster/cmd/start.go index 2f566422b..382cf7a3d 100644 --- a/cmd/vcluster/cmd/start.go +++ b/cmd/vcluster/cmd/start.go @@ -97,7 +97,7 @@ func ExecuteStart(ctx context.Context, vConfig *config.VirtualClusterConfig) err telemetry.Collector.Init(controlPlaneConfig, controlPlaneNamespace, vConfig) // initialize feature gate from environment - err = pro.LicenseInit(ctx, controlPlaneConfig, controlPlaneNamespace, vConfig.Platform.APIKey.Value, vConfig.Platform.APIKey.SecretRef.Namespace, vConfig.Platform.APIKey.SecretRef.Name) + err = pro.LicenseInit(ctx, controlPlaneConfig, controlPlaneNamespace, vConfig) if err != nil { return fmt.Errorf("init license: %w", err) } diff --git a/cmd/vclusterctl/cmd/connect.go b/cmd/vclusterctl/cmd/connect.go index 9b1fa4f7c..c770d4918 100644 --- a/cmd/vclusterctl/cmd/connect.go +++ b/cmd/vclusterctl/cmd/connect.go @@ -596,7 +596,7 @@ func (cmd *ConnectCmd) setServerIfExposed(ctx context.Context, vClusterName stri err := wait.PollUntilContextTimeout(ctx, time.Second*2, time.Minute*5, true, func(ctx context.Context) (done bool, err error) { // first check for load balancer service, look for the other service if it's not there loadBalancerMissing := false - service, err := cmd.kubeClient.CoreV1().Services(cmd.Namespace).Get(ctx, translate.GetLoadBalancerSVCName(vClusterName), metav1.GetOptions{}) + service, err := cmd.kubeClient.CoreV1().Services(cmd.Namespace).Get(ctx, vClusterName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { loadBalancerMissing = true diff --git a/config/config.go b/config/config.go index 933c165af..bb1a0d548 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,8 @@ package config -import "regexp" +import ( + "regexp" +) // Config is the vCluster config. This struct describes valid helm values for vCluster as well as configuration used by the vCluster binary itself. type Config struct { @@ -252,18 +254,23 @@ type ServiceMapping struct { } type ResolveDNS struct { - Hostname string `json:"hostname"` - Service string `json:"service"` - Namespace string `json:"namespace"` - Target ResolveDNSTarget `json:"target,omitempty"` + Hostname string `json:"hostname"` + Service string `json:"service"` + Namespace string `json:"namespace"` + + Target ResolveDNSTarget `json:"target,omitempty"` } type ResolveDNSTarget struct { - Hostname string `json:"hostname,omitempty"` - VCluster string `json:"vcluster"` - Service string `json:"service"` - Namespace string `json:"namespace"` - Mode string `json:"mode"` + Hostname string `json:"hostname,omitempty"` + IP string `json:"ip,omitempty"` + + // HostService to target, format is hostNamespace/hostService + HostService string `json:"hostService,omitempty"` + // HostNamespace to target + HostNamespace string `json:"hostNamespace,omitempty"` + // VClusterService format is hostNamespace/vClusterName/vClusterNamespace/vClusterService + VClusterService string `json:"vClusterService,omitempty"` } type NetworkingAdvanced struct { @@ -986,6 +993,9 @@ type Experimental struct { // VirtualClusterKubeConfig allows you to override distro specifics and specify where vCluster will find the required certificates and vCluster config. VirtualClusterKubeConfig VirtualClusterKubeConfig `json:"virtualClusterKubeConfig,omitempty"` + + // DenyProxyRequests denies certain requests in the vCluster proxy. + DenyProxyRequests []DenyRule `json:"denyProxyRequests,omitempty" pro:"true"` } type ExperimentalMultiNamespaceMode struct { @@ -1259,3 +1269,77 @@ type PatchSync struct { Secret *bool `json:"secret,omitempty" yaml:"secret,omitempty"` ConfigMap *bool `json:"configmap,omitempty" yaml:"configmap,omitempty"` } + +type DenyRule struct { + // The name of the check. + // +optional + Name string `json:"name,omitempty"` + + // Namespace describe a list of namespaces that will be affected by the check. + // An empty list means that all namespaces will be affected. + // In case of ClusterScoped rules, only the Namespace resource is affected. + // +optional + Namespaces []string `json:"namespaces,omitempty"` + + // Rules describes on which verbs and on what resources/subresources the webhook is enforced. + // The webhook is enforced if it matches any Rule. + // The version of the request must match the rule version exactly. Equivalent matching is not supported. + // +optional + Rules []RuleWithVerbs `json:"rules,omitempty"` + + // ExcludedUsers describe a list of users for which the checks will be skipped. + // Impersonation attempts on these users will still be subjected to the checks. + // +optional + ExcludedUsers []string `json:"excludedUsers,omitempty"` +} + +type RuleWithVerbs struct { + // APIGroups is the API groups the resources belong to. '*' is all groups. + // If '*' is present, the length of the slice must be one. + // Required. + // +listType=atomic + APIGroups []string `json:"apiGroups,omitempty" protobuf:"bytes,1,rep,name=apiGroups"` + + // APIVersions is the API versions the resources belong to. '*' is all versions. + // If '*' is present, the length of the slice must be one. + // Required. + // +listType=atomic + APIVersions []string `json:"apiVersions,omitempty" protobuf:"bytes,2,rep,name=apiVersions"` + + // Resources is a list of resources this rule applies to. + // + // For example: + // 'pods' means pods. + // 'pods/log' means the log subresource of pods. + // '*' means all resources, but not subresources. + // 'pods/*' means all subresources of pods. + // '*/scale' means all scale subresources. + // '*/*' means all resources and their subresources. + // + // If wildcard is present, the validation rule will ensure resources do not + // overlap with each other. + // + // Depending on the enclosing object, subresources might not be allowed. + // Required. + // +listType=atomic + Resources []string `json:"resources,omitempty" protobuf:"bytes,3,rep,name=resources"` + + // scope specifies the scope of this rule. + // Valid values are "Cluster", "Namespaced", and "*" + // "Cluster" means that only cluster-scoped resources will match this rule. + // Namespace API objects are cluster-scoped. + // "Namespaced" means that only namespaced resources will match this rule. + // "*" means that there are no scope restrictions. + // Subresources match the scope of their parent resource. + // Default is "*". + // + // +optional + Scope *string `json:"scope,omitempty" protobuf:"bytes,4,rep,name=scope"` + + // Verb is the kube verb associated with the request for API requests, not the http verb. This includes things like list and watch. + // For non-resource requests, this is the lowercase http verb. + // If '*' is present, the length of the slice must be one. + // Required. + // +listType=atomic + Verbs []string `json:"operations,omitempty"` +} diff --git a/pkg/config/coredns_plugin_validation.go b/pkg/config/coredns_plugin_validation.go new file mode 100644 index 000000000..fa2e5fe48 --- /dev/null +++ b/pkg/config/coredns_plugin_validation.go @@ -0,0 +1,83 @@ +package config + +import ( + "fmt" + "strings" + + vclusterconfig "github.com/loft-sh/vcluster/config" +) + +func validateMappings(resolveDNS []vclusterconfig.ResolveDNS) error { + for i, mapping := range resolveDNS { + // parse service format + options := 0 + if mapping.Service != "" { + options++ + if strings.Count(mapping.Service, "/") != 1 { + return fmt.Errorf("error validating networking.resolveDNS[%d].service: expected format namespace/name, but got %s", i, mapping.Service) + } + } + if mapping.Hostname != "" { + options++ + } + if mapping.Namespace != "" { + if mapping.Target.HostNamespace == "" { + return fmt.Errorf("error validating networking.resolveDNS[%d].namespace: when using namespace, target.hostNamespace is required", i) + } + + options++ + } else if mapping.Target.HostNamespace != "" { + return fmt.Errorf("error validating networking.resolveDNS[%d]: when using target.hostNamespace, .namespace is required", i) + } + + if options == 0 { + return fmt.Errorf("at least one option required for networking.resolveDNS[%d]", i) + } else if options > 1 { + return fmt.Errorf("only a single option allowed for networking.resolveDNS[%d]", i) + } + + // validate targets + err := validateTarget(mapping.Target) + if err != nil { + return fmt.Errorf("error validating networking.resolveDNS[%d].to", i) + } + } + + return nil +} + +func validateTarget(target vclusterconfig.ResolveDNSTarget) error { + options := 0 + if target.Hostname != "" { + options++ + } + if target.IP != "" { + options++ + } + if target.HostNamespace != "" { + options++ + } + if target.HostService != "" { + options++ + + // check if service is defined with the namespace/name format + if strings.Count(target.HostService, "/") != 1 { + return fmt.Errorf("expected namespace/name format for .to.service, but got %s", target.HostService) + } + } + if target.VClusterService != "" { + options++ + + // check if vcluster service is defined with namespace/name format + if strings.Count(target.VClusterService, "/") != 3 { + return fmt.Errorf("expected hostNamespace/vClusterName/vClusterNamespace/vClusterService format for .to.vClusterService, but got %s", target.VClusterService) + } + } + if options == 0 { + return fmt.Errorf("at least one option required for .to") + } else if options > 1 { + return fmt.Errorf("only a single option allowed for .to") + } + + return nil +} diff --git a/pkg/config/validation.go b/pkg/config/validation.go index 6974e3353..2ee69f3f8 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -11,6 +11,7 @@ import ( "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/util/toleration" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/api/validation" ) var allowedPodSecurityStandards = map[string]bool{ @@ -80,6 +81,20 @@ func ValidateConfigAndSetDefaults(config *VirtualClusterConfig) error { return err } + // check deny proxy requests + for _, c := range config.Experimental.DenyProxyRequests { + err := validateCheck(c) + if err != nil { + return err + } + } + + // check resolve dns + err = validateMappings(config.Networking.ResolveDNS) + if err != nil { + return err + } + return nil } @@ -345,3 +360,76 @@ func validateWebhookClientCfg(clientCfg admissionregistrationv1.WebhookClientCon return nil } + +func validateCheck(check config.DenyRule) error { + for _, ns := range check.Namespaces { + errors := validation.ValidateNamespaceName(ns, false) + if len(errors) != 0 { + return fmt.Errorf("invalid Namespaces in %q check: %v", check.Name, errors) + } + } + var err error + for _, r := range check.Rules { + err = validateWildcardOrExact(r.Verbs, "create", "get", "update", "patch", "delete") + if err != nil { + return fmt.Errorf("invalid Verb defined in the %q check: %w", check.Name, err) + } + + err = validateWildcardOrAny(r.APIGroups) + if err != nil { + return fmt.Errorf("invalid APIGroup defined in the %q check: %w", check.Name, err) + } + + err = validateWildcardOrAny(r.APIVersions) + if err != nil { + return fmt.Errorf("invalid APIVersion defined in the %q check: %w", check.Name, err) + } + + if r.Scope != nil { + switch *r.Scope { + case string(admissionregistrationv1.ClusterScope): + case string(admissionregistrationv1.NamespacedScope): + case string(admissionregistrationv1.AllScopes): + default: + return fmt.Errorf("invalid Scope defined in the %q check: %q", check.Name, *r.Scope) + } + } + } + return nil +} + +func validateWildcardOrExact(values []string, validValues ...string) error { + if len(values) == 1 && values[0] == "*" { + return nil + } + for _, val := range values { + if val == "*" { + return fmt.Errorf("when wildcard(*) is used, it must be the only value in the list") + } + + // empty list of validValues means any value is valid + valid := len(validValues) == 0 + for _, v := range validValues { + if val == v { + valid = true + break + } + } + if !valid { + return fmt.Errorf("invalid value %q", val) + } + } + return nil +} + +func validateWildcardOrAny(values []string) error { + if len(values) == 1 && values[0] == "*" { + return nil + } + for _, val := range values { + if val == "*" { + return fmt.Errorf("when wildcard(*) is used, it must be the only value in the list") + } + } + return nil +} diff --git a/pkg/pro/license.go b/pkg/pro/license.go index 67a9f0be1..5d845efac 100644 --- a/pkg/pro/license.go +++ b/pkg/pro/license.go @@ -3,11 +3,12 @@ package pro import ( "context" + "github.com/loft-sh/vcluster/pkg/config" "k8s.io/client-go/rest" ) // LicenseInit is used to initialize the license reader -var LicenseInit = func(_ context.Context, _ *rest.Config, _, _, _, _ string) error { +var LicenseInit = func(_ context.Context, _ *rest.Config, _ string, _ *config.VirtualClusterConfig) error { return nil } diff --git a/pkg/server/cert/syncer.go b/pkg/server/cert/syncer.go index b2dd2387a..5c74967f9 100644 --- a/pkg/server/cert/syncer.go +++ b/pkg/server/cert/syncer.go @@ -110,7 +110,7 @@ func (s *syncer) getSANs(ctx context.Context) ([]string, error) { } // get load balancer ip - // currently, the load balancer service is named -lb, but the syncer image might run in legacy environments + // currently, the load balancer service is named , but the syncer image might run in legacy environments // where the load balancer service is the same service, the service is only updated if the helm template is rerun, // so we are leaving this snippet in, but the load balancer ip will be read via the lbSVC var below for _, ing := range svc.Status.LoadBalancer.Ingress { @@ -142,18 +142,17 @@ func (s *syncer) getSANs(ctx context.Context) ([]string, error) { } // get cluster ip of load balancer service - lbSVCName := translate.GetLoadBalancerSVCName(s.serviceName) lbSVC := &corev1.Service{} err = s.currentNamespaceCient.Get(ctx, types.NamespacedName{ Namespace: s.currentNamespace, - Name: lbSVCName, + Name: s.serviceName, }, lbSVC) // proceed only if load balancer service exists if !kerrors.IsNotFound(err) { if err != nil { - return nil, fmt.Errorf("error getting vcluster load balancer service %s/%s: %w", s.currentNamespace, lbSVCName, err) + return nil, fmt.Errorf("error getting vcluster load balancer service %s/%s: %w", s.currentNamespace, s.serviceName, err) } else if lbSVC.Spec.ClusterIP == "" { - return nil, fmt.Errorf("target service %s/%s is missing a clusterIP", s.currentNamespace, lbSVCName) + return nil, fmt.Errorf("target service %s/%s is missing a clusterIP", s.currentNamespace, s.serviceName) } for _, ing := range lbSVC.Status.LoadBalancer.Ingress { @@ -165,9 +164,10 @@ func (s *syncer) getSANs(ctx context.Context) ([]string, error) { } } // append hostnames for load balancer service - retSANs = append(retSANs, - lbSVCName, - lbSVCName+"."+s.currentNamespace, "*."+translate.VClusterName+"."+s.currentNamespace+"."+constants.NodeSuffix, + retSANs = append( + retSANs, + s.serviceName, + s.serviceName+"."+s.currentNamespace, "*."+translate.VClusterName+"."+s.currentNamespace+"."+constants.NodeSuffix, ) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 860462e11..34d3981d3 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -227,9 +227,7 @@ func NewServer(ctx *config.ControllerContext, requestHeaderCaFile, clientCaFile } for _, f := range ctx.AdditionalServerFilters { - h = f(h, servertypes.FilterOptions{ - LocalScheme: uncachedLocalClient.Scheme(), - }) + h = f(h) } for _, handler := range ctx.ExtraHandlers { diff --git a/pkg/server/types/types.go b/pkg/server/types/types.go index 9ef8b7651..9dd4e2220 100644 --- a/pkg/server/types/types.go +++ b/pkg/server/types/types.go @@ -2,15 +2,9 @@ package types import ( "net/http" - - "k8s.io/apimachinery/pkg/runtime" ) -type FilterOptions struct { - LocalScheme *runtime.Scheme -} - -type Filter func(http.Handler, FilterOptions) http.Handler +type Filter func(http.Handler) http.Handler type OriginalUserKeyType int diff --git a/pkg/setup/controllers.go b/pkg/setup/controllers.go index 0f666dd64..c24d38b5d 100644 --- a/pkg/setup/controllers.go +++ b/pkg/setup/controllers.go @@ -107,7 +107,7 @@ func StartControllers( controllerContext, types.NamespacedName{ Namespace: controlPlaneNamespace, - Name: controlPlaneService + "-lb", + Name: controlPlaneService, }, controlPlaneClient, types.NamespacedName{ diff --git a/pkg/util/translate/lb_service_name.go b/pkg/util/translate/lb_service_name.go deleted file mode 100644 index f0dc1e52b..000000000 --- a/pkg/util/translate/lb_service_name.go +++ /dev/null @@ -1,9 +0,0 @@ -package translate - -import "fmt" - -// GetLoadBalancerSVCName retrieves the service name if service name is set to type LoadBalancer. -// A separate service is created in this case so as to expose only the apiserver and not the kubelet port -func GetLoadBalancerSVCName(serviceName string) string { - return fmt.Sprintf("%s-lb", serviceName) -}