diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 762dab98a..25feff8b2 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -47,6 +47,25 @@ type ControlPlaneVirtualIPSpec struct { // +kubebuilder:validation:Enum=KubeVIP // +kubebuilder:default=KubeVIP Provider string `json:"provider,omitempty"` + + // Configuration for the chosen control-plane virtual IP provider. + // +kubebuilder:validation:Optional + Configuration *ControlPlaneVirtualIPConfiguration `json:"configuration,omitempty"` +} + +type ControlPlaneVirtualIPConfiguration struct { + // The virtual IP on which the API server is serving. + // If left empty, the value from controlPlaneEndpoint.host will be used. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Format=ipv4 + Address string `json:"address,omitempty"` + + // The port on which the API server is serving. + // If left empty, the value from controlPlaneEndpoint.port will be used. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port,omitempty"` } // LocalObjectReference contains enough information to let you locate the diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml index bd3cd2fcd..9cbb9a7ed 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml @@ -598,6 +598,24 @@ spec: virtualIP: description: Configuration for the virtual IP provider. properties: + configuration: + description: Configuration for the chosen control-plane virtual IP provider. + properties: + address: + description: |- + The virtual IP on which the API server is serving. + If left empty, the value from controlPlaneEndpoint.host will be used. + format: ipv4 + type: string + port: + description: |- + The port on which the API server is serving. + If left empty, the value from controlPlaneEndpoint.port will be used. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object provider: default: KubeVIP description: Virtual IP provider to deploy. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 31ea04516..8dfb97213 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -550,7 +550,7 @@ func (in *ControlPlaneEndpointSpec) DeepCopyInto(out *ControlPlaneEndpointSpec) if in.VirtualIPSpec != nil { in, out := &in.VirtualIPSpec, &out.VirtualIPSpec *out = new(ControlPlaneVirtualIPSpec) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -564,9 +564,31 @@ func (in *ControlPlaneEndpointSpec) DeepCopy() *ControlPlaneEndpointSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlPlaneVirtualIPConfiguration) DeepCopyInto( + out *ControlPlaneVirtualIPConfiguration, +) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneVirtualIPConfiguration. +func (in *ControlPlaneVirtualIPConfiguration) DeepCopy() *ControlPlaneVirtualIPConfiguration { + if in == nil { + return nil + } + out := new(ControlPlaneVirtualIPConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlPlaneVirtualIPSpec) DeepCopyInto(out *ControlPlaneVirtualIPSpec) { *out = *in + if in.Configuration != nil { + in, out := &in.Configuration, &out.Configuration + *out = new(ControlPlaneVirtualIPConfiguration) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneVirtualIPSpec. diff --git a/charts/cluster-api-runtime-extensions-nutanix/addons/ccm/nutanix/values-template.yaml b/charts/cluster-api-runtime-extensions-nutanix/addons/ccm/nutanix/values-template.yaml index 707a1b4af..fe34d44b1 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/addons/ccm/nutanix/values-template.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/addons/ccm/nutanix/values-template.yaml @@ -4,9 +4,9 @@ prismCentralInsecure: {{ .PrismCentralInsecure }} {{- with .PrismCentralAdditionalTrustBundle }} prismCentralAdditionalTrustBundle: "{{ . }}" {{- end }} -{{- with .ControlPlaneEndpointHost }} -ignoredNodeIPs: [ {{ printf "%q" . }} ] - {{- end }} +{{- with .IPsToIgnore }} +ignoredNodeIPs: [ {{ joinQuoted . }} ] +{{- end }} # The Secret containing the credentials will be created by the handler. createSecret: false diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/virtual-ip/kube-vip/manifests/kube-vip-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/virtual-ip/kube-vip/manifests/kube-vip-configmap.yaml index f0be06cd5..3c210aa11 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/virtual-ip/kube-vip/manifests/kube-vip-configmap.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/virtual-ip/kube-vip/manifests/kube-vip-configmap.yaml @@ -22,7 +22,7 @@ data: - name: vip_arp value: "true" - name: port - value: '{{ `{{ .ControlPlaneEndpoint.Port }}` }}' + value: '{{ `{{ .Port }}` }}' - name: vip_nodename valueFrom: fieldRef: @@ -46,7 +46,7 @@ data: - name: vip_retryperiod value: "2" - name: address - value: '{{ `{{ .ControlPlaneEndpoint.Host }}` }}' + value: '{{ `{{ .Address }}` }}' - name: prometheus_server image: ghcr.io/kube-vip/kube-vip:v0.8.3 imagePullPolicy: IfNotPresent diff --git a/docs/content/customization/nutanix/control-plane-endpoint.md b/docs/content/customization/nutanix/control-plane-endpoint.md index f28c15441..4a5629265 100644 --- a/docs/content/customization/nutanix/control-plane-endpoint.md +++ b/docs/content/customization/nutanix/control-plane-endpoint.md @@ -2,7 +2,7 @@ title = "Control Plane Endpoint" +++ -Configure Control Plane Endpoint. Defines the host IP and port of the CAPX Kubernetes cluster. +Configure Control Plane Endpoint. Defines the host IP and port of the Nutanix Kubernetes cluster. ## Examples @@ -51,6 +51,93 @@ spec: name: kube-vip namespace: kube-system spec: + containers: + - name: kube-vip + args: + - manager + env: + - name: port + value: '6443' + - name: address + value: 'x.x.x.x' + ... + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0600" + postKubeadmCommands: + # Only added for clusters version >=v1.29.0 + - |- + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: ... + fi + preKubeadmCommands: + # Only added for clusters version >=v1.29.0 + - |- + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: ... + fi +``` + +### Set Control Plane Endpoint and a Different Virtual IP + +It is also possible to set a separate virtual IP to be used by kube-vip from the control plane endpoint. +This is useful in VPC setups or other instances +when you have an external floating IP already associated with the virtual IP. + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + nutanix: + controlPlaneEndpoint: + host: x.x.x.x + port: 6443 + virtualIP: + configuration: + address: y.y.y.y +``` + +Applying this configuration will result in the following value being set: + +- `NutanixCluster`: + +```yaml +spec: + template: + spec: + controlPlaneEndpoint: + host: x.x.x.x + port: 6443 +``` + +- `KubeadmControlPlaneTemplate` + +```yaml + spec: + kubeadmConfigSpec: + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-vip + namespace: kube-system + spec: + containers: + - name: kube-vip + args: + - manager + env: + - name: port + value: '6443' + - name: address + value: 'y.y.y.y' ... owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml diff --git a/hack/addons/update-kube-vip-manifests.sh b/hack/addons/update-kube-vip-manifests.sh index 29dd54eff..45116c697 100755 --- a/hack/addons/update-kube-vip-manifests.sh +++ b/hack/addons/update-kube-vip-manifests.sh @@ -33,8 +33,8 @@ docker container run --rm ghcr.io/kube-vip/kube-vip:"${KUBE_VIP_VERSION}" \ gojq --yaml-input --yaml-output \ 'del(.metadata.creationTimestamp, .status) | .spec.containers[].imagePullPolicy |= "IfNotPresent" | - (.spec.containers[0].env[] | select(.name == "port").value) |= "{{ `{{ .ControlPlaneEndpoint.Port }}` }}" | - (.spec.containers[0].env[] | select(.name == "address").value) |= "{{ `{{ .ControlPlaneEndpoint.Host }}` }}" + (.spec.containers[0].env[] | select(.name == "port").value) |= "{{ `{{ .Port }}` }}" | + (.spec.containers[0].env[] | select(.name == "address").value) |= "{{ `{{ .Address }}` }}" ' >"${ASSETS_DIR}/${FILE_NAME}" kubectl create configmap "{{ .Values.hooks.virtualIP.kubeVip.defaultTemplateConfigMap.name }}" --dry-run=client --output yaml \ diff --git a/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go b/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go index a6e1ebbf3..4c9b12fc9 100644 --- a/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go +++ b/pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "strings" "text/template" "github.com/go-logr/logr" @@ -140,7 +141,15 @@ func templateValuesFunc( nutanixConfig *v1alpha1.NutanixSpec, ) func(*clusterv1.Cluster, string) (string, error) { return func(_ *clusterv1.Cluster, valuesTemplate string) (string, error) { - helmValuesTemplate, err := template.New("").Parse(valuesTemplate) + joinQuoted := template.FuncMap{ + "joinQuoted": func(items []string) string { + for i, item := range items { + items[i] = fmt.Sprintf("%q", item) + } + return strings.Join(items, ", ") + }, + } + helmValuesTemplate, err := template.New("").Funcs(joinQuoted).Parse(valuesTemplate) if err != nil { return "", fmt.Errorf("failed to parse Helm values template: %w", err) } @@ -150,7 +159,7 @@ func templateValuesFunc( PrismCentralPort uint16 PrismCentralInsecure bool PrismCentralAdditionalTrustBundle string - ControlPlaneEndpointHost string + IPsToIgnore []string } address, port, err := nutanixConfig.PrismCentralEndpoint.ParseURL() @@ -162,7 +171,7 @@ func templateValuesFunc( PrismCentralPort: port, PrismCentralInsecure: nutanixConfig.PrismCentralEndpoint.Insecure, PrismCentralAdditionalTrustBundle: nutanixConfig.PrismCentralEndpoint.AdditionalTrustBundle, - ControlPlaneEndpointHost: nutanixConfig.ControlPlaneEndpoint.Host, + IPsToIgnore: ipsToIgnore(nutanixConfig), } var b bytes.Buffer @@ -174,3 +183,17 @@ func templateValuesFunc( return b.String(), nil } } + +func ipsToIgnore(nutanixConfig *v1alpha1.NutanixSpec) []string { + toIgnore := []string{nutanixConfig.ControlPlaneEndpoint.Host} + // Also ignore the virtual IP if it is set. + if nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec != nil && + nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec.Configuration != nil && + nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec.Configuration.Address != "" { + toIgnore = append( + toIgnore, + nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec.Configuration.Address, + ) + } + return toIgnore +} diff --git a/pkg/handlers/generic/lifecycle/ccm/nutanix/handler_test.go b/pkg/handlers/generic/lifecycle/ccm/nutanix/handler_test.go index 3059f7c43..2daccbb1c 100644 --- a/pkg/handlers/generic/lifecycle/ccm/nutanix/handler_test.go +++ b/pkg/handlers/generic/lifecycle/ccm/nutanix/handler_test.go @@ -39,6 +39,16 @@ prismCentralPort: 9440 prismCentralInsecure: true ignoredNodeIPs: [ "1.2.3.4" ] +# The Secret containing the credentials will be created by the handler. +createSecret: false +secretName: nutanix-ccm-credentials +` + + expectedWithVirtualIPSet = `prismCentralEndPoint: prism-central.nutanix.com +prismCentralPort: 9440 +prismCentralInsecure: true +ignoredNodeIPs: [ "1.2.3.4", "5.6.7.8" ] + # The Secret containing the credentials will be created by the handler. createSecret: false secretName: nutanix-ccm-credentials @@ -127,6 +137,41 @@ func Test_templateValues(t *testing.T) { in: valuesTemplate, expected: expectedWithoutAdditionalTrustBundle, }, + { + name: "With VirtualIP Set", + clusterConfig: &apivariables.ClusterConfigSpec{ + Addons: &apivariables.Addons{ + GenericAddons: v1alpha1.GenericAddons{ + CCM: &v1alpha1.CCM{ + Credentials: &v1alpha1.CCMCredentials{ + SecretRef: v1alpha1.LocalObjectReference{ + Name: "creds", + }, + }, + }, + }, + }, + Nutanix: &v1alpha1.NutanixSpec{ + PrismCentralEndpoint: v1alpha1.NutanixPrismCentralEndpointSpec{ + URL: fmt.Sprintf( + "https://prism-central.nutanix.com:%d", + v1alpha1.DefaultPrismCentralPort, + ), + Insecure: true, + }, + ControlPlaneEndpoint: v1alpha1.ControlPlaneEndpointSpec{ + Host: "1.2.3.4", + VirtualIPSpec: &v1alpha1.ControlPlaneVirtualIPSpec{ + Configuration: &v1alpha1.ControlPlaneVirtualIPConfiguration{ + Address: "5.6.7.8", + }, + }, + }, + }, + }, + in: valuesTemplate, + expected: expectedWithVirtualIPSet, + }, } for idx := range tests { tt := tests[idx] diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/inject_test.go b/pkg/handlers/generic/mutation/controlplanevirtualip/inject_test.go index ff242a1be..21adcd764 100644 --- a/pkg/handlers/generic/mutation/controlplanevirtualip/inject_test.go +++ b/pkg/handlers/generic/mutation/controlplanevirtualip/inject_test.go @@ -264,7 +264,7 @@ spec: - name: vip_arp value: "true" - name: address - value: "{{ .ControlPlaneEndpoint.Host }}" + value: "{{ .Address }}" - name: port - value: "{{ .ControlPlaneEndpoint.Port }}" + value: "{{ .Port }}" ` diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip_test.go b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip_test.go index 312d06afa..a2fef1967 100644 --- a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip_test.go +++ b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip_test.go @@ -105,6 +105,79 @@ func Test_GenerateFilesAndCommands(t *testing.T) { }, }, }, + { + name: "should return templated data with both IP and port from virtual IP configuration overrides", + controlPlaneEndpointSpec: v1alpha1.ControlPlaneEndpointSpec{ + Host: "10.20.100.10", + Port: 6443, + VirtualIPSpec: &v1alpha1.ControlPlaneVirtualIPSpec{ + Configuration: &v1alpha1.ControlPlaneVirtualIPConfiguration{ + Address: "172.20.100.10", + Port: 8443, + }, + }, + }, + configMap: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-kube-vip-template", + Namespace: "default", + }, + Data: map[string]string{ + "data": validKubeVIPTemplate, + }, + }, + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Version: "v1.28.0", + }, + }, + }, + expectedFiles: []bootstrapv1.File{ + { + Content: expectedKubeVIPPodWithOverrides, + Owner: kubeVIPFileOwner, + Path: kubeVIPFilePath, + Permissions: kubeVIPFilePermissions, + }, + }, + }, + { + name: "should return templated data with IP from virtual IP configuration overrides", + controlPlaneEndpointSpec: v1alpha1.ControlPlaneEndpointSpec{ + Host: "10.20.100.10", + Port: 8443, + VirtualIPSpec: &v1alpha1.ControlPlaneVirtualIPSpec{ + Configuration: &v1alpha1.ControlPlaneVirtualIPConfiguration{ + Address: "172.20.100.10", + }, + }, + }, + configMap: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-kube-vip-template", + Namespace: "default", + }, + Data: map[string]string{ + "data": validKubeVIPTemplate, + }, + }, + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Version: "v1.28.0", + }, + }, + }, + expectedFiles: []bootstrapv1.File{ + { + Content: expectedKubeVIPPodWithOverrides, + Owner: kubeVIPFileOwner, + Path: kubeVIPFilePath, + Permissions: kubeVIPFilePermissions, + }, + }, + }, } for idx := range tests { @@ -227,9 +300,9 @@ spec: - name: vip_arp value: "true" - name: address - value: "{{ .ControlPlaneEndpoint.Host }}" + value: "{{ .Address }}" - name: port - value: "{{ .ControlPlaneEndpoint.Port }}" + value: "{{ .Port }}" ` expectedKubeVIPPod = ` @@ -253,4 +326,26 @@ spec: - name: port value: "6443" ` + + expectedKubeVIPPodWithOverrides = ` +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system +spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v1.1.1 + imagePullPolicy: IfNotPresent + args: + - manager + env: + - name: vip_arp + value: "true" + - name: address + value: "172.20.100.10" + - name: port + value: "8443" +` ) diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/providers.go b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/providers.go index 1a8564326..16cf993f2 100644 --- a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/providers.go +++ b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/providers.go @@ -5,6 +5,7 @@ package providers import ( "bytes" + "cmp" "context" "fmt" "text/template" @@ -35,11 +36,20 @@ func templateValues( } type input struct { - ControlPlaneEndpoint v1alpha1.ControlPlaneEndpointSpec + Address string + Port int32 } + // If specified, use the virtual IP address and/or port, + // otherwise fall back to the control plane endpoint host and port. + var virtualIPConfig v1alpha1.ControlPlaneVirtualIPConfiguration + if controlPlaneEndpoint.VirtualIPSpec != nil && + controlPlaneEndpoint.VirtualIPSpec.Configuration != nil { + virtualIPConfig = *controlPlaneEndpoint.VirtualIPSpec.Configuration + } templateInput := input{ - ControlPlaneEndpoint: controlPlaneEndpoint, + Address: cmp.Or(virtualIPConfig.Address, controlPlaneEndpoint.Host), + Port: cmp.Or(virtualIPConfig.Port, controlPlaneEndpoint.Port), } var b bytes.Buffer