From a0c73f92f3e95b7ffdcdff5857188390457e83f6 Mon Sep 17 00:00:00 2001
From: Atanas Dinov <atanas.dinov@suse.com>
Date: Wed, 26 Jun 2024 21:16:04 +0300
Subject: [PATCH] Apply manifests in RKE2 templates

Signed-off-by: Atanas Dinov <atanas.dinov@suse.com>
---
 pkg/combustion/kubernetes_test.go             |  4 +--
 .../rke2-multi-node-installer.sh.tpl          | 34 ++++++++++++++++---
 .../rke2-single-node-installer.sh.tpl         | 34 ++++++++++++++++---
 3 files changed, 62 insertions(+), 10 deletions(-)

diff --git a/pkg/combustion/kubernetes_test.go b/pkg/combustion/kubernetes_test.go
index 6984cf3b..a0d19288 100644
--- a/pkg/combustion/kubernetes_test.go
+++ b/pkg/combustion/kubernetes_test.go
@@ -782,8 +782,8 @@ func TestConfigureKubernetes_SuccessfulRKE2ServerWithManifests(t *testing.T) {
 	assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=$ARTEFACTS_DIR/kubernetes/install")
 	assert.Contains(t, contents, "sh $ARTEFACTS_DIR/kubernetes/install-k8s.sh")
 	assert.Contains(t, contents, "systemctl enable rke2-server.service")
-	assert.Contains(t, contents, "mkdir -p /var/lib/rancher/rke2/server/manifests/")
-	assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/manifests/* /var/lib/rancher/rke2/server/manifests/")
+	assert.Contains(t, contents, "mkdir -p /opt/k8s/manifests")
+	assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/manifests/* /opt/k8s/manifests/")
 	assert.Contains(t, contents, "cp $ARTEFACTS_DIR/kubernetes/registries.yaml /etc/rancher/rke2/registries.yaml")
 
 	// Config file assertions
diff --git a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
index 0a4da7b7..e1462ce6 100644
--- a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
+++ b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
@@ -27,19 +27,45 @@ mount /var
 mkdir -p /var/lib/rancher/rke2/agent/images/
 cp {{ .imagesPath }}/* /var/lib/rancher/rke2/agent/images/
 
+umount /var
+
 CONFIGFILE={{ .configFilePath }}/$NODETYPE.yaml
 
 if [ "$HOSTNAME" = {{ .initialiser }} ]; then
     CONFIGFILE={{ .configFilePath }}/{{ .initialiserConfigFile }}
 
     {{- if .manifestsPath }}
-    mkdir -p /var/lib/rancher/rke2/server/manifests/
-    cp {{ .manifestsPath }}/* /var/lib/rancher/rke2/server/manifests/
+    mkdir -p /opt/k8s/manifests
+    cp {{ .manifestsPath }}/* /opt/k8s/manifests/
+
+    cat <<- EOF > /etc/systemd/system/kubernetes-resources-install.service
+    [Unit]
+    Description=Kubernetes Resources Install
+    Requires=rke2-server.service
+    After=rke2-server.service
+    ConditionPathExists=/var/lib/rancher/rke2/bin/kubectl
+    ConditionPathExists=/etc/rancher/rke2/rke2.yaml
+
+    [Install]
+    WantedBy=multi-user.target
+
+    [Service]
+    Type=oneshot
+    Restart=on-failure
+    RestartSec=30
+    # Copy kubectl in order to avoid SELinux permission issues
+    ExecStartPre=cp /var/lib/rancher/rke2/bin/kubectl /opt/k8s/kubectl
+    ExecStart=/opt/k8s/kubectl apply -f /opt/k8s/manifests --kubeconfig /etc/rancher/rke2/rke2.yaml
+    # Disable the service and clean up
+    ExecStartPost=/bin/sh -c "systemctl disable kubernetes-resources-install.service"
+    ExecStartPost=rm -f /etc/systemd/system/kubernetes-resources-install.service
+    ExecStartPost=rm -rf /opt/k8s
+    EOF
+
+    systemctl enable kubernetes-resources-install.service
     {{- end }}
 fi
 
-umount /var
-
 {{- if .apiHost }}
 echo "{{ .apiVIP }} {{ .apiHost }}" >> /etc/hosts
 {{- end }}
diff --git a/pkg/combustion/templates/rke2-single-node-installer.sh.tpl b/pkg/combustion/templates/rke2-single-node-installer.sh.tpl
index b884a99f..af84adca 100644
--- a/pkg/combustion/templates/rke2-single-node-installer.sh.tpl
+++ b/pkg/combustion/templates/rke2-single-node-installer.sh.tpl
@@ -6,13 +6,39 @@ mount /var
 mkdir -p /var/lib/rancher/rke2/agent/images/
 cp {{ .imagesPath }}/* /var/lib/rancher/rke2/agent/images/
 
+umount /var
+
 {{- if .manifestsPath }}
-mkdir -p /var/lib/rancher/rke2/server/manifests/
-cp {{ .manifestsPath }}/* /var/lib/rancher/rke2/server/manifests/
+mkdir -p /opt/k8s/manifests
+cp {{ .manifestsPath }}/* /opt/k8s/manifests/
+
+cat <<- EOF > /etc/systemd/system/kubernetes-resources-install.service
+[Unit]
+Description=Kubernetes Resources Install
+Requires=rke2-server.service
+After=rke2-server.service
+ConditionPathExists=/var/lib/rancher/rke2/bin/kubectl
+ConditionPathExists=/etc/rancher/rke2/rke2.yaml
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=oneshot
+Restart=on-failure
+RestartSec=30
+# Copy kubectl in order to avoid SELinux permission issues
+ExecStartPre=cp /var/lib/rancher/rke2/bin/kubectl /opt/k8s/kubectl
+ExecStart=/opt/k8s/kubectl apply -f /opt/k8s/manifests --kubeconfig /etc/rancher/rke2/rke2.yaml
+# Disable the service and clean up
+ExecStartPost=/bin/sh -c "systemctl disable kubernetes-resources-install.service"
+ExecStartPost=rm -f /etc/systemd/system/kubernetes-resources-install.service
+ExecStartPost=rm -rf /opt/k8s
+EOF
+
+systemctl enable kubernetes-resources-install.service
 {{- end }}
 
-umount /var
-
 {{- if and .apiVIP .apiHost }}
 echo "{{ .apiVIP }} {{ .apiHost }}" >> /etc/hosts
 {{- end }}