From 24911abd381e139281afa3788438a4f9732a9088 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Mon, 23 Sep 2024 17:29:17 +0900
Subject: [PATCH] gh-87 Fix for k8s cicd failing due to cri-o dependencies

---
 .../node_scripts/common.sh                    | 81 +++++++++++--------
 .../node_scripts/master.sh                    | 10 +--
 .../node_scripts/worker.sh                    | 10 +--
 .../yaml/kubeadm-config.yaml                  |  5 ++
 4 files changed, 62 insertions(+), 44 deletions(-)

diff --git a/cicd/k8s-flannel-incluster-multus/node_scripts/common.sh b/cicd/k8s-flannel-incluster-multus/node_scripts/common.sh
index c01ad688f..ded6b8b5d 100755
--- a/cicd/k8s-flannel-incluster-multus/node_scripts/common.sh
+++ b/cicd/k8s-flannel-incluster-multus/node_scripts/common.sh
@@ -15,6 +15,7 @@ cat <<EOF | sudo tee /etc/systemd/resolved.conf.d/dns_servers.conf
 DNS=${DNS_SERVERS}
 EOF
 
+VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
 sudo systemctl restart systemd-resolved
 
 # disable swap
@@ -23,18 +24,56 @@ sudo swapoff -a
 # keeps the swap off during reboot
 (crontab -l 2>/dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true
 sudo apt-get update -y
-# Install CRI-O Runtime
 
-VERSION="$(echo ${KUBERNETES_VERSION} | grep -oE '[0-9]+\.[0-9]+')"
-CRIO_VERSION=1.27
-# Create the .conf file to load the modules at bootup
-cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
-overlay
-br_netfilter
+sudo apt install -y snapd
+#sudo snap install docker
+
+sudo apt install -y docker.io
+sudo usermod -aG docker vagrant
+
+# cri-docker Install
+VER=$(curl -s https://api.github.com/repos/Mirantis/cri-dockerd/releases/latest|grep tag_name | cut -d '"' -f 4|sed 's/v//g')
+echo $VER
+wget -q https://github.com/Mirantis/cri-dockerd/releases/download/v${VER}/cri-dockerd-${VER}.amd64.tgz --retry-connrefused  --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 --continue
+tar xvf cri-dockerd-${VER}.amd64.tgz
+sudo mv cri-dockerd/cri-dockerd /usr/local/bin/
+
+# cri-docker Version Check
+cri-dockerd --version
+
+wget -q https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.service --retry-connrefused  --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 --continue
+wget -q https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.socket --retry-connrefused  --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 --continue
+sudo mv cri-docker.socket cri-docker.service /etc/systemd/system/
+sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
+
+sudo systemctl daemon-reload
+sudo systemctl enable cri-docker.service
+sudo systemctl enable --now cri-docker.socket
+
+# cri-docker Active Check
+sudo systemctl restart docker && sudo systemctl restart cri-docker
+sudo systemctl status cri-docker.socket --no-pager 
+
+# Docker cgroup Change Require to Systemd
+sudo mkdir -p /etc/docker || true
+cat <<EOF | sudo tee /etc/docker/daemon.json
+{
+  "exec-opts": ["native.cgroupdriver=systemd"],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2"
+}
 EOF
 
-sudo modprobe overlay
-sudo modprobe br_netfilter
+sudo systemctl restart docker && sudo systemctl restart cri-docker
+sudo docker info | grep Cgroup
+
+# Kernel Forwarding 
+cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
+br_netfilter
+EOF
 
 # Install ipvs related modules
 sudo modprobe ip_vs
@@ -54,30 +93,8 @@ EOF
 
 sudo sysctl --system
 
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-deb https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
-EOF
-cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
-deb http://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /
-EOF
-
-curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
-curl -L https://provo-mirror.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
-
-sudo apt-get update
-sudo apt-get install cri-o cri-o-runc -y
-
-cat >> /etc/default/crio << EOF
-${ENVIRONMENT}
-EOF
-sudo systemctl daemon-reload
-sudo systemctl enable crio --now
-
-echo "CRI runtime installed successfully"
-
-sudo apt-get update
 sudo apt-get install -y apt-transport-https ca-certificates curl gpg
-curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key | sudo gpg --no-tty --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
 echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'$VERSION'/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
 
 sudo apt-get update -y
diff --git a/cicd/k8s-flannel-incluster-multus/node_scripts/master.sh b/cicd/k8s-flannel-incluster-multus/node_scripts/master.sh
index 43d431e53..c971e90a3 100755
--- a/cicd/k8s-flannel-incluster-multus/node_scripts/master.sh
+++ b/cicd/k8s-flannel-incluster-multus/node_scripts/master.sh
@@ -6,12 +6,6 @@ set -euxo pipefail
 
 NODENAME=$(hostname -s)
 
-sudo sed -i 's#10.85.0.0/16#10.244.0.0/24#g' /etc/cni/net.d/100-crio-bridge.conflist
-
-sudo kubeadm config images pull
-
-echo "Preflight Check Passed: Downloaded All Required Images"
-
 #sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap
 sudo kubeadm init --ignore-preflight-errors Swap --config /vagrant/yaml/kubeadm-config.yaml
 
@@ -35,7 +29,9 @@ cp -i /etc/kubernetes/admin.conf $config_path/config
 touch $config_path/join.sh
 chmod +x $config_path/join.sh
 
-kubeadm token create --print-join-command > $config_path/join.sh
+join_cmd=`kubeadm token create --print-join-command`
+echo $join_cmd "--cri-socket /var/run/cri-dockerd.sock" > $config_path/join.sh
+
 
 sudo -i -u vagrant bash << EOF
 whoami
diff --git a/cicd/k8s-flannel-incluster-multus/node_scripts/worker.sh b/cicd/k8s-flannel-incluster-multus/node_scripts/worker.sh
index 0fd5eaee9..08d1ebd35 100755
--- a/cicd/k8s-flannel-incluster-multus/node_scripts/worker.sh
+++ b/cicd/k8s-flannel-incluster-multus/node_scripts/worker.sh
@@ -4,11 +4,11 @@
 
 set -euxo pipefail
 
-if [[ $(hostname -s) == "worker1" ]]; then
-  sudo sed -i 's#10.85.0.0/16#10.244.1.0/24#g' /etc/cni/net.d/100-crio-bridge.conflist
-else
-  sudo sed -i 's#10.85.0.0/16#10.244.2.0/24#g' /etc/cni/net.d/100-crio-bridge.conflist
-fi
+#if [[ $(hostname -s) == "worker1" ]]; then
+#  sudo sed -i 's#10.85.0.0/16#10.244.1.0/24#g' /etc/cni/net.d/100-crio-bridge.conflist
+#else
+#  sudo sed -i 's#10.85.0.0/16#10.244.2.0/24#g' /etc/cni/net.d/100-crio-bridge.conflist
+#fi
 
 config_path="/vagrant/configs"
 
diff --git a/cicd/k8s-flannel-incluster-multus/yaml/kubeadm-config.yaml b/cicd/k8s-flannel-incluster-multus/yaml/kubeadm-config.yaml
index e8de10b86..fccd933c0 100644
--- a/cicd/k8s-flannel-incluster-multus/yaml/kubeadm-config.yaml
+++ b/cicd/k8s-flannel-incluster-multus/yaml/kubeadm-config.yaml
@@ -16,6 +16,7 @@ nodeRegistration:
   taints: null
   kubeletExtraArgs:
     node-ip: 192.168.80.250
+  criSocket: unix:///var/run/cri-dockerd.sock
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 certificatesDir: /etc/kubernetes/pki
@@ -68,3 +69,7 @@ metricsBindAddress: 127.0.0.1:10249
 nodePortAddresses: null
 oomScoreAdj: -999
 portRange: ""
+---
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: systemd