kubelet base environment fixes

images/base/files/etc/default/kubelet

@@ -1 +1 @@
-KUBELET_EXTRA_ARGS=--fail-swap-on=false
+KUBELET_EXTRA_ARGS=--runtime-cgroups=/system.slice/containerd.service

images/base/files/etc/systemd/system/containerd.service

@@ -3,27 +3,26 @@
 [Unit]
 Description=containerd container runtime
 Documentation=https://containerd.io
-After=network.target
+After=network.target local-fs.target
 # disable rate limiting
 StartLimitIntervalSec=0
 
 [Service]
 ExecStartPre=-/sbin/modprobe overlay
 ExecStart=/usr/local/bin/containerd
-Restart=always
-RestartSec=1
 
+Type=notify
 Delegate=yes
 KillMode=process
 Restart=always
+RestartSec=1
 # Having non-zero Limit*s causes performance problems due to accounting overhead
 # in the kernel. We recommend using cgroups to do container-local accounting.
 LimitNPROC=infinity
 LimitCORE=infinity
 LimitNOFILE=infinity
-# Comment TasksMax if your systemd version does not supports it.
-# Only systemd 226 and above support this version.
 TasksMax=infinity
+OOMScoreAdjust=-999
 
 [Install]
 WantedBy=multi-user.target

images/base/files/etc/systemd/system/kubelet.service

@@ -19,6 +19,7 @@ RestartSec=1s
 CPUAccounting=true
 MemoryAccounting=true
 Slice=kubelet.slice
+KillMode=process
 
 [Install]
 WantedBy=multi-user.target

images/base/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf

@@ -15,4 +15,4 @@ ExecStartPre=/bin/sh -euc "if [ -f /sys/fs/cgroup/cgroup.controllers ]; then cre
 # This eventually leads to kubelet failing to start, see: https://github.com/kubernetes-sigs/kind/issues/2323
 ExecStartPre=/bin/sh -euc "if [ ! -f /sys/fs/cgroup/cgroup.controllers ] && [ ! -d /sys/fs/cgroup/systemd/kubelet ]; then mkdir -p /sys/fs/cgroup/systemd/kubelet; fi"
 ExecStart=
-ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS --cgroup-root=/kubelet
+ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS

images/base/files/usr/local/bin/entrypoint

@@ -181,8 +181,8 @@ fix_mount() {
   mount --make-rshared /
 }
 
-# helper used by fix_cgroup
-mount_kubelet_cgroup_root() {
+# helper used by mount_kubelet_cgroup_root
+mount_kubelet_cgroup_root_subsystem() {
   local cgroup_root=$1
   local subsystem=$2
   if [ -z "${cgroup_root}" ]; then
@@ -201,6 +201,22 @@ mount_kubelet_cgroup_root() {
   mount --bind "${subsystem}/${cgroup_root}" "${subsystem}/${cgroup_root}"
 }
 
+# helper used by fix_cgroup
+mount_kubelet_cgroup_root() {
+  local cgroup_subsystems=$1
+  echo "${cgroup_subsystems}" |
+  while IFS= read -r subsystem; do
+    mount_kubelet_cgroup_root_subsystem /kubelet "${subsystem}"
+    mount_kubelet_cgroup_root_subsystem /kubelet.slice "${subsystem}"
+  done
+  # workaround for hosts not running systemd
+  # we only do this for kubelet.slice because it's not relevant when not using
+  # the systemd cgroup driver
+  if [[ ! "${cgroup_subsystems}" = */sys/fs/cgroup/systemd* ]]; then
+    mount_kubelet_cgroup_root_subsystem /kubelet.slice /sys/fs/cgroup/systemd
+  fi
+}
+
 fix_cgroup() {
   if [[ -f "/sys/fs/cgroup/cgroup.controllers" ]]; then
     echo 'INFO: detected cgroup v2'
@@ -226,8 +242,17 @@ fix_cgroup() {
   # current process. this tells us what cgroup-path the container is in.
   local current_cgroup
   current_cgroup=$(grep -E '^[^:]*:([^:]*,)?cpu(,[^,:]*)?:.*' /proc/self/cgroup | cut -d: -f3)
+  local cgroup_subsystems
+  cgroup_subsystems=$(findmnt -lun -o source,target -t cgroup | grep -F "${current_cgroup}" | awk '{print $2}')
   if [ "$current_cgroup" = "/" ]; then
-    echo "INFO: cgroupns detected, no need to fix cgroups"
+    echo "INFO: detected cgroupns"
+    # kubelet will try to manage cgroups / pods that are not owned by it when
+    # "nesting" clusters, unless we instruct it to use a different cgroup root.
+    # We do this, and when doing so we must fixup this alternative root
+    # currently this is hardcoded to be /kubelet
+    # under systemd cgroup driver, kubelet appends .slice
+    mount --make-rprivate /sys/fs/cgroup
+    mount_kubelet_cgroup_root "${cgroup_subsystems}"
     return
   fi
 
@@ -310,17 +335,7 @@ fix_cgroup() {
   # currently this is hardcoded to be /kubelet
   # under systemd cgroup driver, kubelet appends .slice
   mount --make-rprivate /sys/fs/cgroup
-  echo "${cgroup_subsystems}" |
-  while IFS= read -r subsystem; do
-    mount_kubelet_cgroup_root /kubelet "${subsystem}"
-    mount_kubelet_cgroup_root /kubelet.slice "${subsystem}"
-  done
-  # workaround for hosts not running systemd
-  # we only do this for kubelet.slice because it's not relevant when not using
-  # the systemd cgroup driver
-  if [[ ! "${cgroup_subsystems}" = */sys/fs/cgroup/systemd* ]]; then
-    mount_kubelet_cgroup_root /kubelet.slice /sys/fs/cgroup/systemd
-  fi
+  mount_kubelet_cgroup_root "${cgroup_subsystems}"
 }
 
 fix_machine_id() {