From 2796ee8ec3b5d86ed03a4cd7f24e2daf60d32146 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 28 Aug 2022 09:49:28 -0700 Subject: [PATCH] Configure Graceful Node Shutdown and lengthen max inhibitor delay * Configure Kubelet Graceful Node Shutdown to detect system shutdown events and stop running containers gracefully when possible * Allow up to 30s for critical pods to gracefully shutdown * Allow up to 15s for regular pods to gracefully shutdown * Node will be marked as NotReady promptly, instead of having to wait for health checks * Kubelet uses systemd inhibitor locks to delay shutdown for a limited number of seconds * Raise the default max inhibitor time from 5s to 45s Verify systemd inhibitor locks are present: ``` sudo systemd-inhibit --list WHO UID USER PID COMM WHAT WHY MODE kubelet 0 root 4581 kubelet shutdown Kubelet needs time to handle node shutdown delay ``` Tail journal logs and then shutdown a node via systemctl reboot or via the cloud console to watch container shutdown Rel: * https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/ * https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/ * https://github.com/kubernetes/kubernetes/issues/107043 * https://github.com/coreos/fedora-coreos-tracker/issues/821 * https://www.freedesktop.org/software/systemd/man/systemd-inhibit.html * https://github.com/kubernetes/kubernetes/blob/release-1.24/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go * https://github.com/godbus/dbus/blob/master/conn.go --- CHANGES.md | 5 +++++ aws/fedora-coreos/kubernetes/butane/controller.yaml | 7 +++++++ aws/fedora-coreos/kubernetes/workers/butane/worker.yaml | 7 +++++++ aws/flatcar-linux/kubernetes/butane/controller.yaml | 7 +++++++ aws/flatcar-linux/kubernetes/workers/butane/worker.yaml | 7 +++++++ azure/fedora-coreos/kubernetes/butane/controller.yaml | 7 +++++++ azure/fedora-coreos/kubernetes/workers/butane/worker.yaml | 7 +++++++ azure/flatcar-linux/kubernetes/butane/controller.yaml | 7 +++++++ azure/flatcar-linux/kubernetes/workers/butane/worker.yaml | 7 +++++++ bare-metal/fedora-coreos/kubernetes/butane/controller.yaml | 7 +++++++ bare-metal/fedora-coreos/kubernetes/butane/worker.yaml | 7 +++++++ bare-metal/flatcar-linux/kubernetes/butane/controller.yaml | 7 +++++++ bare-metal/flatcar-linux/kubernetes/butane/worker.yaml | 7 +++++++ .../fedora-coreos/kubernetes/butane/controller.yaml | 7 +++++++ digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml | 7 +++++++ .../flatcar-linux/kubernetes/butane/controller.yaml | 7 +++++++ digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml | 7 +++++++ .../fedora-coreos/kubernetes/butane/controller.yaml | 7 +++++++ .../fedora-coreos/kubernetes/workers/butane/worker.yaml | 7 +++++++ .../flatcar-linux/kubernetes/butane/controller.yaml | 7 +++++++ .../flatcar-linux/kubernetes/workers/butane/worker.yaml | 7 +++++++ 21 files changed, 145 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 33ce781e1..846c49009 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,11 @@ Notable changes between versions. * Kubernetes [v1.25.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.25.md#v1250) * Disable LocalStorageCapacityIsolationFSQuotaMonitoring feature gate ([#1220](https://github.com/poseidon/typhoon/pull/1220)) +* Configure Kubelet Graceful Node Shutdown + * Allow up to 30s for critical pods to gracefully shutdown on node shutdown + * Allow up to 15s for regular pods to gracefully shutdown on node shutdown + * Mark node NotReady promptly on node shutdown + * Lengthen systemd inhibitor lock max delay from 5s to 45s ### Fedora CoreOS diff --git a/aws/fedora-coreos/kubernetes/butane/controller.yaml b/aws/fedora-coreos/kubernetes/butane/controller.yaml index 484ac6f6a..60b57e12e 100644 --- a/aws/fedora-coreos/kubernetes/butane/controller.yaml +++ b/aws/fedora-coreos/kubernetes/butane/controller.yaml @@ -154,6 +154,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -194,6 +196,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml index 990747bd4..725032001 100644 --- a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -122,10 +122,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/aws/flatcar-linux/kubernetes/butane/controller.yaml b/aws/flatcar-linux/kubernetes/butane/controller.yaml index cd14a80d8..bf5eaeae9 100644 --- a/aws/flatcar-linux/kubernetes/butane/controller.yaml +++ b/aws/flatcar-linux/kubernetes/butane/controller.yaml @@ -153,6 +153,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -193,6 +195,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml b/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml index 3da2888ce..be3141c16 100644 --- a/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -121,10 +121,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/azure/fedora-coreos/kubernetes/butane/controller.yaml b/azure/fedora-coreos/kubernetes/butane/controller.yaml index cc7f255f4..a39d667c9 100644 --- a/azure/fedora-coreos/kubernetes/butane/controller.yaml +++ b/azure/fedora-coreos/kubernetes/butane/controller.yaml @@ -149,6 +149,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -189,6 +191,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml index 014d23a6e..0f0bdc4a9 100644 --- a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -117,10 +117,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/azure/flatcar-linux/kubernetes/butane/controller.yaml b/azure/flatcar-linux/kubernetes/butane/controller.yaml index 9b4b8c82d..0d4a704f1 100644 --- a/azure/flatcar-linux/kubernetes/butane/controller.yaml +++ b/azure/flatcar-linux/kubernetes/butane/controller.yaml @@ -149,6 +149,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -189,6 +191,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml b/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml index 1716853f8..d2f77d732 100644 --- a/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -117,10 +117,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml index 6eaca5677..091f7a356 100644 --- a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml @@ -159,6 +159,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -199,6 +201,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml b/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml index 4199c2065..c05283b40 100644 --- a/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml @@ -113,10 +113,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml b/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml index 49c6945a8..cdf8930d0 100644 --- a/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml +++ b/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml @@ -160,6 +160,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -200,6 +202,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml b/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml index 493f14989..de2ef1943 100644 --- a/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml +++ b/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml @@ -118,10 +118,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml index e9f60d47e..93a7744dd 100644 --- a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml @@ -156,6 +156,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -196,6 +198,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml index bcaeb4447..4b822b85a 100644 --- a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml @@ -122,10 +122,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml b/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml index ade785003..799a312b2 100644 --- a/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml +++ b/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml @@ -158,6 +158,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -198,6 +200,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml b/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml index f1ffaa5b0..e86daadb5 100644 --- a/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml +++ b/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml @@ -121,10 +121,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml index 3e063c7a7..f58587850 100644 --- a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml @@ -148,6 +148,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -188,6 +190,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml index 5f1b56591..19b294708 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -116,10 +116,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml b/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml index 93bc1c299..cc701bdde 100644 --- a/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml +++ b/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml @@ -148,6 +148,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -188,6 +190,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml b/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml index a35ef07bb..ac46de252 100644 --- a/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -116,10 +116,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: