[Release-1.25] - Fix rootless node password location

[dereknola]

Backport fix for Fix rootless node password location
Original Issue: #3636

• Fix rootless node password location #7887

[aganesh-suse]

Validated using commit id a9b8c87 in release-1.25

Docs:
https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cpu-cpuset-and-io-delegation
https://docs.k3s.io/advanced#advanced-rootless-configuration

Environment Details

Commit Validated: a9b8c87
Version Reproduced: v1.25.11+k3s1

Infrastructure

Cloud

Node(s) CPU architecture, OS, and version:

$ cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.1 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.1 LTS (Jammy Jellyfish)"

Cluster Configuration:

Single node

Validation using commit id a9b8c87

$ curl https://get.k3s.io --output install.sh
$ sudo chmod +x install.sh
$ wget https://raw.githubusercontent.com/k3s-io/k3s/master/k3s-rootless.service
$ mkdir -p /home/ubuntu/.config/systemd/user/
$ cp k3s-rootless.service /home/ubuntu/.config/systemd/user/k3s-rootless.service
$ cat /home/ubuntu/.config/systemd/user/k3s-rootless.service
...
[Unit]
Description=k3s (Rootless)

[Service]
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
Environment=K3S_ROOTLESS_CIDR="10.41.0.0/16"
Environment=K3S_ROOTLESS_PORT_DRIVER=slirp4netns
Environment=K3S_ROOTLESS_DISABLE_HOST_LOOPBACK=true
Environment=K3S_ROOTLESS_MTU=1500
...


$ sudo cat /etc/sysctl.conf
net.ipv4.ip_forward=1
net.ipv6.conf.all.forwarding=1

$ sudo sysctl -w net.ipv4.ip_forward=1
$ sudo sysctl -w net.ipv6.conf.all.forwarding=1
$ sudo apt update
$ sudo apt install uidmap
$ sudo cat /etc/default/grub
---
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"
GRUB_CMDLINE_LINUX="systemd.unified_cgroup_hierarchy=1"
---

$ sudo update-grub
$ sudo mkdir -p /etc/systemd/system/user@.service.d
$ cat <<EOF | sudo tee /etc/systemd/system/user@.service.d/delegate.conf
[Service]
Delegate=cpu cpuset io memory pids
EOF

$ sudo systemctl daemon-reload
$ sudo INSTALL_K3S_COMMIT=a9b8c87fcce467ed7b06b03f2eca7ab4fb45d5df INSTALL_K3S_SKIP_ENABLE=true ./install.sh
$ sudo reboot
$ grep cgroup /proc/mounts
cgroup2 /sys/fs/cgroup cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0

$ stat -c %T -f /sys/fs/cgroup
cgroup2fs

$ cat /sys/fs/cgroup/cgroup.controllers
cpuset cpu io memory hugetlb pids rdma misc

$ sudo sysctl -p
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1

$ systemctl --user enable --now k3s-rootless
$ systemctl --user status k3s-rootless

$ systemctl --user status k3s-rootless
 k3s-rootless.service - k3s (Rootless)
     Loaded: loaded (/home/ubuntu/.config/systemd/user/k3s-rootless.service; enabled; vendor preset: enabled)
     Active: active (running) since Fri 2023-07-14 17:32:00 UTC; 1h 40min ago
   Main PID: 784 (k3s-server)
      Tasks: 171
     Memory: 1.1G
        CPU: 6min 14.131s
     CGroup: /user.slice/user-1000.slice/user@1000.service/app.slice/k3s-rootless.service
             ├─k3s
             │ └─915 "k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ""
             ├─k3s_evac
             │ ├─ 784 "/usr/local/bin/k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ""
             │ ├─ 882 "/proc/self/exe init" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ""
             │ ├─ 892 slirp4netns --mtu 1500 -r 3 --disable-host-loopback --cidr 10.41.0.0/16 --api-socket /tmp/rootless3672698607/.s4nn.sock 882 tap0
             │ ├─ 896 "k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ""
             │ ├─ 939 containerd -c /home/ubuntu/.rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /home/ubuntu/.rancher/k3s/agent/containerd
             │ ├─1442 /home/ubuntu/.rancher/k3s/data/1b15f1c8e7a14263eef14473e1eb19a46990a534da3df5fe73b576bd7b9aebc9/bin/containerd-shim-runc-v2 -namespace k8s.io -id 00773f0dd7af3ea0ccd6685c3401773edfbebf8458ca4a82957cf5e0ded2965a -address /run/k3s/containerd/containerd.sock
             │ ├─1451 /home/ubuntu/.rancher/k3s/data/1b15f1c8e7a14263eef14473e1eb19a46990a534da3df5fe73b576bd7b9aebc9/bin/containerd-shim-runc-v2 -namespace k8s.io -id 15f18cc74135edcb257076c0e14ca7c338fbf655e8695d7ebc0904703fee9598 -address /run/k3s/containerd/containerd.sock
             │ ├─1470 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/15f18cc74135edcb257076c0e14ca7c338fbf655e8695d7ebc0904703fee9598/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/45/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─1471 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/00773f0dd7af3ea0ccd6685c3401773edfbebf8458ca4a82957cf5e0ded2965a/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/46/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─1666 /home/ubuntu/.rancher/k3s/data/1b15f1c8e7a14263eef14473e1eb19a46990a534da3df5fe73b576bd7b9aebc9/bin/containerd-shim-runc-v2 -namespace k8s.io -id 8c7427dced890e4843dd93f65dc5bba4fea5b0f4932453cbfc78e914dd2a35f7 -address /run/k3s/containerd/containerd.sock
             │ ├─1672 /home/ubuntu/.rancher/k3s/data/1b15f1c8e7a14263eef14473e1eb19a46990a534da3df5fe73b576bd7b9aebc9/bin/containerd-shim-runc-v2 -namespace k8s.io -id 55edf682bb58fa6c5ef2239f785173a8de034e399e8dfa5bcc6570342f430c77 -address /run/k3s/containerd/containerd.sock
             │ ├─1707 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/8c7427dced890e4843dd93f65dc5bba4fea5b0f4932453cbfc78e914dd2a35f7/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/47/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─1717 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/55edf682bb58fa6c5ef2239f785173a8de034e399e8dfa5bcc6570342f430c77/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/48/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─1799 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/4895dd337300d39312af7d5e52223aa57f8e3c7024cde19653a8908a07c15bf3/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/49/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─1824 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/10669482d419c0dfa177cd8c84e13bd946c18945915400699c46affe21110ef0/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/50/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─1885 /home/ubuntu/.rancher/k3s/data/1b15f1c8e7a14263eef14473e1eb19a46990a534da3df5fe73b576bd7b9aebc9/bin/containerd-shim-runc-v2 -namespace k8s.io -id 111a662fb8ad3f94c2c20b5d498118d67b47153f6b417dae6a4be8f7eabac541 -address /run/k3s/containerd/containerd.sock
             │ ├─1907 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/111a662fb8ad3f94c2c20b5d498118d67b47153f6b417dae6a4be8f7eabac541/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/51/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─2007 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/405309d4f83e021129751d1f56d91f486e95cf233fcd065bf13ca5a6d83e1078/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/52/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─2040 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/6d36349a24d897c72c5da7f2f85b3a47076cc398d26e63180a1a73e2b3e32fd0/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/54/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ ├─2678 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/4ca7446bf81f86e859a1e92997f6638bb8eae957041ae01654c9a43022ca66ae/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/56/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             │ └─2734 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/a0e2c66b978c5822b43dc21d5325e31089a92660721fc5db9ec0f16d024eac3d/rootfs -o workdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.fuse-overlayfs/snapshots/57/work,upperdir=/home/ubuntu/.rancher/k3s/agent/containerd/io.c>
             └─kubepods
               ├─besteffort
               │ ├─pod57d9ce98-b522-4d6e-867a-6fba14f465e8
               │ │ ├─405309d4f83e021129751d1f56d91f486e95cf233fcd065bf13ca5a6d83e1078
               │ │ │ └─2049 traefik traefik --global.checknewversion --global.sendanonymoususage --entrypoints.metrics.address=:9100/tcp --entrypoints.traefik.address=:9000/tcp --entrypoints.web.address=:8000/tcp --entrypoints.websecure.address=:8443/tcp --api.dashboard=true --ping=true --metrics.prometheus=true --metrics.prometheus.entrypoint=>
               │ │ └─55edf682bb58fa6c5ef2239f785173a8de034e399e8dfa5bcc6570342f430c77
               │ │   └─1761 /pause
               │ ├─pod7e863fb1-9f28-4a2c-86c3-10c0e3308e71
               │ │ ├─10669482d419c0dfa177cd8c84e13bd946c18945915400699c46affe21110ef0
               │ │ │ └─1856 /bin/sh /usr/bin/entry
               │ │ ├─15f18cc74135edcb257076c0e14ca7c338fbf655e8695d7ebc0904703fee9598
               │ │ │ └─1519 /pause
               │ │ └─6d36349a24d897c72c5da7f2f85b3a47076cc398d26e63180a1a73e2b3e32fd0
               │ │   └─2064 /bin/sh /usr/bin/entry
               │ └─podf55d0d94-76e0-45e3-b7f2-0c12f6f13626
               │   ├─111a662fb8ad3f94c2c20b5d498118d67b47153f6b417dae6a4be8f7eabac541
               │   │ └─1933 /pause
               │   └─4ca7446bf81f86e859a1e92997f6638bb8eae957041ae01654c9a43022ca66ae
               │     └─2689 local-path-provisioner start --config /etc/config/config.json
               └─burstable
                 ├─podb41374cb-c542-4f31-8505-3a9726358d3a
                 │ ├─00773f0dd7af3ea0ccd6685c3401773edfbebf8458ca4a82957cf5e0ded2965a
                 │ │ └─1529 /pause
                 │ └─4895dd337300d39312af7d5e52223aa57f8e3c7024cde19653a8908a07c15bf3
                 │   └─1845 /coredns -conf /etc/coredns/Corefile
                 └─podc9b9707a-19fd-436e-a0b8-4163aabda19a
                   ├─8c7427dced890e4843dd93f65dc5bba4fea5b0f4932453cbfc78e914dd2a35f7
                   │ └─1755 /pause

$ export KUBECONFIG=/home/ubuntu/.kube/k3s.yaml
$ kubectl get pods -A
NAMESPACE     NAME                                      READY   STATUS      RESTARTS       AGE
kube-system   helm-install-traefik-crd-fsbwj            0/1     Completed   0              113m
kube-system   helm-install-traefik-x4vdw                0/1     Completed   1              113m
kube-system   svclb-traefik-253abb23-l7hlj              2/2     Running     2 (106m ago)   113m
kube-system   traefik-66fd46ccd-v89w6                   1/1     Running     1 (106m ago)   113m
kube-system   coredns-8b9777675-pxxkc                   1/1     Running     1 (106m ago)   113m
kube-system   local-path-provisioner-69dff9496c-tbjxv   1/1     Running     2 (106m ago)   113m
kube-system   metrics-server-854c559bd-sldsh            1/1     Running     2 (106m ago)   113m
ubuntu@ip-172-31-29-225:~$ kubectl get nodes
NAME               STATUS   ROLES                  AGE    VERSION
ip-172-31-29-225   Ready    control-plane,master   114m   v1.25.11+k3s-a9b8c87f
$

$ journalctl --user -u k3s-rootless|grep "unable to read node password file"
$

Reproduced the issue using the above steps on k3s v1.25.11+k3s1

$  k3s -v
k3s version v1.25.11+k3s1 (582f07cf)
go version go1.19.10

$ journalctl --user -u k3s-rootless|grep "unable to read node password file"
Jul 14 17:24:57 ip-172-31-20-40 k3s[1020]: time="2023-07-14T17:24:57Z" level=error msg="Sending HTTP 500 response to 127.0.0.1:34342: unable to read node password file: open /home/ubuntu/.rancher/k3s/server/agent/etc/rancher/node/password: no such file or directory"