ad hoc SSH debugging

Earthfile

@@ -52,6 +52,7 @@ local:
 build-local:
   BUILD +local --CONTROLLER=yawol-controller
   BUILD +local --CONTROLLER=yawol-cloud-controller
+  BUILD +local --CONTROLLER=yawollet
 
 build-test:
     FROM +deps
@@ -62,7 +63,17 @@ build-test:
 get-envoy-local:
     FROM +envoy
     COPY +get-envoy/envoy /envoy
-    SAVE ARTIFACT /envoy AS LOCAL out/envoy
+    SAVE ARTIFACT /envoy AS LOCAL out/envoy/envoy
+
+get-envoy-libs-local:
+    FROM +envoy
+    COPY +get-envoy/envoylibs /envoylibs
+    SAVE ARTIFACT /envoylibs AS LOCAL out/envoy/lib
+
+get-promtail-local:
+    FROM +promtail
+    COPY +promtail/promtail /promtail
+    SAVE ARTIFACT /promtail AS LOCAL out/promtail
 
 get-envoy:
     FROM +envoy

api/v1beta1/loadbalancer_types.go

@@ -55,6 +55,15 @@ const (
 	ServiceAdditionalNetworks = "yawol.stackit.cloud/additionalNetworks"
 )
 
+// Annotation for settings in lb object
+const (
+	// LoadBalancerAdHocDebug enables adhoc debugging, all LoadBalancer Machines will enable SSH
+	LoadBalancerAdHocDebug = "yawol.stackit.cloud/adHocDebug"
+	// LoadBalancerAdHocDebugSSHKey defines the public ssh key for adhoc debugging
+	// All LoadBalancer Machines will add this public SSH key
+	LoadBalancerAdHocDebugSSHKey = "yawol.stackit.cloud/adHocDebugSSHKey"
+)
+
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:shortName=lb
 // +kubebuilder:subresource:status

controllers/yawollet/loadbalancer_controller.go

@@ -55,6 +55,11 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
+	// enable ad hoc debugging if configured
+	if err := helper.EnableAdHocDebugging(lb, lbm, r.Recorder, r.LoadbalancerMachineName); err != nil {
+		return ctrl.Result{}, kubernetes.SendErrorAsEvent(r.Recorder, fmt.Errorf("%w: unable to get current snapshot", err), lbm)
+	}
+
 	// current snapshot
 	oldSnapshot, err := r.EnvoyCache.GetSnapshot("lb-id")
 	if err != nil {

docs/development.md

@@ -176,15 +176,22 @@ TCP testing using the admin port of Envoy:
 1. Open http://localhost:8085 in your browser
 2. You should get forwarded to the admin port of Envoy which is listening to localhost:9000
 
-## Live-Debugging yawollet
+## Troubleshooting - SSH access to yawol VM
+
+There are currently 2 debug options to access the `LoadBalancerMachine` VM via SSH:
+
+### Debug settings within the `LoadBalancer` `.spec.debugSettings`
+This will add the SSH key via OpenStack KeyPair. A change will recreate the `LoadBalancerMachines`, because OpenStack
+KeyPairs are only possible while VM creation.
 
 1. Upload ssh key-pair to OpenStack
 
-   ```bash
-   openstack keypair create <name> # create new keypair
-   # or
-   openstack keypair create --public-key <path> <name> # add existing pubkey
-   ```
+```bash
+openstack keypair create <name> # create new keypair
+# or
+openstack keypair create --public-key <path> <name> # add existing pubkey
+```
+
 
 2. Add the following to `LoadBalancer`:
 
@@ -197,4 +204,37 @@ TCP testing using the admin port of Envoy:
    ...
    ```
 
-3. SSH into the VM with username `alpine` and `externalIP` from `LoadBalancer`
+This can be also enabled with the service annotations: `yawol.stackit.cloud/debug` and `yawol.stackit.cloud/debugsshkey`
+
+> You can login with the user: `alpine`
+
+### Ad hoc debugging
+To troubleshoot a running `LoadBalancerMachine` we added a function into the `yawollet` to be able to add a SSH key
+and enable/start sshd on the fly.
+
+This can only be enabled with annotations on the `LoadBalancer`: `yawol.stackit.cloud/adHocDebug` and `yawol.stackit.cloud/adHocDebugSSHKey`
+
+This will not recreate the `LoadBalancerMachine`. Be aware that the `yawol.stackit.cloud/adHocDebugSSHKey` has to contain the complete
+SSH public key.
+
+> You can login with the user: `yawoldebug`
+
+> After you are done please remove the VMs, because yawol will **not** disable SSH again.
+
+
+## Image Build
+
+For the image build ansible is used. To develop on ansible you can run in locally.
+Therefore, you need to get/build all needed binaries and change to the `image` directory:
+
+```
+earthly +get-envoy-local
+earthly +get-envoy-libs-local
+earthly +get-promtail-local
+earthly +build-local
+```
+
+Now you can run ansible:
+```
+ansible-playbook -i <IP-Address>, --private-key=~/.ssh/ske-key --user alpine install-alpine.yaml
+```

image/install-alpine.yaml

@@ -1,6 +1,6 @@
 ---
 - name: install dependencies and envoy to machine
-  hosts: default
+  hosts: all
   become: true
   tasks:
     # alpine updates
@@ -23,6 +23,25 @@
         search_string: 'slaac hwaddr'
         line: 'slaac hwaddr'
 
+    # sshd config
+    - name: disable root login
+      ansible.builtin.lineinfile:
+        path: /etc/ssh/sshd_config
+        regexp: ^PermitRootLogin.*$
+        line: 'PermitRootLogin no'
+
+    - name: disable login without password
+      ansible.builtin.lineinfile:
+        path: /etc/ssh/sshd_config
+        regexp: ^PermitEmptyPasswords.*$
+        line: 'PermitEmptyPasswords no'
+
+    - name: disable password login
+      ansible.builtin.lineinfile:
+        path: /etc/ssh/sshd_config
+        regexp: ^PasswordAuthentication.*$
+        line: 'PasswordAuthentication no'
+
     # keepalived
     - name: install keepalived
       command: "apk add keepalived"
@@ -60,6 +79,35 @@
         state: directory
         mode: '0755'
 
+    - name: add sudoers file for yawol
+      copy:
+        src: ./yawol-sudoers
+        dest: /etc/sudoers.d/yawol
+        owner: root
+        group: root
+        mode: 0644
+
+    # yawoldebug user
+    - name: Creating yawol user
+      user:
+        name: "yawoldebug"
+        shell: /bin/ash
+        password: "*"
+
+    - name: Create a .ssh directory for yawoldebug user
+      file:
+        path: /home/yawoldebug/.ssh
+        owner: "yawoldebug"
+        state: directory
+        mode: '0700'
+
+    - name: Create a .ssh/authorized_keys file for yawoldebug user
+      file:
+        path: /home/yawoldebug/.ssh/authorized_keys
+        owner: "yawoldebug"
+        state: touch
+        mode: '0600'
+
     # envoy
     - name: Copy envoy in place
       copy:
@@ -237,3 +285,10 @@
 
     - name: more cleanup
       command: "cloud-init clean -l -s"
+
+    - name: cleanup ssh-keys
+      lineinfile:
+        path: /home/alpine/.ssh/authorized_keys
+        state: absent
+        regexp: '.*'
+

image/yawol-sudoers

@@ -0,0 +1,4 @@
+yawol ALL = NOPASSWD: /sbin/rc-service sshd start
+yawol ALL = NOPASSWD: /usr/bin/tee /home/yawoldebug/.ssh/authorized_keys
+
+yawoldebug ALL=(ALL) NOPASSWD:ALL

internal/helper/loadbalancer.go

@@ -272,7 +272,10 @@ func getSecGroupRulesForPorts(r record.EventRecorder, lb *yawolv1beta1.LoadBalan
 }
 
 func getSecGroupRulesForDebugSettings(r record.EventRecorder, lb *yawolv1beta1.LoadBalancer) []rules.SecGroupRule {
-	if !lb.Spec.DebugSettings.Enabled {
+	adHocDebug, _ := strconv.ParseBool(lb.Annotations[yawolv1beta1.LoadBalancerAdHocDebug])
+
+	if !lb.Spec.DebugSettings.Enabled &&
+		!adHocDebug {
 		return []rules.SecGroupRule{}
 	}
 

internal/helper/yawollet.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os/exec"
 	"regexp"
 	"strconv"
 	"strings"
@@ -112,6 +113,11 @@ const (
 const dnsName string = `^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9])).([a-zA-Z]{2,6}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$` //nolint:lll // long regex
 var rxDNSName = regexp.MustCompile(dnsName)
 
+// Const declaration for SSH pub key checking
+const sshPublicKey = `^ssh-rsa\s+[A-Za-z0-9+/=]+$`
+
+var rxSSHPublicKey = regexp.MustCompile(sshPublicKey)
+
 // CreateEnvoyConfig create a new envoy snapshot and checks if the new snapshot has changes
 func CreateEnvoyConfig(
 	r record.EventRecorder,
@@ -781,3 +787,60 @@ func UpdateKeepalivedStatus(
 		ConditionTrue,
 		"StatsUpToDate", "Keepalived stat file is newer than 5 min")
 }
+
+// EnableAdHocDebugging enables ad-hoc debugging if enabled via annotations.
+func EnableAdHocDebugging(
+	lb *yawolv1beta1.LoadBalancer,
+	lbm *yawolv1beta1.LoadBalancerMachine,
+	recorder record.EventRecorder,
+	lbmName string,
+) error {
+	enabled, _ := strconv.ParseBool(lb.Annotations[yawolv1beta1.LoadBalancerAdHocDebug])
+	sshKey, sshKeySet := lb.Annotations[yawolv1beta1.LoadBalancerAdHocDebugSSHKey]
+
+	// skip not all needed annotations are set or disabled
+	if !enabled || !sshKeySet {
+		return nil
+	}
+
+	// skip if debugging is enabled anyway
+	if lb.Spec.DebugSettings.Enabled {
+		recorder.Event(lbm, corev1.EventTypeWarning,
+			"AdHocDebuggingNotEnabled",
+			"Ad-hoc debugging will not be enabled because normal debug with is already enabled")
+		return nil
+	}
+
+	if !rxSSHPublicKey.MatchString(sshKey) {
+		recorder.Event(lbm, corev1.EventTypeWarning,
+			"AdHocDebuggingNotEnabled",
+			"Ad-hoc debugging will not be enabled because ssh key is not valid")
+		return nil
+	}
+
+	addAuthorizedKeys := exec.Command( //nolint: gosec // sshKey can only be a ssh public key checked by regex
+		"/bin/sh",
+		"-c",
+		"echo \"\n"+sshKey+"\n\" | sudo tee /home/yawoldebug/.ssh/authorized_keys",
+	)
+	if err := addAuthorizedKeys.Run(); err != nil {
+		recorder.Eventf(lbm, corev1.EventTypeWarning,
+			"AdHocDebuggingNotEnabled",
+			"Ad-hoc debugging cant be enabled because authorized_keys cant be written: %v", err)
+		return nil // no error to be sure the loadbalancer still will be reconciled
+	}
+
+	startSSH := exec.Command("sudo", "/sbin/rc-service", "sshd", "start")
+	if err := startSSH.Run(); err != nil {
+		recorder.Eventf(lbm, corev1.EventTypeWarning,
+			"AdHocDebuggingNotEnabled",
+			"Ad-hoc debugging cant be enabled because ssh cant be started: %v", err)
+		return nil // no error to be sure the loadbalancer still will be reconciled
+	}
+
+	recorder.Eventf(lbm, corev1.EventTypeWarning,
+		"AdHocDebuggingEnabled",
+		"Successfully enabled ad-hoc debugging access to LoadBalancerMachine '%s'. "+
+			"Please make sure to disable debug access once you are finished and to roll all LoadBalancerMachines", lbmName)
+	return nil
+}