Add cleanup interface, move cilium cleanup and add calico cleanup (#491)

---------

Co-authored-by: Angelos Kolaitis <angelos.kolaitis@canonical.com>

build-scripts/patches/moonray/apply

@@ -14,8 +14,6 @@ rm "${DIR}/../../../tests/integration/tests/test_loadbalancer.py"
 rm "${DIR}/../../../tests/integration/tests/test_gateway.py"
 ## TODO: restore when ingress is implemented
 rm "${DIR}/../../../tests/integration/tests/test_ingress.py"
-## TODO: restore when cleanup is implemented
-rm "${DIR}/../../../tests/integration/tests/test_cleanup.py"
 
 git commit -a -m "Remove unrelated tests"
 

k8s/lib.sh

@@ -43,37 +43,11 @@ k8s::common::is_strict() {
   fi
 }
 
-# Cleanup configuration left by the network component
-#   - Iptables Rules
-#   - Network Interfaces
-#   - Traffic Control(tc) rules
-# https://github.com/cilium/cilium/blob/7318ce2d0d89a91227e3f313adebce892f3c388e/cilium-dbg/cmd/cleanup.go#L132-L139
+# Cleanup configuration left by the network feature
 k8s::remove::network() {
   k8s::common::setup_env
 
-  local default_interface
-
-  for link in cilium_vxlan cilium_host cilium_net
-  do
-    ip link delete ${link} || true
-  done
-
-  iptables-save | grep -iv cilium | iptables-restore
-  ip6tables-save | grep -iv cilium | ip6tables-restore
-  iptables-legacy-save | grep -iv cilium | iptables-legacy-restore
-  ip6tables-legacy-save | grep -iv cilium | ip6tables-legacy-restore
-
-  default_interface="$(k8s::util::default_interface)"
-
-  for d in ingress egress
-  do
-    tc filter del dev $default_interface ${d} || true
-  done
-
-  rm -rf /var/run/cilium/cilium.pid
-  if [ -f /opt/cni/bin/cilium-dbg ]; then
-      /opt/cni/bin/cilium-dbg cleanup --all-state --force || true
-  fi
+  k8s::cmd::k8s x-cleanup network || true
 }
 
 # [DANGER] Cleanup containers and runtime state. Note that the order of operations below is crucial.

src/k8s/cmd/k8s/k8s.go

@@ -97,6 +97,7 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 		newXWaitForCmd(env),
 		newXCAPICmd(env),
 		newListImagesCmd(env),
+		newXCleanupCmd(env),
 	)
 
 	cmd.DisableAutoGenTag = true

src/k8s/cmd/k8s/k8s_x_cleanup.go

@@ -0,0 +1,41 @@
+package k8s
+
+import (
+	"context"
+	"time"
+
+	cmdutil "github.com/canonical/k8s/cmd/util"
+	"github.com/canonical/k8s/pkg/k8sd/features"
+	"github.com/spf13/cobra"
+)
+
+func newXCleanupCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
+	var opts struct {
+		timeout time.Duration
+	}
+
+	cleanupNetworkCmd := &cobra.Command{
+		Use:   "network",
+		Short: "Cleanup left-over network resources",
+		Run: func(cmd *cobra.Command, args []string) {
+			ctx, cancel := context.WithTimeout(cmd.Context(), opts.timeout)
+			defer cancel()
+
+			if err := features.Cleanup.CleanupNetwork(ctx, env.Snap); err != nil {
+				cmd.PrintErrf("Error: failed to cleanup network: %v\n", err)
+				env.Exit(1)
+			}
+		},
+	}
+	cleanupNetworkCmd.Flags().DurationVar(&opts.timeout, "timeout", 5*time.Minute, "the max time to wait for the command to execute")
+
+	cmd := &cobra.Command{
+		Use:    "x-cleanup",
+		Short:  "Cleanup left-over resources from the cluster's features",
+		Hidden: true,
+	}
+
+	cmd.AddCommand(cleanupNetworkCmd)
+
+	return cmd
+}

src/k8s/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/pelletier/go-toml v1.9.5
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/net v0.23.0
+	golang.org/x/sys v0.19.0
 	gopkg.in/yaml.v2 v2.4.0
 	helm.sh/helm/v3 v3.14.2
 	k8s.io/api v0.29.0
@@ -142,7 +143,6 @@ require (
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect

src/k8s/pkg/k8sd/features/calico/cleanup.go

@@ -0,0 +1,82 @@
+package calico
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/canonical/k8s/pkg/snap"
+	"golang.org/x/sys/unix"
+)
+
+func CleanupNetwork(ctx context.Context, snap snap.Snap) error {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return fmt.Errorf("failed to list network interfaces: %w", err)
+	}
+
+	// Find the interfaces created by Calico
+	for _, iface := range interfaces {
+		// Check if the interface name matches the regex pattern
+		// Adapted from MicroK8s' link removal hook:
+		// https://github.com/canonical/microk8s/blob/dff3627959d4774198000795a0a0afcaa003324b/microk8s-resources/default-hooks/remove.d/10-cni-link#L15
+		match, err := regexp.MatchString("^vxlan[-v6]*.calico|cali[a-f0-9]*|tunl[0-9]*$", iface.Name)
+		if err != nil {
+			return fmt.Errorf("failed to match regex pattern: %w", err)
+		}
+		if match {
+			// Perform cleanup for Calico interface
+			if err := exec.CommandContext(ctx, "ip", "link", "delete", iface.Name).Run(); err != nil {
+				return fmt.Errorf("failed to delete interface %s: %w", iface.Name, err)
+			}
+		}
+	}
+
+	// Delete network namespaces that start with "cali-"
+	netnsDir := "/run/netns"
+	entries, err := os.ReadDir(netnsDir)
+	if err != nil {
+		return fmt.Errorf("failed to list files under %s: %w", netnsDir, err)
+	}
+
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), "cali-") {
+			nsPath := path.Join(netnsDir, entry.Name())
+
+			if err := unix.Unmount(nsPath, unix.MNT_DETACH); err != nil {
+				return fmt.Errorf("failed to unmount network namespace %s: %w", entry.Name(), err)
+			}
+
+			if err := os.Remove(nsPath); err != nil {
+				return fmt.Errorf("failed to remove network namespace %s: %w", entry.Name(), err)
+			}
+		}
+	}
+
+	for _, cmd := range []string{"iptables", "ip6tables", "iptables-legacy", "ip6tables-legacy"} {
+		out, err := exec.Command(fmt.Sprintf("%s-save", cmd)).Output()
+		if err != nil {
+			return fmt.Errorf("failed to read iptables rules: %w", err)
+		}
+
+		lines := strings.Split(string(out), "\n")
+		for i, line := range lines {
+			if strings.Contains(line, "cali") {
+				lines[i] = ""
+			}
+		}
+
+		restore := exec.Command(fmt.Sprintf("%s-restore", cmd))
+		restore.Stdin = strings.NewReader(strings.Join(lines, "\n"))
+		if err := restore.Run(); err != nil {
+			return fmt.Errorf("failed to restore iptables rules: %w", err)
+		}
+	}
+
+	return nil
+}

src/k8s/pkg/k8sd/features/cilium/cleanup.go

@@ -0,0 +1,22 @@
+package cilium
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/canonical/k8s/pkg/snap"
+)
+
+func CleanupNetwork(ctx context.Context, snap snap.Snap) error {
+	os.Remove("/var/run/cilium/cilium.pid")
+
+	if _, err := os.Stat("/opt/cni/bin/cilium-dbg"); err == nil {
+		if err := exec.CommandContext(ctx, "/opt/cni/bin/cilium-dbg", "cleanup", "--all-state", "--force").Run(); err != nil {
+			return fmt.Errorf("cilium-dbg cleanup failed: %w", err)
+		}
+	}
+
+	return nil
+}

src/k8s/pkg/k8sd/features/cleanup.go

@@ -0,0 +1,19 @@
+package features
+
+import (
+	"context"
+
+	"github.com/canonical/k8s/pkg/snap"
+)
+
+type CleanupInterface interface {
+	CleanupNetwork(context.Context, snap.Snap) error
+}
+
+type cleanup struct {
+	cleanupNetwork func(context.Context, snap.Snap) error
+}
+
+func (c *cleanup) CleanupNetwork(ctx context.Context, snap snap.Snap) error {
+	return c.cleanupNetwork(ctx, snap)
+}

src/k8s/pkg/k8sd/features/implementation_default.go

@@ -27,3 +27,7 @@ var StatusChecks StatusInterface = &statusChecks{
 	checkNetwork: cilium.CheckNetwork,
 	checkDNS:     coredns.CheckDNS,
 }
+
+var Cleanup CleanupInterface = &cleanup{
+	cleanupNetwork: cilium.CleanupNetwork,
+}

src/k8s/pkg/k8sd/features/implementation_moonray.go

@@ -28,3 +28,7 @@ var StatusChecks StatusInterface = &statusChecks{
 	checkNetwork: calico.CheckNetwork,
 	checkDNS:     coredns.CheckDNS,
 }
+
+var Cleanup CleanupInterface = &cleanup{
+	cleanupNetwork: calico.CleanupNetwork,
+}