From 2bb842f003a9dd726620b78994ddd667f03ab1a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20Tekin=20=C3=96z?= Date: Wed, 3 Jul 2024 12:12:47 +0300 Subject: [PATCH] Switch default calico encapsulation to VXLAN, add annotations for autodetection (#514) --- src/k8s/pkg/k8sd/features/calico/chart.go | 6 + src/k8s/pkg/k8sd/features/calico/cleanup.go | 7 +- src/k8s/pkg/k8sd/features/calico/internal.go | 140 ++++++++++++++++++ .../pkg/k8sd/features/calico/internal_test.go | 100 +++++++++++++ src/k8s/pkg/k8sd/features/calico/network.go | 38 +++-- 5 files changed, 278 insertions(+), 13 deletions(-) create mode 100644 src/k8s/pkg/k8sd/features/calico/internal.go create mode 100644 src/k8s/pkg/k8sd/features/calico/internal_test.go diff --git a/src/k8s/pkg/k8sd/features/calico/chart.go b/src/k8s/pkg/k8sd/features/calico/chart.go index 04a860a9c..89c4d3edd 100644 --- a/src/k8s/pkg/k8sd/features/calico/chart.go +++ b/src/k8s/pkg/k8sd/features/calico/chart.go @@ -35,4 +35,10 @@ var ( calicoCtlImage = "ghcr.io/canonical/k8s-snap/calico/ctl" // calicoCtlTag represents the tag to use for the calicoctl image. calicoCtlTag = "v3.28.0" + + // defaultEncapsulation represents the default defaultEncapsulation method to use for Calico. + defaultEncapsulation = "VXLAN" + + // defaultAPIServerEnabled determines if the Calico API server should be enabled. + defaultAPIServerEnabled = false ) diff --git a/src/k8s/pkg/k8sd/features/calico/cleanup.go b/src/k8s/pkg/k8sd/features/calico/cleanup.go index 5c09f99ea..ad0acd2ba 100644 --- a/src/k8s/pkg/k8sd/features/calico/cleanup.go +++ b/src/k8s/pkg/k8sd/features/calico/cleanup.go @@ -14,6 +14,8 @@ import ( "golang.org/x/sys/unix" ) +var reDefaultCalicoInterface = regexp.MustCompile("^vxlan[-v6]*.calico|cali[a-f0-9]*|tunl[0-9]*$") + func CleanupNetwork(ctx context.Context, snap snap.Snap) error { interfaces, err := net.Interfaces() if err != nil { @@ -25,10 +27,7 @@ func CleanupNetwork(ctx context.Context, snap snap.Snap) error { // Check if the interface name matches the regex pattern // Adapted from MicroK8s' link removal hook: // https://github.com/canonical/microk8s/blob/dff3627959d4774198000795a0a0afcaa003324b/microk8s-resources/default-hooks/remove.d/10-cni-link#L15 - match, err := regexp.MatchString("^vxlan[-v6]*.calico|cali[a-f0-9]*|tunl[0-9]*$", iface.Name) - if err != nil { - return fmt.Errorf("failed to match regex pattern: %w", err) - } + match := reDefaultCalicoInterface.MatchString(iface.Name) if match { // Perform cleanup for Calico interface if err := exec.CommandContext(ctx, "ip", "link", "delete", iface.Name).Run(); err != nil { diff --git a/src/k8s/pkg/k8sd/features/calico/internal.go b/src/k8s/pkg/k8sd/features/calico/internal.go new file mode 100644 index 000000000..930bc674a --- /dev/null +++ b/src/k8s/pkg/k8sd/features/calico/internal.go @@ -0,0 +1,140 @@ +package calico + +import ( + "fmt" + "strings" + + "github.com/canonical/k8s/pkg/k8sd/types" +) + +const ( + annotationAPIServerEnabled = "k8sd/v1alpha1/calico/apiserver-enabled" + annotationEncapsulationV4 = "k8sd/v1alpha1/calico/encapsulation-v4" + annotationEncapsulationV6 = "k8sd/v1alpha1/calico/encapsulation-v6" + annotationAutodetectionV4FirstFound = "k8sd/v1alpha1/calico/autodetection-v4/firstFound" + annotationAutodetectionV4Kubernetes = "k8sd/v1alpha1/calico/autodetection-v4/kubernetes" + annotationAutodetectionV4Interface = "k8sd/v1alpha1/calico/autodetection-v4/interface" + annotationAutodetectionV4SkipInterface = "k8sd/v1alpha1/calico/autodetection-v4/skipInterface" + annotationAutodetectionV4CanReach = "k8sd/v1alpha1/calico/autodetection-v4/canReach" + annotationAutodetectionV4CIDRs = "k8sd/v1alpha1/calico/autodetection-v4/cidrs" + annotationAutodetectionV6FirstFound = "k8sd/v1alpha1/calico/autodetection-v6/firstFound" + annotationAutodetectionV6Kubernetes = "k8sd/v1alpha1/calico/autodetection-v6/kubernetes" + annotationAutodetectionV6Interface = "k8sd/v1alpha1/calico/autodetection-v6/interface" + annotationAutodetectionV6SkipInterface = "k8sd/v1alpha1/calico/autodetection-v6/skipInterface" + annotationAutodetectionV6CanReach = "k8sd/v1alpha1/calico/autodetection-v6/canReach" + annotationAutodetectionV6CIDRs = "k8sd/v1alpha1/calico/autodetection-v6/cidrs" +) + +type config struct { + encapsulationV4 string + encapsulationV6 string + apiServerEnabled bool + autodetectionV4 map[string]any + autodetectionV6 map[string]any +} + +func checkEncapsulation(v string) error { + switch v { + case "VXLAN", "IPIP", "IPIPCrossSubnet", "VXLANCrossSubnet", "None": + return nil + } + return fmt.Errorf("unsupported encapsulation type: %s", v) +} + +func parseAutodetectionAnnotations(annotations types.Annotations, autodetectionMap map[string]string) (map[string]any, error) { + var autodetectionAnnotations []string + var autodetectionKey string + var autodetectionValue any + + for annotation, key := range autodetectionMap { + if v, ok := annotations.Get(annotation); ok { + autodetectionAnnotations = append(autodetectionAnnotations, annotation) + autodetectionKey = key + autodetectionValue = v + } + } + + if len(autodetectionAnnotations) > 1 { + return nil, fmt.Errorf("multiple annotations found: %s", strings.Join(autodetectionAnnotations, ", ")) + } + + // If any annotation is set, return the map otherwise it's left nil + if autodetectionKey != "" { + switch autodetectionKey { + case "firstFound": + autodetectionValue = autodetectionValue == "true" + case "cidrs": + autodetectionValue = strings.Split(autodetectionValue.(string), ",") + } + + return map[string]any{ + autodetectionKey: autodetectionValue, + }, nil + } + + return nil, nil +} + +func internalConfig(annotations types.Annotations) (config, error) { + c := config{ + encapsulationV4: defaultEncapsulation, + encapsulationV6: defaultEncapsulation, + apiServerEnabled: defaultAPIServerEnabled, + } + + if v, ok := annotations.Get(annotationAPIServerEnabled); ok { + c.apiServerEnabled = v == "true" + } + + if v, ok := annotations.Get(annotationEncapsulationV4); ok { + if err := checkEncapsulation(v); err != nil { + return config{}, fmt.Errorf("invalid encapsulation-v4 annotation: %w", err) + } + c.encapsulationV4 = v + } + + if v, ok := annotations.Get(annotationEncapsulationV6); ok { + if err := checkEncapsulation(v); err != nil { + return config{}, fmt.Errorf("invalid encapsulation-v6 annotation: %w", err) + } + c.encapsulationV6 = v + } + + v4Map := map[string]string{ + annotationAutodetectionV4FirstFound: "firstFound", + annotationAutodetectionV4Kubernetes: "kubernetes", + annotationAutodetectionV4Interface: "interface", + annotationAutodetectionV4SkipInterface: "skipInterface", + annotationAutodetectionV4CanReach: "canReach", + annotationAutodetectionV4CIDRs: "cidrs", + } + + autodetectionV4, err := parseAutodetectionAnnotations(annotations, v4Map) + if err != nil { + return config{}, fmt.Errorf("error parsing autodetection-v4 annotations: %w", err) + } + + if autodetectionV4 != nil { + c.autodetectionV4 = autodetectionV4 + } + + v6Map := map[string]string{ + annotationAutodetectionV6FirstFound: "firstFound", + annotationAutodetectionV6Kubernetes: "kubernetes", + annotationAutodetectionV6Interface: "interface", + annotationAutodetectionV6SkipInterface: "skipInterface", + annotationAutodetectionV6CanReach: "canReach", + annotationAutodetectionV6CIDRs: "cidrs", + } + + autodetectionV6, err := parseAutodetectionAnnotations(annotations, v6Map) + if err != nil { + return config{}, fmt.Errorf("error parsing autodetection-v6 annotations: %w", err) + } + + if autodetectionV6 != nil { + c.autodetectionV6 = autodetectionV6 + } + + return c, nil +} diff --git a/src/k8s/pkg/k8sd/features/calico/internal_test.go b/src/k8s/pkg/k8sd/features/calico/internal_test.go new file mode 100644 index 000000000..dd4c630fb --- /dev/null +++ b/src/k8s/pkg/k8sd/features/calico/internal_test.go @@ -0,0 +1,100 @@ +package calico + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestInternalConfig(t *testing.T) { + for _, tc := range []struct { + name string + annotations map[string]string + expectedConfig config + expectError bool + }{ + { + name: "Empty", + annotations: map[string]string{}, + expectedConfig: config{ + apiServerEnabled: false, + encapsulationV4: "VXLAN", + encapsulationV6: "VXLAN", + }, + expectError: false, + }, + { + name: "Valid", + annotations: map[string]string{ + annotationAPIServerEnabled: "true", + annotationEncapsulationV4: "IPIP", + }, + expectedConfig: config{ + apiServerEnabled: true, + encapsulationV4: "IPIP", + encapsulationV6: "VXLAN", + }, + expectError: false, + }, + { + name: "InvalidEncapsulation", + annotations: map[string]string{ + annotationEncapsulationV4: "Invalid", + }, + expectError: true, + }, + { + name: "InvalidAPIServerEnabled", + annotations: map[string]string{ + annotationAPIServerEnabled: "invalid", + annotationEncapsulationV4: "VXLAN", + }, + expectedConfig: config{ + apiServerEnabled: false, + encapsulationV4: "VXLAN", + encapsulationV6: "VXLAN", + }, + expectError: false, + }, + { + name: "MultipleAutodetectionV4", + annotations: map[string]string{ + annotationAutodetectionV4FirstFound: "true", + annotationAutodetectionV4Kubernetes: "true", + }, + expectError: true, + }, + { + name: "ValidAutodetectionCidrs", + annotations: map[string]string{ + annotationAutodetectionV4CIDRs: "10.1.0.0/16,2001:0db8::/32", + }, + expectedConfig: config{ + apiServerEnabled: false, + encapsulationV4: "VXLAN", + encapsulationV6: "VXLAN", + autodetectionV4: map[string]any{ + "cidrs": []string{"10.1.0.0/16", "2001:0db8::/32"}, + }, + autodetectionV6: nil, + }, + expectError: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + annotations := make(map[string]string) + for k, v := range tc.annotations { + annotations[k] = v + } + + parsed, err := internalConfig(annotations) + if tc.expectError { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(parsed).To(Equal(tc.expectedConfig)) + } + }) + } +} diff --git a/src/k8s/pkg/k8sd/features/calico/network.go b/src/k8s/pkg/k8sd/features/calico/network.go index 6ea50df92..756b6d842 100644 --- a/src/k8s/pkg/k8sd/features/calico/network.go +++ b/src/k8s/pkg/k8sd/features/calico/network.go @@ -13,7 +13,7 @@ import ( // ApplyNetwork will deploy Calico when cfg.Enabled is true. // ApplyNetwork will remove Calico when cfg.Enabled is false. // ApplyNetwork returns an error if anything fails. -func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ types.Annotations) error { +func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annotations types.Annotations) error { m := snap.HelmClient() if !cfg.GetEnabled() { @@ -23,6 +23,11 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ type return nil } + config, err := internalConfig(annotations) + if err != nil { + return fmt.Errorf("failed to parse annotations: %w", err) + } + podIpPools := []map[string]any{} ipv4PodCIDR, ipv6PodCIDR, err := utils.ParseCIDRs(cfg.GetPodCIDR()) if err != nil { @@ -30,14 +35,16 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ type } if ipv4PodCIDR != "" { podIpPools = append(podIpPools, map[string]any{ - "name": "ipv4-ippool", - "cidr": ipv4PodCIDR, + "name": "ipv4-ippool", + "cidr": ipv4PodCIDR, + "encapsulation": config.encapsulationV4, }) } if ipv6PodCIDR != "" { podIpPools = append(podIpPools, map[string]any{ - "name": "ipv6-ippool", - "cidr": ipv6PodCIDR, + "name": "ipv6-ippool", + "cidr": ipv6PodCIDR, + "encapsulation": config.encapsulationV6, }) } @@ -53,6 +60,18 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ type serviceCIDRs = append(serviceCIDRs, ipv6ServiceCIDR) } + calicoNetworkValues := map[string]any{ + "ipPools": podIpPools, + } + + if config.autodetectionV4 != nil { + calicoNetworkValues["nodeAddressAutodetectionV4"] = config.autodetectionV4 + } + + if config.autodetectionV6 != nil { + calicoNetworkValues["nodeAddressAutodetectionV6"] = config.autodetectionV6 + } + values := map[string]any{ "tigeraOperator": map[string]any{ "registry": imageRepo, @@ -64,10 +83,11 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ type "tag": calicoCtlTag, }, "installation": map[string]any{ - "calicoNetwork": map[string]any{ - "ipPools": podIpPools, - }, - "registry": imageRepo, + "calicoNetwork": calicoNetworkValues, + "registry": imageRepo, + }, + "apiServer": map[string]any{ + "enabled": config.apiServerEnabled, }, "serviceCIDRs": serviceCIDRs, }