From e8475bfbcf77357355a62d0ec6ae2f4e1dd17085 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Wed, 18 Sep 2024 15:03:58 +0200 Subject: [PATCH] Add IPv6-only support for moonray (#664) --- .github/workflows/integration-informing.yaml | 4 + src/k8s/pkg/client/kubernetes/endpoints.go | 9 +- src/k8s/pkg/k8sd/api/certificates_refresh.go | 25 +++- src/k8s/pkg/k8sd/app/hooks_bootstrap.go | 31 +++-- src/k8s/pkg/k8sd/app/hooks_join.go | 23 ++- src/k8s/pkg/k8sd/pki/k8sdqlite.go | 2 +- src/k8s/pkg/k8sd/pki/worker.go | 2 +- src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go | 4 +- .../k8sd/setup/k8s_apiserver_proxy_test.go | 10 +- src/k8s/pkg/k8sd/setup/kube_proxy.go | 4 +- src/k8s/pkg/k8sd/setup/kube_proxy_test.go | 8 +- src/k8s/pkg/k8sd/setup/kubelet.go | 2 +- src/k8s/pkg/k8sd/setup/util_kubeconfig.go | 4 +- src/k8s/pkg/proxy/userspace.go | 5 +- src/k8s/pkg/utils/cidr.go | 17 +++ src/k8s/pkg/utils/cidr_test.go | 41 ++++++ .../templates/bootstrap-ipv6-only.yaml | 16 +++ .../integration/templates/etcd/etcd-tls.conf | 2 +- .../templates/nginx-ipv6-only.yaml | 36 +++++ tests/integration/tests/test_dualstack.py | 58 -------- tests/integration/tests/test_networking.py | 131 ++++++++++++++++++ tests/integration/tests/test_util/util.py | 22 +++ 22 files changed, 361 insertions(+), 95 deletions(-) create mode 100644 tests/integration/templates/bootstrap-ipv6-only.yaml create mode 100644 tests/integration/templates/nginx-ipv6-only.yaml delete mode 100644 tests/integration/tests/test_dualstack.py create mode 100644 tests/integration/tests/test_networking.py diff --git a/.github/workflows/integration-informing.yaml b/.github/workflows/integration-informing.yaml index 708094078..94cc7c01d 100644 --- a/.github/workflows/integration-informing.yaml +++ b/.github/workflows/integration-informing.yaml @@ -86,6 +86,10 @@ jobs: export TEST_SUBSTRATE=lxd export TEST_LXD_IMAGE=${{ matrix.os }} export TEST_INSPECTION_REPORTS_DIR="$HOME/inspection-reports" + # IPv6-only is only supported on moonray + if [[ "${{ matrix.patch }}" == "moonray" ]]; then + export TEST_IPV6_ONLY="true" + fi cd tests/integration && sg lxd -c 'tox -e integration' - name: Prepare inspection reports if: failure() diff --git a/src/k8s/pkg/client/kubernetes/endpoints.go b/src/k8s/pkg/client/kubernetes/endpoints.go index 3d6709800..0840effb9 100644 --- a/src/k8s/pkg/client/kubernetes/endpoints.go +++ b/src/k8s/pkg/client/kubernetes/endpoints.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" + "github.com/canonical/k8s/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -40,7 +41,13 @@ func (c *Client) GetKubeAPIServerEndpoints(ctx context.Context) ([]string, error } for _, addr := range subset.Addresses { if addr.IP != "" { - addresses = append(addresses, fmt.Sprintf("%s:%d", addr.IP, portNumber)) + var address string + if utils.IsIPv4(addr.IP) { + address = addr.IP + } else { + address = fmt.Sprintf("[%s]", addr.IP) + } + addresses = append(addresses, fmt.Sprintf("%s:%d", address, portNumber)) } } } diff --git a/src/k8s/pkg/k8sd/api/certificates_refresh.go b/src/k8s/pkg/k8sd/api/certificates_refresh.go index 2d8153dc1..b08dada6b 100644 --- a/src/k8s/pkg/k8sd/api/certificates_refresh.go +++ b/src/k8s/pkg/k8sd/api/certificates_refresh.go @@ -81,6 +81,13 @@ func refreshCertsRunControlPlane(s state.State, r *http.Request, snap snap.Snap) return response.InternalError(fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname())) } + var localhostAddress string + if nodeIP.To4() == nil { + localhostAddress = "[::1]" + } else { + localhostAddress = "127.0.0.1" + } + serviceIPs, err := utils.GetKubernetesServiceIPsFromServiceCIDRs(clusterConfig.Network.GetServiceCIDR()) if err != nil { return response.InternalError(fmt.Errorf("failed to get IP address(es) from ServiceCIDR %q: %w", clusterConfig.Network.GetServiceCIDR(), err)) @@ -119,7 +126,7 @@ func refreshCertsRunControlPlane(s state.State, r *http.Request, snap snap.Snap) return response.InternalError(fmt.Errorf("failed to write control plane certificates: %w", err)) } - if err := setup.SetupControlPlaneKubeconfigs(snap.KubernetesConfigDir(), clusterConfig.APIServer.GetSecurePort(), *certificates); err != nil { + if err := setup.SetupControlPlaneKubeconfigs(snap.KubernetesConfigDir(), localhostAddress, clusterConfig.APIServer.GetSecurePort(), *certificates); err != nil { return response.InternalError(fmt.Errorf("failed to generate control plane kubeconfigs: %w", err)) } @@ -262,11 +269,23 @@ func refreshCertsRunWorker(s state.State, r *http.Request, snap snap.Snap) respo return response.InternalError(fmt.Errorf("failed to write worker PKI: %w", err)) } + nodeIP := net.ParseIP(s.Address().Hostname()) + if nodeIP == nil { + return response.InternalError(fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname())) + } + + var localhostAddress string + if nodeIP.To4() == nil { + localhostAddress = "[::1]" + } else { + localhostAddress = "127.0.0.1" + } + // Kubeconfigs - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), "127.0.0.1:6443", certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { return response.InternalError(fmt.Errorf("failed to generate kubelet kubeconfig: %w", err)) } - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), "127.0.0.1:6443", certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { return response.InternalError(fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err)) } diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index 4144a0f84..c02854b43 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -178,11 +178,18 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT return fmt.Errorf("failed to write worker node certificates: %w", err) } + var localhostAddress string + if nodeIP.To4() == nil { + localhostAddress = "[::1]" + } else { + localhostAddress = "127.0.0.1" + } + // Kubeconfigs - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), "127.0.0.1:6443", certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { return fmt.Errorf("failed to generate kubelet kubeconfig: %w", err) } - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), "127.0.0.1:6443", certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { return fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err) } @@ -229,10 +236,10 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil { return fmt.Errorf("failed to configure kubelet: %w", err) } - if err := setup.KubeProxy(ctx, snap, s.Name(), response.PodCIDR, joinConfig.ExtraNodeKubeProxyArgs); err != nil { + if err := setup.KubeProxy(ctx, snap, s.Name(), response.PodCIDR, localhostAddress, joinConfig.ExtraNodeKubeProxyArgs); err != nil { return fmt.Errorf("failed to configure kube-proxy: %w", err) } - if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { + if err := setup.K8sAPIServerProxy(snap, response.APIServers, localhostAddress, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) } if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil { @@ -277,6 +284,13 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst return fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname()) } + var localhostAddress string + if nodeIP.To4() == nil { + localhostAddress = "[::1]" + } else { + localhostAddress = "127.0.0.1" + } + // Create directories if err := setup.EnsureAllDirectories(snap); err != nil { return fmt.Errorf("failed to create directories: %w", err) @@ -296,7 +310,7 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst // NOTE: Default certificate expiration is set to 20 years. certificates := pki.NewK8sDqlitePKI(pki.K8sDqlitePKIOpts{ Hostname: s.Name(), - IPSANs: []net.IP{{127, 0, 0, 1}}, + IPSANs: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, NotBefore: notBefore, NotAfter: notBefore.AddDate(20, 0, 0), AllowSelfSignedCA: true, @@ -395,14 +409,15 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst } // Generate kubeconfigs - if err := setup.SetupControlPlaneKubeconfigs(snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), *certificates); err != nil { + if err := setup.SetupControlPlaneKubeconfigs(snap.KubernetesConfigDir(), localhostAddress, cfg.APIServer.GetSecurePort(), *certificates); err != nil { return fmt.Errorf("failed to generate kubeconfigs: %w", err) } // Configure datastore switch cfg.Datastore.GetType() { case "k8s-dqlite": - if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil { + address := fmt.Sprintf("%s:%d", utils.ToIPString(nodeIP), cfg.Datastore.GetK8sDqlitePort()) + if err := setup.K8sDqlite(snap, address, nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil { return fmt.Errorf("failed to configure k8s-dqlite: %w", err) } case "external": @@ -417,7 +432,7 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil { return fmt.Errorf("failed to configure kubelet: %w", err) } - if err := setup.KubeProxy(ctx, snap, s.Name(), cfg.Network.GetPodCIDR(), bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil { + if err := setup.KubeProxy(ctx, snap, s.Name(), cfg.Network.GetPodCIDR(), localhostAddress, bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil { return fmt.Errorf("failed to configure kube-proxy: %w", err) } if err := setup.KubeControllerManager(snap, bootstrapConfig.ExtraNodeKubeControllerManagerArgs); err != nil { diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 0e164858f..075d2f6bb 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -48,6 +48,13 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri return fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname()) } + var localhostAddress string + if nodeIP.To4() == nil { + localhostAddress = "[::1]" + } else { + localhostAddress = "127.0.0.1" + } + // Create directories if err := setup.EnsureAllDirectories(snap); err != nil { return fmt.Errorf("failed to create directories: %w", err) @@ -64,7 +71,7 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri // NOTE: Default certificate expiration is set to 20 years. certificates := pki.NewK8sDqlitePKI(pki.K8sDqlitePKIOpts{ Hostname: s.Name(), - IPSANs: []net.IP{{127, 0, 0, 1}}, + IPSANs: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, NotBefore: notBefore, NotAfter: notBefore.AddDate(20, 0, 0), }) @@ -140,7 +147,7 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri return fmt.Errorf("failed to write control plane certificates: %w", err) } - if err := setup.SetupControlPlaneKubeconfigs(snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), *certificates); err != nil { + if err := setup.SetupControlPlaneKubeconfigs(snap.KubernetesConfigDir(), localhostAddress, cfg.APIServer.GetSecurePort(), *certificates); err != nil { return fmt.Errorf("failed to generate kubeconfigs: %w", err) } @@ -158,10 +165,16 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri } cluster := make([]string, len(members)) for _, member := range members { - cluster = append(cluster, fmt.Sprintf("%s:%d", member.Address.Addr(), cfg.Datastore.GetK8sDqlitePort())) + var address string + if member.Address.Addr().Is6() { + address = fmt.Sprintf("[%s]", member.Address.Addr()) + } else { + address = member.Address.Addr().String() + } + cluster = append(cluster, fmt.Sprintf("%s:%d", address, cfg.Datastore.GetK8sDqlitePort())) } - address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()) + address := fmt.Sprintf("%s:%d", utils.ToIPString(nodeIP), cfg.Datastore.GetK8sDqlitePort()) if err := setup.K8sDqlite(snap, address, cluster, joinConfig.ExtraNodeK8sDqliteArgs); err != nil { return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err) } @@ -177,7 +190,7 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil { return fmt.Errorf("failed to configure kubelet: %w", err) } - if err := setup.KubeProxy(ctx, snap, s.Name(), cfg.Network.GetPodCIDR(), joinConfig.ExtraNodeKubeProxyArgs); err != nil { + if err := setup.KubeProxy(ctx, snap, s.Name(), cfg.Network.GetPodCIDR(), localhostAddress, joinConfig.ExtraNodeKubeProxyArgs); err != nil { return fmt.Errorf("failed to configure kube-proxy: %w", err) } if err := setup.KubeControllerManager(snap, joinConfig.ExtraNodeKubeControllerManagerArgs); err != nil { diff --git a/src/k8s/pkg/k8sd/pki/k8sdqlite.go b/src/k8s/pkg/k8sd/pki/k8sdqlite.go index b1d74bc37..0e81d823d 100644 --- a/src/k8s/pkg/k8sd/pki/k8sdqlite.go +++ b/src/k8s/pkg/k8sd/pki/k8sdqlite.go @@ -64,7 +64,7 @@ func (c *K8sDqlitePKI) CompleteCertificates() error { return fmt.Errorf("k8s-dqlite certificate not specified and generating self-signed certificates is not allowed") } - template, err := pkiutil.GenerateCertificate(pkix.Name{CommonName: "k8s"}, c.notBefore, c.notAfter, false, append(c.dnsSANs, c.hostname), append(c.ipSANs, net.IP{127, 0, 0, 1})) + template, err := pkiutil.GenerateCertificate(pkix.Name{CommonName: "k8s"}, c.notBefore, c.notAfter, false, append(c.dnsSANs, c.hostname), append(c.ipSANs, net.ParseIP("127.0.0.1"), net.ParseIP("::1"))) if err != nil { return fmt.Errorf("failed to generate k8s-dqlite certificate: %w", err) } diff --git a/src/k8s/pkg/k8sd/pki/worker.go b/src/k8s/pkg/k8sd/pki/worker.go index d945f053f..590b53a99 100644 --- a/src/k8s/pkg/k8sd/pki/worker.go +++ b/src/k8s/pkg/k8sd/pki/worker.go @@ -48,7 +48,7 @@ func (c *ControlPlanePKI) CompleteWorkerNodePKI(hostname string, nodeIP net.IP, c.notAfter, false, []string{hostname}, - []net.IP{{127, 0, 0, 1}, nodeIP}, + []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1"), nodeIP}, ) if err != nil { return nil, fmt.Errorf("failed to generate kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go index 8a54bb60a..9b3a3b886 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go @@ -11,7 +11,7 @@ import ( ) // K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy. -func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*string) error { +func K8sAPIServerProxy(snap snap.Snap, servers []string, localhostAddress string, extraArgs map[string]*string) error { configFile := filepath.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json") if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil { return fmt.Errorf("failed to write proxy configuration file: %w", err) @@ -20,7 +20,7 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*s if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", map[string]string{ "--endpoints": configFile, "--kubeconfig": filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), - "--listen": "127.0.0.1:6443", + "--listen": fmt.Sprintf("%s:6443", localhostAddress), }, nil); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go index 3236e464b..ded12d037 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go @@ -27,7 +27,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) - g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", nil)).To(Succeed()) tests := []struct { key string @@ -61,7 +61,7 @@ func TestK8sApiServerProxy(t *testing.T) { "--listen": nil, // This should trigger a delete "--my-extra-arg": utils.Pointer("my-extra-val"), } - g.Expect(setup.K8sAPIServerProxy(s, nil, extraArgs)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", extraArgs)).To(Succeed()) tests := []struct { key string @@ -98,7 +98,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceExtraConfigDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", nil)).ToNot(Succeed()) }) t.Run("MissingServiceArgumentsDir", func(t *testing.T) { @@ -107,7 +107,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", nil)).ToNot(Succeed()) }) t.Run("JSONFileContent", func(t *testing.T) { @@ -118,7 +118,7 @@ func TestK8sApiServerProxy(t *testing.T) { endpoints := []string{"192.168.0.1", "192.168.0.2", "192.168.0.3"} fileName := filepath.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json") - g.Expect(setup.K8sAPIServerProxy(s, endpoints, nil)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, endpoints, "127.0.0.1", nil)).To(Succeed()) b, err := os.ReadFile(fileName) g.Expect(err).NotTo(HaveOccurred()) diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy.go b/src/k8s/pkg/k8sd/setup/kube_proxy.go index 0e64a60aa..ff29b0443 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy.go @@ -12,10 +12,10 @@ import ( ) // KubeProxy configures kube-proxy on the local node. -func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string, extraArgs map[string]*string) error { +func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string, localhostAddress string, extraArgs map[string]*string) error { serviceArgs := map[string]string{ "--cluster-cidr": podCIDR, - "--healthz-bind-address": "127.0.0.1", + "--healthz-bind-address": fmt.Sprintf("%s:10256", localhostAddress), "--kubeconfig": filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), "--profiling": "false", } diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go index 1852c8478..b6f77bb8e 100644 --- a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go @@ -31,7 +31,7 @@ func TestKubeProxy(t *testing.T) { g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) t.Run("Args", func(t *testing.T) { - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", "127.0.0.1", nil)).To(BeNil()) for key, expectedVal := range map[string]string{ "--cluster-cidr": "10.1.0.0/16", @@ -55,7 +55,7 @@ func TestKubeProxy(t *testing.T) { "--healthz-bind-address": nil, "--my-extra-arg": utils.Pointer("my-extra-val"), } - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", extraArgs)).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", "127.0.0.1", extraArgs)).To(BeNil()) for key, expectedVal := range map[string]string{ "--cluster-cidr": "10.1.0.0/16", @@ -80,7 +80,7 @@ func TestKubeProxy(t *testing.T) { s.Mock.OnLXD = true t.Run("ArgsOnLXD", func(t *testing.T) { - g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", "127.0.0.1", nil)).To(BeNil()) for key, expectedVal := range map[string]string{ "--conntrack-max-per-core": "0", @@ -103,7 +103,7 @@ func TestKubeProxy(t *testing.T) { s.Mock.ServiceArgumentsDir = filepath.Join(dir, "k8s") g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) - g.Expect(setup.KubeProxy(context.Background(), s, "dev", "10.1.0.0/16", nil)).To(BeNil()) + g.Expect(setup.KubeProxy(context.Background(), s, "dev", "10.1.0.0/16", "127.0.0.1", nil)).To(BeNil()) val, err := snaputil.GetServiceArgument(s, "kube-proxy", "--hostname-override") g.Expect(err).To(BeNil()) diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go index ff4cd9e0f..b6a5115dc 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet.go +++ b/src/k8s/pkg/k8sd/setup/kubelet.go @@ -81,7 +81,7 @@ func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, args["--cluster-domain"] = clusterDomain } if nodeIP != nil && !nodeIP.IsLoopback() { - args["--node-ip"] = nodeIP.String() + args["--node-ip"] = utils.ToIPString(nodeIP) } if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", args, nil); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) diff --git a/src/k8s/pkg/k8sd/setup/util_kubeconfig.go b/src/k8s/pkg/k8sd/setup/util_kubeconfig.go index 3793a97f0..413fa04a5 100644 --- a/src/k8s/pkg/k8sd/setup/util_kubeconfig.go +++ b/src/k8s/pkg/k8sd/setup/util_kubeconfig.go @@ -57,7 +57,7 @@ func KubeconfigString(url string, caPEM string, crtPEM string, keyPEM string) (s } // SetupControlPlaneKubeconfigs writes kubeconfig files for the control plane components. -func SetupControlPlaneKubeconfigs(kubeConfigDir string, securePort int, pki pki.ControlPlanePKI) error { +func SetupControlPlaneKubeconfigs(kubeConfigDir string, localhostAddress string, securePort int, pki pki.ControlPlanePKI) error { for _, kubeconfig := range []struct { file string crt string @@ -69,7 +69,7 @@ func SetupControlPlaneKubeconfigs(kubeConfigDir string, securePort int, pki pki. {file: "scheduler.conf", crt: pki.KubeSchedulerClientCert, key: pki.KubeSchedulerClientKey}, {file: "kubelet.conf", crt: pki.KubeletClientCert, key: pki.KubeletClientKey}, } { - if err := Kubeconfig(filepath.Join(kubeConfigDir, kubeconfig.file), fmt.Sprintf("127.0.0.1:%d", securePort), pki.CACert, kubeconfig.crt, kubeconfig.key); err != nil { + if err := Kubeconfig(filepath.Join(kubeConfigDir, kubeconfig.file), fmt.Sprintf("%s:%d", localhostAddress, securePort), pki.CACert, kubeconfig.crt, kubeconfig.key); err != nil { return fmt.Errorf("failed to write kubeconfig %s: %w", kubeconfig.file, err) } } diff --git a/src/k8s/pkg/proxy/userspace.go b/src/k8s/pkg/proxy/userspace.go index 4fa143592..4082d7947 100644 --- a/src/k8s/pkg/proxy/userspace.go +++ b/src/k8s/pkg/proxy/userspace.go @@ -27,6 +27,8 @@ import ( "net" "sync" "time" + + "github.com/canonical/k8s/pkg/utils" ) type remote struct { @@ -78,7 +80,8 @@ func (tp *tcpproxy) Run() error { tp.MonitorInterval = 5 * time.Minute } for _, srv := range tp.Endpoints { - addr := fmt.Sprintf("%s:%d", srv.Target, srv.Port) + ip := net.ParseIP(srv.Target) + addr := fmt.Sprintf("%s:%d", utils.ToIPString(ip), srv.Port) tp.remotes = append(tp.remotes, &remote{srv: srv, addr: addr}) } diff --git a/src/k8s/pkg/utils/cidr.go b/src/k8s/pkg/utils/cidr.go index 124f75aea..e5fe01bba 100644 --- a/src/k8s/pkg/utils/cidr.go +++ b/src/k8s/pkg/utils/cidr.go @@ -125,3 +125,20 @@ func ParseCIDRs(CIDRstring string) (string, string, error) { } return ipv4CIDR, ipv6CIDR, nil } + +// IsIPv4 returns true if the address is a valid IPv4 address, false otherwise. +// The address may contain a port number. +func IsIPv4(address string) bool { + ip := strings.Split(address, ":")[0] + parsedIP := net.ParseIP(ip) + return parsedIP != nil && parsedIP.To4() != nil +} + +// ToIPString returns the string representation of an IP address. +// If the IP address is an IPv6 address, it is enclosed in square brackets. +func ToIPString(ip net.IP) string { + if ip.To4() != nil { + return ip.String() + } + return "[" + ip.String() + "]" +} diff --git a/src/k8s/pkg/utils/cidr_test.go b/src/k8s/pkg/utils/cidr_test.go index 3dfbe3941..1d0e9da96 100644 --- a/src/k8s/pkg/utils/cidr_test.go +++ b/src/k8s/pkg/utils/cidr_test.go @@ -164,3 +164,44 @@ func TestParseCIDRs(t *testing.T) { }) } } + +func TestIsIPv4(t *testing.T) { + tests := []struct { + address string + expected bool + }{ + {"192.168.1.1:80", true}, + {"127.0.0.1", true}, + {"::1", false}, + {"[fe80::1]:80", false}, + {"256.256.256.256", false}, // Invalid IPv4 address + } + + for _, tc := range tests { + t.Run(tc.address, func(t *testing.T) { + g := NewWithT(t) + result := utils.IsIPv4(tc.address) + g.Expect(result).To(Equal(tc.expected)) + }) + } +} + +func TestToIPString(t *testing.T) { + tests := []struct { + ip net.IP + expected string + }{ + {net.ParseIP("192.168.1.1"), "192.168.1.1"}, + {net.ParseIP("::1"), "[::1]"}, + {net.ParseIP("fe80::1"), "[fe80::1]"}, + {net.ParseIP("127.0.0.1"), "127.0.0.1"}, + } + + for _, tc := range tests { + t.Run(tc.expected, func(t *testing.T) { + g := NewWithT(t) + result := utils.ToIPString(tc.ip) + g.Expect(result).To(Equal(tc.expected)) + }) + } +} diff --git a/tests/integration/templates/bootstrap-ipv6-only.yaml b/tests/integration/templates/bootstrap-ipv6-only.yaml new file mode 100644 index 000000000..442857805 --- /dev/null +++ b/tests/integration/templates/bootstrap-ipv6-only.yaml @@ -0,0 +1,16 @@ +cluster-config: + network: + enabled: true + dns: + enabled: true + cluster-domain: cluster.local + local-storage: + enabled: true + local-path: /storage/path + default: false + gateway: + enabled: true + metrics-server: + enabled: true +pod-cidr: fd01::/108 +service-cidr: fd98::/108 diff --git a/tests/integration/templates/etcd/etcd-tls.conf b/tests/integration/templates/etcd/etcd-tls.conf index 59243ba98..83cd61a86 100644 --- a/tests/integration/templates/etcd/etcd-tls.conf +++ b/tests/integration/templates/etcd/etcd-tls.conf @@ -18,6 +18,6 @@ [alt_names] IP.1 = 127.0.0.1 - IP.2 = $IP + IP.2 = $IP DNS.1 = localhost DNS.2 = $NAME diff --git a/tests/integration/templates/nginx-ipv6-only.yaml b/tests/integration/templates/nginx-ipv6-only.yaml new file mode 100644 index 000000000..93a5647a2 --- /dev/null +++ b/tests/integration/templates/nginx-ipv6-only.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginxipv6 +spec: + selector: + matchLabels: + run: nginxipv6 + replicas: 1 + template: + metadata: + labels: + run: nginxipv6 + spec: + containers: + - name: nginxipv6 + image: rocks.canonical.com/cdk/diverdane/nginxdualstack:1.0.0 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-ipv6 + labels: + run: nginxipv6 +spec: + type: NodePort + ipFamilies: + - IPv6 + ipFamilyPolicy: SingleStack + ports: + - port: 80 + protocol: TCP + selector: + run: nginxipv6 diff --git a/tests/integration/tests/test_dualstack.py b/tests/integration/tests/test_dualstack.py deleted file mode 100644 index 53d586504..000000000 --- a/tests/integration/tests/test_dualstack.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright 2024 Canonical, Ltd. -# -import logging -from ipaddress import IPv4Address, IPv6Address, ip_address -from typing import List - -import pytest -from test_util import config, harness, util - -LOG = logging.getLogger(__name__) - - -@pytest.mark.node_count(1) -@pytest.mark.bootstrap_config( - (config.MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text() -) -@pytest.mark.dualstack() -def test_dualstack(instances: List[harness.Instance]): - main = instances[0] - dualstack_config = (config.MANIFESTS_DIR / "nginx-dualstack.yaml").read_text() - - # Deploy nginx with dualstack service - main.exec( - ["k8s", "kubectl", "apply", "-f", "-"], input=str.encode(dualstack_config) - ) - addresses = ( - util.stubbornly(retries=5, delay_s=3) - .on(main) - .exec( - [ - "k8s", - "kubectl", - "get", - "svc", - "nginx-dualstack", - "-o", - "jsonpath='{.spec.clusterIPs[*]}'", - ], - text=True, - capture_output=True, - ) - .stdout - ) - - for ip in addresses.split(): - addr = ip_address(ip.strip("'")) - if isinstance(addr, IPv6Address): - address = f"http://[{str(addr)}]" - elif isinstance(addr, IPv4Address): - address = f"http://{str(addr)}" - else: - pytest.fail(f"Unknown IP address type: {addr}") - - # need to shell out otherwise this runs into permission errors - util.stubbornly(retries=3, delay_s=1).on(main).exec( - ["curl", address], shell=True - ) diff --git a/tests/integration/tests/test_networking.py b/tests/integration/tests/test_networking.py new file mode 100644 index 000000000..c1e30ea95 --- /dev/null +++ b/tests/integration/tests/test_networking.py @@ -0,0 +1,131 @@ +# +# Copyright 2024 Canonical, Ltd. +# +import logging +import os +from ipaddress import IPv4Address, IPv6Address, ip_address +from typing import List + +import pytest +from test_util import config, harness, util + +LOG = logging.getLogger(__name__) + + +@pytest.mark.node_count(1) +@pytest.mark.bootstrap_config( + (config.MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text() +) +@pytest.mark.dualstack() +def test_dualstack(instances: List[harness.Instance]): + main = instances[0] + dualstack_config = (config.MANIFESTS_DIR / "nginx-dualstack.yaml").read_text() + + # Deploy nginx with dualstack service + main.exec( + ["k8s", "kubectl", "apply", "-f", "-"], input=str.encode(dualstack_config) + ) + addresses = ( + util.stubbornly(retries=5, delay_s=3) + .on(main) + .exec( + [ + "k8s", + "kubectl", + "get", + "svc", + "nginx-dualstack", + "-o", + "jsonpath='{.spec.clusterIPs[*]}'", + ], + text=True, + capture_output=True, + ) + .stdout + ) + + for ip in addresses.split(): + addr = ip_address(ip.strip("'")) + if isinstance(addr, IPv6Address): + address = f"http://[{str(addr)}]" + elif isinstance(addr, IPv4Address): + address = f"http://{str(addr)}" + else: + pytest.fail(f"Unknown IP address type: {addr}") + + # need to shell out otherwise this runs into permission errors + util.stubbornly(retries=3, delay_s=1).on(main).exec( + ["curl", address], shell=True + ) + + +@pytest.mark.node_count(3) +@pytest.mark.disable_k8s_bootstrapping() +@pytest.mark.dualstack() +@pytest.mark.skipif( + os.getenv("TEST_IPV6_ONLY") in ["false", None], + reason="IPv6 is currently only supported for moonray/calico", +) +def test_ipv6_only(instances: List[harness.Instance]): + main = instances[0] + joining_cp = instances[1] + joining_worker = instances[2] + + ipv6_bootstrap_config = ( + config.MANIFESTS_DIR / "bootstrap-ipv6-only.yaml" + ).read_text() + + ipv6_address = util.get_global_unicast_ipv6(main) + main.exec( + ["k8s", "bootstrap", "--file", "-", "--address", ipv6_address], + input=str.encode(ipv6_bootstrap_config), + ) + + join_token = util.get_join_token(main, joining_cp) + ipv6_address = util.get_global_unicast_ipv6(joining_cp) + joining_cp.exec(["k8s", "join-cluster", join_token, "--address", ipv6_address]) + + join_token_worker = util.get_join_token(main, joining_worker, "--worker") + ipv6_address = util.get_global_unicast_ipv6(joining_worker) + joining_worker.exec( + ["k8s", "join-cluster", join_token_worker, "--address", ipv6_address] + ) + + # Deploy nginx with ipv6 service + ipv6_config = (config.MANIFESTS_DIR / "nginx-ipv6-only.yaml").read_text() + main.exec(["k8s", "kubectl", "apply", "-f", "-"], input=str.encode(ipv6_config)) + addresses = ( + util.stubbornly(retries=5, delay_s=3) + .on(main) + .exec( + [ + "k8s", + "kubectl", + "get", + "svc", + "nginx-ipv6", + "-o", + "jsonpath='{.spec.clusterIPs[*]}'", + ], + text=True, + capture_output=True, + ) + .stdout + ) + + for ip in addresses.split(): + addr = ip_address(ip.strip("'")) + if isinstance(addr, IPv6Address): + address = f"http://[{str(addr)}]" + elif isinstance(addr, IPv4Address): + assert False, "IPv4 address found in IPv6-only cluster" + else: + pytest.fail(f"Unknown IP address type: {addr}") + + # need to shell out otherwise this runs into permission errors + util.stubbornly(retries=3, delay_s=1).on(main).exec( + ["curl", address], shell=True + ) + + # This might take a while + util.stubbornly(retries=30, delay_s=20).until(util.ready_nodes(main) == 3) diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 8d9875d18..754dc744b 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -3,6 +3,7 @@ # import json import logging +import re import shlex import subprocess from functools import partial @@ -261,3 +262,24 @@ def get_default_ip(instance: harness.Instance): ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True ) return p.stdout.decode().split(" ")[8] + + +def get_global_unicast_ipv6(instance: harness.Instance, interface="eth0") -> str: + # --- + # 2: eth0: mtu 1500 qdisc fq_codel state UP group default qlen 1000 + # link/ether 00:16:3e:0f:4d:1e brd ff:ff:ff:ff:ff:ff + # inet + # inet6 fe80::216:3eff:fe0f:4d1e/64 scope link + # --- + # Fetching the global unicast address for the specified interface, e.g. fe80::216:3eff:fe0f:4d1e + result = instance.exec( + ["ip", "-6", "addr", "show", "dev", interface, "scope", "global"], + capture_output=True, + text=True, + ) + output = result.stdout + ipv6_regex = re.compile(r"inet6\s+([a-f0-9:]+)\/[0-9]*\s+scope global") + match = ipv6_regex.search(output) + if match: + return match.group(1) + return None