From 9e0a43c44482c0ab0c7e1d43852715e64ee2ef12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20Tekin=20=C3=96z?= Date: Fri, 27 Sep 2024 21:20:25 +0300 Subject: [PATCH] Point cilium to talk to the local apiserver or apiserver-proxy (#697) --- src/k8s/pkg/k8sd/api/certificates_refresh.go | 4 +- src/k8s/pkg/k8sd/app/hooks_bootstrap.go | 26 +++++++-- src/k8s/pkg/k8sd/app/hooks_join.go | 2 +- src/k8s/pkg/k8sd/controllers/feature.go | 2 +- src/k8s/pkg/k8sd/features/calico/network.go | 12 ++-- .../pkg/k8sd/features/calico/network_test.go | 52 ++++++++++++------ src/k8s/pkg/k8sd/features/cilium/network.go | 17 ++++-- .../pkg/k8sd/features/cilium/network_test.go | 55 +++++++++++++------ src/k8s/pkg/k8sd/features/interface.go | 8 +-- src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go | 4 +- .../k8sd/setup/k8s_apiserver_proxy_test.go | 12 ++-- src/k8s/pkg/k8sd/setup/kube_apiserver.go | 5 +- src/k8s/pkg/k8sd/setup/kube_apiserver_test.go | 12 ++-- .../pkg/k8sd/types/cluster_config_merge.go | 1 + .../pkg/k8sd/types/cluster_config_network.go | 16 +++--- 15 files changed, 147 insertions(+), 81 deletions(-) diff --git a/src/k8s/pkg/k8sd/api/certificates_refresh.go b/src/k8s/pkg/k8sd/api/certificates_refresh.go index b08dada6b..4e3694114 100644 --- a/src/k8s/pkg/k8sd/api/certificates_refresh.go +++ b/src/k8s/pkg/k8sd/api/certificates_refresh.go @@ -282,10 +282,10 @@ func refreshCertsRunWorker(s state.State, r *http.Request, snap snap.Snap) respo } // Kubeconfigs - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), fmt.Sprintf("%s:%d", localhostAddress, clusterConfig.APIServer.GetSecurePort()), certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { return response.InternalError(fmt.Errorf("failed to generate kubelet kubeconfig: %w", err)) } - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), fmt.Sprintf("%s:%d", localhostAddress, clusterConfig.APIServer.GetSecurePort()), certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { return response.InternalError(fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err)) } diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index ccd828d1d..2e35b9be6 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -10,6 +10,8 @@ import ( "net" "net/http" "path/filepath" + "strconv" + "strings" "time" apiv1 "github.com/canonical/k8s-snap-api/api/v1" @@ -184,11 +186,22 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT localhostAddress = "127.0.0.1" } + port := "6443" + if len(response.APIServers) == 0 { + return fmt.Errorf("no APIServers found in worker node info") + } + // Get the secure port from the first APIServer since they should all be the same. + port = response.APIServers[0][strings.LastIndex(response.APIServers[0], ":")+1:] + securePort, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("failed to parse apiserver secure port: %w", err) + } + // Kubeconfigs - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), fmt.Sprintf("%s:%d", localhostAddress, securePort), certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { return fmt.Errorf("failed to generate kubelet kubeconfig: %w", err) } - if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), fmt.Sprintf("%s:6443", localhostAddress), certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { + if err := setup.Kubeconfig(filepath.Join(snap.KubernetesConfigDir(), "proxy.conf"), fmt.Sprintf("%s:%d", localhostAddress, securePort), certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { return fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err) } @@ -203,6 +216,9 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT // TODO(neoaggelos): We should be explicit here and try to avoid having worker nodes use // or set other cluster configuration keys by accident. cfg := types.ClusterConfig{ + APIServer: types.APIServer{ + SecurePort: utils.Pointer(securePort), + }, Network: types.Network{ PodCIDR: utils.Pointer(response.PodCIDR), ServiceCIDR: utils.Pointer(response.ServiceCIDR), @@ -239,7 +255,7 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT if err := setup.KubeProxy(ctx, snap, s.Name(), response.PodCIDR, localhostAddress, joinConfig.ExtraNodeKubeProxyArgs); err != nil { return fmt.Errorf("failed to configure kube-proxy: %w", err) } - if err := setup.K8sAPIServerProxy(snap, response.APIServers, localhostAddress, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { + if err := setup.K8sAPIServerProxy(snap, response.APIServers, securePort, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil { return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) } if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil { @@ -291,6 +307,8 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst localhostAddress = "127.0.0.1" } + cfg.Network.LocalhostAddress = utils.Pointer(localhostAddress) + // Create directories if err := setup.EnsureAllDirectories(snap); err != nil { return fmt.Errorf("failed to create directories: %w", err) @@ -441,7 +459,7 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst if err := setup.KubeScheduler(snap, bootstrapConfig.ExtraNodeKubeSchedulerArgs); err != nil { return fmt.Errorf("failed to configure kube-scheduler: %w", err) } - if err := setup.KubeAPIServer(snap, nodeIP, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil { + if err := setup.KubeAPIServer(snap, cfg.APIServer.GetSecurePort(), nodeIP, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil { return fmt.Errorf("failed to configure kube-apiserver: %w", err) } diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 075d2f6bb..ac67b526f 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -199,7 +199,7 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri if err := setup.KubeScheduler(snap, joinConfig.ExtraNodeKubeSchedulerArgs); err != nil { return fmt.Errorf("failed to configure kube-scheduler: %w", err) } - if err := setup.KubeAPIServer(snap, nodeIP, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil { + if err := setup.KubeAPIServer(snap, cfg.APIServer.GetSecurePort(), nodeIP, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil { return fmt.Errorf("failed to configure kube-apiserver: %w", err) } diff --git a/src/k8s/pkg/k8sd/controllers/feature.go b/src/k8s/pkg/k8sd/controllers/feature.go index 909a43bc3..4013e0cbc 100644 --- a/src/k8s/pkg/k8sd/controllers/feature.go +++ b/src/k8s/pkg/k8sd/controllers/feature.go @@ -79,7 +79,7 @@ func (c *FeatureController) Run( ctx = log.NewContext(ctx, log.FromContext(ctx).WithValues("controller", "feature")) go c.reconcileLoop(ctx, getClusterConfig, setFeatureStatus, features.Network, c.triggerNetworkCh, c.reconciledNetworkCh, func(cfg types.ClusterConfig) (types.FeatureStatus, error) { - return features.Implementation.ApplyNetwork(ctx, c.snap, cfg.Network, cfg.Annotations) + return features.Implementation.ApplyNetwork(ctx, c.snap, cfg.APIServer, cfg.Network, cfg.Annotations) }) go c.reconcileLoop(ctx, getClusterConfig, setFeatureStatus, features.Gateway, c.triggerGatewayCh, c.reconciledGatewayCh, func(cfg types.ClusterConfig) (types.FeatureStatus, error) { diff --git a/src/k8s/pkg/k8sd/features/calico/network.go b/src/k8s/pkg/k8sd/features/calico/network.go index 82e98c788..f89651df2 100644 --- a/src/k8s/pkg/k8sd/features/calico/network.go +++ b/src/k8s/pkg/k8sd/features/calico/network.go @@ -17,16 +17,16 @@ const ( deleteFailedMsgTmpl = "Failed to delete Calico, the error was: %v" ) -// ApplyNetwork will deploy Calico when cfg.Enabled is true. -// ApplyNetwork will remove Calico when cfg.Enabled is false. +// ApplyNetwork will deploy Calico when network.Enabled is true. +// ApplyNetwork will remove Calico when network.Enabled is false. // ApplyNetwork will always return a FeatureStatus indicating the current status of the // deployment. // ApplyNetwork returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. -func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annotations types.Annotations) (types.FeatureStatus, error) { +func ApplyNetwork(ctx context.Context, snap snap.Snap, apiserver types.APIServer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { m := snap.HelmClient() - if !cfg.GetEnabled() { + if !network.GetEnabled() { if _, err := m.Apply(ctx, ChartCalico, helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to uninstall network: %w", err) return types.FeatureStatus{ @@ -54,7 +54,7 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annota } podIpPools := []map[string]any{} - ipv4PodCIDR, ipv6PodCIDR, err := utils.ParseCIDRs(cfg.GetPodCIDR()) + ipv4PodCIDR, ipv6PodCIDR, err := utils.ParseCIDRs(network.GetPodCIDR()) if err != nil { err = fmt.Errorf("invalid pod cidr: %w", err) return types.FeatureStatus{ @@ -79,7 +79,7 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annota } serviceCIDRs := []string{} - ipv4ServiceCIDR, ipv6ServiceCIDR, err := utils.ParseCIDRs(cfg.GetServiceCIDR()) + ipv4ServiceCIDR, ipv6ServiceCIDR, err := utils.ParseCIDRs(network.GetServiceCIDR()) if err != nil { err = fmt.Errorf("invalid service cidr: %v", err) return types.FeatureStatus{ diff --git a/src/k8s/pkg/k8sd/features/calico/network_test.go b/src/k8s/pkg/k8sd/features/calico/network_test.go index c0a324028..75d31e0af 100644 --- a/src/k8s/pkg/k8sd/features/calico/network_test.go +++ b/src/k8s/pkg/k8sd/features/calico/network_test.go @@ -39,11 +39,14 @@ func TestDisabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(false), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := calico.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := calico.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) @@ -65,11 +68,14 @@ func TestDisabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(false), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := calico.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := calico.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) @@ -94,12 +100,15 @@ func TestEnabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(true), PodCIDR: ptr.To("invalid-cidr"), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := calico.ApplyNetwork(context.Background(), snapM, cfg, defaultAnnotations) + status, err := calico.ApplyNetwork(context.Background(), snapM, apiserver, network, defaultAnnotations) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) @@ -115,13 +124,16 @@ func TestEnabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(true), PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), ServiceCIDR: ptr.To("invalid-cidr"), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := calico.ApplyNetwork(context.Background(), snapM, cfg, defaultAnnotations) + status, err := calico.ApplyNetwork(context.Background(), snapM, apiserver, network, defaultAnnotations) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) @@ -140,13 +152,16 @@ func TestEnabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(true), PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), ServiceCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := calico.ApplyNetwork(context.Background(), snapM, cfg, defaultAnnotations) + status, err := calico.ApplyNetwork(context.Background(), snapM, apiserver, network, defaultAnnotations) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) @@ -157,7 +172,7 @@ func TestEnabled(t *testing.T) { callArgs := helmM.ApplyCalledWith[0] g.Expect(callArgs.Chart).To(Equal(calico.ChartCalico)) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) - validateValues(t, callArgs.Values, cfg) + validateValues(t, callArgs.Values, network) }) t.Run("Success", func(t *testing.T) { g := NewWithT(t) @@ -168,13 +183,16 @@ func TestEnabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(true), PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), ServiceCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := calico.ApplyNetwork(context.Background(), snapM, cfg, defaultAnnotations) + status, err := calico.ApplyNetwork(context.Background(), snapM, apiserver, network, defaultAnnotations) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) @@ -185,17 +203,17 @@ func TestEnabled(t *testing.T) { callArgs := helmM.ApplyCalledWith[0] g.Expect(callArgs.Chart).To(Equal(calico.ChartCalico)) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) - validateValues(t, callArgs.Values, cfg) + validateValues(t, callArgs.Values, network) }) } -func validateValues(t *testing.T, values map[string]any, cfg types.Network) { +func validateValues(t *testing.T, values map[string]any, network types.Network) { g := NewWithT(t) - podIPv4CIDR, podIPv6CIDR, err := utils.ParseCIDRs(cfg.GetPodCIDR()) + podIPv4CIDR, podIPv6CIDR, err := utils.ParseCIDRs(network.GetPodCIDR()) g.Expect(err).ToNot(HaveOccurred()) - svcIPv4CIDR, svcIPv6CIDR, err := utils.ParseCIDRs(cfg.GetServiceCIDR()) + svcIPv4CIDR, svcIPv6CIDR, err := utils.ParseCIDRs(network.GetServiceCIDR()) g.Expect(err).ToNot(HaveOccurred()) // calico network diff --git a/src/k8s/pkg/k8sd/features/cilium/network.go b/src/k8s/pkg/k8sd/features/cilium/network.go index 4418e8b1d..e0573d72b 100644 --- a/src/k8s/pkg/k8sd/features/cilium/network.go +++ b/src/k8s/pkg/k8sd/features/cilium/network.go @@ -17,18 +17,18 @@ const ( networkDeployFailedMsgTmpl = "Failed to deploy Cilium Network, the error was: %v" ) -// ApplyNetwork will deploy Cilium when cfg.Enabled is true. -// ApplyNetwork will remove Cilium when cfg.Enabled is false. +// ApplyNetwork will deploy Cilium when network.Enabled is true. +// ApplyNetwork will remove Cilium when network.Enabled is false. // ApplyNetwork requires that bpf and cgroups2 are already mounted and available when running under strict snap confinement. If they are not, it will fail (since Cilium will not have the required permissions to mount them). // ApplyNetwork requires that `/sys` is mounted as a shared mount when running under classic snap confinement. This is to ensure that Cilium will be able to automatically mount bpf and cgroups2 on the pods. // ApplyNetwork will always return a FeatureStatus indicating the current status of the // deployment. // ApplyNetwork returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. -func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ types.Annotations) (types.FeatureStatus, error) { +func ApplyNetwork(ctx context.Context, snap snap.Snap, apiserver types.APIServer, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { m := snap.HelmClient() - if !cfg.GetEnabled() { + if !network.GetEnabled() { if _, err := m.Apply(ctx, ChartCilium, helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to uninstall network: %w", err) return types.FeatureStatus{ @@ -44,7 +44,7 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ type }, nil } - ipv4CIDR, ipv6CIDR, err := utils.ParseCIDRs(cfg.GetPodCIDR()) + ipv4CIDR, ipv6CIDR, err := utils.ParseCIDRs(network.GetPodCIDR()) if err != nil { err = fmt.Errorf("invalid kube-proxy --cluster-cidr value: %v", err) return types.FeatureStatus{ @@ -87,10 +87,17 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, _ type "clusterPoolIPv6PodCIDRList": ipv6CIDR, }, }, + // https://docs.cilium.io/en/v1.15/network/kubernetes/kubeproxy-free/#kube-proxy-hybrid-modes "nodePort": map[string]any{ "enabled": true, + // kube-proxy also binds to the same port for health checks so we need to disable it + "enableHealthCheck": false, }, "disableEnvoyVersionCheck": true, + // socketLB requires an endpoint to the apiserver that's not managed by the kube-proxy + // so we point to the localhost:secureport to talk to either the kube-apiserver or the kube-apiserver-proxy + "k8sServiceHost": network.GetLocalhostAddress(), + "k8sServicePort": apiserver.GetSecurePort(), } if snap.Strict() { diff --git a/src/k8s/pkg/k8sd/features/cilium/network_test.go b/src/k8s/pkg/k8sd/features/cilium/network_test.go index 175cd95a0..978bebc0e 100644 --- a/src/k8s/pkg/k8sd/features/cilium/network_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/network_test.go @@ -32,11 +32,14 @@ func TestNetworkDisabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(false), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := cilium.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := cilium.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) @@ -58,11 +61,14 @@ func TestNetworkDisabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(false), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := cilium.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := cilium.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) @@ -87,12 +93,15 @@ func TestNetworkEnabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ + network := types.Network{ Enabled: ptr.To(true), PodCIDR: ptr.To("invalid-cidr"), } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), + } - status, err := cilium.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := cilium.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) @@ -109,12 +118,16 @@ func TestNetworkEnabled(t *testing.T) { Strict: true, }, } - cfg := types.Network{ - Enabled: ptr.To(true), - PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), + network := types.Network{ + Enabled: ptr.To(true), + LocalhostAddress: ptr.To("127.0.0.1"), + PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), + } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), } - status, err := cilium.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := cilium.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) @@ -125,7 +138,7 @@ func TestNetworkEnabled(t *testing.T) { callArgs := helmM.ApplyCalledWith[0] g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) - validateNetworkValues(t, callArgs.Values, cfg, snapM) + validateNetworkValues(t, callArgs.Values, network, snapM) }) t.Run("HelmApplyFails", func(t *testing.T) { g := NewWithT(t) @@ -139,12 +152,16 @@ func TestNetworkEnabled(t *testing.T) { HelmClient: helmM, }, } - cfg := types.Network{ - Enabled: ptr.To(true), - PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), + network := types.Network{ + Enabled: ptr.To(true), + LocalhostAddress: ptr.To("127.0.0.1"), + PodCIDR: ptr.To("192.0.2.0/24,2001:db8::/32"), + } + apiserver := types.APIServer{ + SecurePort: ptr.To(6443), } - status, err := cilium.ApplyNetwork(context.Background(), snapM, cfg, nil) + status, err := cilium.ApplyNetwork(context.Background(), snapM, apiserver, network, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) @@ -155,15 +172,15 @@ func TestNetworkEnabled(t *testing.T) { callArgs := helmM.ApplyCalledWith[0] g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) - validateNetworkValues(t, callArgs.Values, cfg, snapM) + validateNetworkValues(t, callArgs.Values, network, snapM) }) } -func validateNetworkValues(t *testing.T, values map[string]any, cfg types.Network, snap snap.Snap) { +func validateNetworkValues(t *testing.T, values map[string]any, network types.Network, snap snap.Snap) { t.Helper() g := NewWithT(t) - ipv4CIDR, ipv6CIDR, err := utils.ParseCIDRs(cfg.GetPodCIDR()) + ipv4CIDR, ipv6CIDR, err := utils.ParseCIDRs(network.GetPodCIDR()) g.Expect(err).ToNot(HaveOccurred()) bpfMount, err := utils.GetMountPath("bpf") @@ -177,6 +194,8 @@ func validateNetworkValues(t *testing.T, values map[string]any, cfg types.Networ g.Expect(values["cgroup"].(map[string]any)["hostRoot"]).To(Equal(cgrMount)) } + g.Expect(values["k8sServiceHost"]).To(Equal("127.0.0.1")) + g.Expect(values["k8sServicePort"]).To(Equal(6443)) g.Expect(values["ipam"].(map[string]any)["operator"].(map[string]any)["clusterPoolIPv4PodCIDRList"]).To(Equal(ipv4CIDR)) g.Expect(values["ipam"].(map[string]any)["operator"].(map[string]any)["clusterPoolIPv6PodCIDRList"]).To(Equal(ipv6CIDR)) g.Expect(values["ipv4"].(map[string]any)["enabled"]).To(Equal((ipv4CIDR != ""))) diff --git a/src/k8s/pkg/k8sd/features/interface.go b/src/k8s/pkg/k8sd/features/interface.go index 18eb4a978..ea3651a75 100644 --- a/src/k8s/pkg/k8sd/features/interface.go +++ b/src/k8s/pkg/k8sd/features/interface.go @@ -12,7 +12,7 @@ type Interface interface { // ApplyDNS is used to configure the DNS feature on Canonical Kubernetes. ApplyDNS(context.Context, snap.Snap, types.DNS, types.Kubelet, types.Annotations) (types.FeatureStatus, string, error) // ApplyNetwork is used to configure the network feature on Canonical Kubernetes. - ApplyNetwork(context.Context, snap.Snap, types.Network, types.Annotations) (types.FeatureStatus, error) + ApplyNetwork(context.Context, snap.Snap, types.APIServer, types.Network, types.Annotations) (types.FeatureStatus, error) // ApplyLoadBalancer is used to configure the load-balancer feature on Canonical Kubernetes. ApplyLoadBalancer(context.Context, snap.Snap, types.LoadBalancer, types.Network, types.Annotations) (types.FeatureStatus, error) // ApplyIngress is used to configure the ingress controller feature on Canonical Kubernetes. @@ -28,7 +28,7 @@ type Interface interface { // implementation implements Interface. type implementation struct { applyDNS func(context.Context, snap.Snap, types.DNS, types.Kubelet, types.Annotations) (types.FeatureStatus, string, error) - applyNetwork func(context.Context, snap.Snap, types.Network, types.Annotations) (types.FeatureStatus, error) + applyNetwork func(context.Context, snap.Snap, types.APIServer, types.Network, types.Annotations) (types.FeatureStatus, error) applyLoadBalancer func(context.Context, snap.Snap, types.LoadBalancer, types.Network, types.Annotations) (types.FeatureStatus, error) applyIngress func(context.Context, snap.Snap, types.Ingress, types.Network, types.Annotations) (types.FeatureStatus, error) applyGateway func(context.Context, snap.Snap, types.Gateway, types.Network, types.Annotations) (types.FeatureStatus, error) @@ -40,8 +40,8 @@ func (i *implementation) ApplyDNS(ctx context.Context, snap snap.Snap, dns types return i.applyDNS(ctx, snap, dns, kubelet, annotations) } -func (i *implementation) ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network, annotations types.Annotations) (types.FeatureStatus, error) { - return i.applyNetwork(ctx, snap, cfg, annotations) +func (i *implementation) ApplyNetwork(ctx context.Context, snap snap.Snap, apiserver types.APIServer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { + return i.applyNetwork(ctx, snap, apiserver, network, annotations) } func (i *implementation) ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go index 9b3a3b886..e288afbfd 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go @@ -11,7 +11,7 @@ import ( ) // K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy. -func K8sAPIServerProxy(snap snap.Snap, servers []string, localhostAddress string, extraArgs map[string]*string) error { +func K8sAPIServerProxy(snap snap.Snap, servers []string, securePort int, extraArgs map[string]*string) error { configFile := filepath.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json") if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil { return fmt.Errorf("failed to write proxy configuration file: %w", err) @@ -20,7 +20,7 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string, localhostAddress string if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", map[string]string{ "--endpoints": configFile, "--kubeconfig": filepath.Join(snap.KubernetesConfigDir(), "kubelet.conf"), - "--listen": fmt.Sprintf("%s:6443", localhostAddress), + "--listen": fmt.Sprintf(":%d", securePort), }, nil); err != nil { return fmt.Errorf("failed to write arguments file: %w", err) } diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go index 385c23e39..ad03d1dee 100644 --- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go @@ -27,7 +27,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) - g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", nil)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, 6443, nil)).To(Succeed()) tests := []struct { key string @@ -35,7 +35,7 @@ func TestK8sApiServerProxy(t *testing.T) { }{ {key: "--endpoints", expectedVal: filepath.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json")}, {key: "--kubeconfig", expectedVal: filepath.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")}, - {key: "--listen", expectedVal: "127.0.0.1:6443"}, + {key: "--listen", expectedVal: ":6443"}, } for _, tc := range tests { t.Run(tc.key, func(t *testing.T) { @@ -61,7 +61,7 @@ func TestK8sApiServerProxy(t *testing.T) { "--listen": nil, // This should trigger a delete "--my-extra-arg": utils.Pointer("my-extra-val"), } - g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", extraArgs)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, 6443, extraArgs)).To(Succeed()) tests := []struct { key string @@ -98,7 +98,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceExtraConfigDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, 6443, nil)).ToNot(Succeed()) }) t.Run("MissingServiceArgumentsDir", func(t *testing.T) { @@ -107,7 +107,7 @@ func TestK8sApiServerProxy(t *testing.T) { s := mustSetupSnapAndDirectories(t, setK8sApiServerMock) s.Mock.ServiceArgumentsDir = "nonexistent" - g.Expect(setup.K8sAPIServerProxy(s, nil, "127.0.0.1", nil)).ToNot(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, nil, 6443, nil)).ToNot(Succeed()) }) t.Run("JSONFileContent", func(t *testing.T) { @@ -118,7 +118,7 @@ func TestK8sApiServerProxy(t *testing.T) { endpoints := []string{"192.168.0.1", "192.168.0.2", "192.168.0.3"} fileName := filepath.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json") - g.Expect(setup.K8sAPIServerProxy(s, endpoints, "127.0.0.1", nil)).To(Succeed()) + g.Expect(setup.K8sAPIServerProxy(s, endpoints, 6443, nil)).To(Succeed()) b, err := os.ReadFile(fileName) g.Expect(err).NotTo(HaveOccurred()) diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go index 37c1c0192..457ba1b96 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -5,6 +5,7 @@ import ( "net" "os" "path/filepath" + "strconv" "strings" "github.com/canonical/k8s/pkg/k8sd/types" @@ -49,7 +50,7 @@ var ( ) // KubeAPIServer configures kube-apiserver on the local node. -func KubeAPIServer(snap snap.Snap, nodeIP net.IP, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string, extraArgs map[string]*string) error { +func KubeAPIServer(snap snap.Snap, securePort int, nodeIP net.IP, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string, extraArgs map[string]*string) error { authTokenWebhookConfigFile := filepath.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf") authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { @@ -77,7 +78,7 @@ func KubeAPIServer(snap snap.Snap, nodeIP net.IP, serviceCIDR string, authWebhoo "--kubelet-preferred-address-types": "InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP", "--profiling": "false", "--request-timeout": "300s", - "--secure-port": "6443", + "--secure-port": strconv.Itoa(securePort), "--service-account-issuer": "https://kubernetes.default.svc", "--service-account-key-file": filepath.Join(snap.KubernetesPKIDir(), "serviceaccount.key"), "--service-account-signing-key-file": filepath.Join(snap.KubernetesPKIDir(), "serviceaccount.key"), diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go index 7e150c5de..03b33f8dd 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go @@ -37,7 +37,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, 6443, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -96,7 +96,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, 6443, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -153,7 +153,7 @@ func TestKubeAPIServer(t *testing.T) { "--my-extra-arg": utils.Pointer("my-extra-val"), } // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", extraArgs)).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, 6443, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", extraArgs)).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -214,7 +214,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, net.ParseIP("192.168.0.1"), "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, 6443, net.ParseIP("192.168.0.1"), "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--service-cluster-ip-range")).To(Equal("10.0.0.0/24,fd01::/64")) _, err := utils.ParseArgumentFile(filepath.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -227,7 +227,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, 6443, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--etcd-servers")).To(Equal("datastoreurl1,datastoreurl2")) _, err := utils.ParseArgumentFile(filepath.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -241,7 +241,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Attempt to configure kube-apiserver with an unsupported datastore - err := setup.KubeAPIServer(s, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC", nil) + err := setup.KubeAPIServer(s, 6443, net.ParseIP("192.168.0.1"), "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC", nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(ContainSubstring("unsupported datastore"))) }) diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge.go b/src/k8s/pkg/k8sd/types/cluster_config_merge.go index fb1840d1a..9f42458ab 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge.go @@ -46,6 +46,7 @@ func MergeClusterConfig(existing ClusterConfig, new ClusterConfig) (ClusterConfi // network {name: "pod CIDR", val: &config.Network.PodCIDR, old: existing.Network.PodCIDR, new: new.Network.PodCIDR}, {name: "service CIDR", val: &config.Network.ServiceCIDR, old: existing.Network.ServiceCIDR, new: new.Network.ServiceCIDR}, + {name: "localhost address", val: &config.Network.LocalhostAddress, old: existing.Network.LocalhostAddress, new: new.Network.LocalhostAddress}, // apiserver {name: "kube-apiserver authorization mode", val: &config.APIServer.AuthorizationMode, old: existing.APIServer.AuthorizationMode, new: new.APIServer.AuthorizationMode, allowChange: true}, // kubelet diff --git a/src/k8s/pkg/k8sd/types/cluster_config_network.go b/src/k8s/pkg/k8sd/types/cluster_config_network.go index d429fac7b..9134c51e8 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_network.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_network.go @@ -1,12 +1,14 @@ package types type Network struct { - Enabled *bool `json:"enabled,omitempty"` - PodCIDR *string `json:"pod-cidr,omitempty"` - ServiceCIDR *string `json:"service-cidr,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + PodCIDR *string `json:"pod-cidr,omitempty"` + ServiceCIDR *string `json:"service-cidr,omitempty"` + LocalhostAddress *string `json:"localhost-address,omitempty"` } -func (c Network) GetEnabled() bool { return getField(c.Enabled) } -func (c Network) GetPodCIDR() string { return getField(c.PodCIDR) } -func (c Network) GetServiceCIDR() string { return getField(c.ServiceCIDR) } -func (c Network) Empty() bool { return c == Network{} } +func (c Network) GetEnabled() bool { return getField(c.Enabled) } +func (c Network) GetPodCIDR() string { return getField(c.PodCIDR) } +func (c Network) GetServiceCIDR() string { return getField(c.ServiceCIDR) } +func (c Network) GetLocalhostAddress() string { return getField(c.LocalhostAddress) } +func (c Network) Empty() bool { return c == Network{} }