Add extra arguments for each k8sd service (#485)

Provide a way to pass to provide arbitrary extra arguments to each k8sd service. Those parameters are applied on a node-level and not shared across the cluster.

We want to support power users to supply custom configuration to the individual services, to allow management of setups like setting up kube-apiserver for encryption at rest, kubelet topology configuration, etc.

Also, adds ExtraNodeConfigFiles to load config files to /var/snap/k8s/common/args/conf.d.

docs/src/snap/reference/bootstrap-config-reference.md

@@ -37,6 +37,7 @@ Determines if the feature should be enabled.
 If omitted defaults to `true`
 
 #### cluster-config.dns.cluster-domain
+
 **Type:** `string`<br>
 **Required:** `No` <br>
 
@@ -59,7 +60,8 @@ Can be used to point to an external dns server when feature is disabled.
 **Type:** `list[string]`<br>
 **Required:** `No` <br>
 
-Sets the upstream nameservers used to forward queries for out-of-cluster endpoints.
+Sets the upstream nameservers used to forward queries for out-of-cluster
+endpoints.
 If omitted defaults to `/etc/resolv.conf` and uses the nameservers of the node.
 
 
@@ -83,7 +85,8 @@ If omitted defaults to `false`
 **Type:** `string`<br>
 **Required:** `No` <br>
 
-Sets the name of the secret to be used for providing default encryption to ingresses.
+Sets the name of the secret to be used for providing default encryption to
+ingresses.
 
 Ingresses can specify another TLS secret in their resource definitions,
 in which case the default secret won't be used.
@@ -117,7 +120,8 @@ If omitted defaults to `false`
 **Type:** `list[string]`<br>
 **Required:** `No` <br>
 
-Sets the CIDRs used for assigning IP addresses to Kubernetes services with type `LoadBalancer`.
+Sets the CIDRs used for assigning IP addresses to Kubernetes services with type
+`LoadBalancer`.
 
 #### cluster-config.load-balancer.l2-mode
 
@@ -316,7 +320,8 @@ The CA certificate to be used when communicating with the external datastore.
 **Type:** `string` <br>
 **Required:** `No` <br>
 
-The client certificate to be used when communicating with the external datastore.
+The client certificate to be used when communicating with the external
+datastore.
 
 ### datastore-client-key
 
@@ -386,7 +391,8 @@ If omitted defaults to an auto generated key.
 **Type:** `string` <br>
 **Required:** `No` <br>
 
-The client certificate to be used by kubelet for communicating with the kube-apiserver.
+The client certificate to be used by kubelet for communicating with the
+kube-apiserver.
 If omitted defaults to an auto generated certificate.
 
 ### apiserver-kubelet-client-key
@@ -437,6 +443,79 @@ If omitted defaults to an auto generated certificate.
 The key to be used for the kubelet.
 If omitted defaults to an auto generated key.
 
+### extra-node-config-files
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional files that are uploaded `/var/snap/k8s/common/args/conf.d/<filename>`
+to a node on bootstrap. These files can them be references by Kubernetes
+service arguments.
+The format is `map[<filename>]<filecontent>`.
+
+### extra-node-kube-apiserver-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to the `kube-apiserver` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
+### extra-node-kube-controller-manager-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to the `kube-controller-manager` only for
+that specific node. Overwrites default configuration. A parameter that is
+explicitly set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
+### extra-node-kube-scheduler-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to the `kube-scheduler` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
+### extra-node-kube-proxy-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to the `kube-proxy` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
+### extra-node-kubelet-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to the `kubelet` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
+### extra-node-containerd-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to `containerd` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
+### extra-node-k8s-dqlite-args
+
+**Type:** `map[string]string` <br>
+**Required:** `No` <br>
+
+Additional arguments that are passed to `k8s-dqlite` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]<value>`.
+
 ## Example
 
 The following example configures and enables certain features, sets an external
@@ -478,4 +557,20 @@ k8s-dqlite-port: 9090
 datastore-type: k8s-dqlite
 extra-sans:
 - custom.kubernetes
+extra-node-config-files:
+  bootstrap-extra-file.yaml: extra-args-test-file-content
+extra-node-kube-apiserver-args:
+  --request-timeout: 2m
+extra-node-kube-controller-manager-args:
+  --leader-elect-retry-period: 3s
+extra-node-kube-scheduler-args:
+  --authorization-webhook-cache-authorized-ttl: 11s
+extra-node-kube-proxy-args:
+  --config-sync-period: 14m
+extra-node-kubelet-args:
+  --authentication-token-webhook-cache-ttl: 3m
+extra-node-containerd-args:
+  --log-level: debug
+extra-node-k8s-dqlite-args:
+  --watch-storage-available-size-interval: 6s
 ```

src/k8s/api/v1/bootstrap_config.go

@@ -54,6 +54,18 @@ type BootstrapConfig struct {
 	KubeletKey        *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+	ExtraNodeKubeSchedulerArgs         map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+	ExtraNodeKubeProxyArgs             map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs               map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs            map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
 func (b *BootstrapConfig) GetDatastoreType() string        { return getField(b.DatastoreType) }

src/k8s/api/v1/bootstrap_config_test.go

@@ -41,13 +41,21 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) {
 			},
 			CloudProvider: utils.Pointer("external"),
 		},
-		PodCIDR:       utils.Pointer("10.100.0.0/16"),
-		ServiceCIDR:   utils.Pointer("10.200.0.0/16"),
-		DisableRBAC:   utils.Pointer(false),
-		SecurePort:    utils.Pointer(6443),
-		K8sDqlitePort: utils.Pointer(9090),
-		DatastoreType: utils.Pointer("k8s-dqlite"),
-		ExtraSANs:     []string{"custom.kubernetes"},
+		PodCIDR:                            utils.Pointer("10.100.0.0/16"),
+		ServiceCIDR:                        utils.Pointer("10.200.0.0/16"),
+		DisableRBAC:                        utils.Pointer(false),
+		SecurePort:                         utils.Pointer(6443),
+		K8sDqlitePort:                      utils.Pointer(9090),
+		DatastoreType:                      utils.Pointer("k8s-dqlite"),
+		ExtraSANs:                          []string{"custom.kubernetes"},
+		ExtraNodeConfigFiles:               map[string]string{"extra-node-config-file": "file-content"},
+		ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+		ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+		ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+		ExtraNodeKubeProxyArgs:             map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+		ExtraNodeKubeletArgs:               map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+		ExtraNodeContainerdArgs:            map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+		ExtraNodeK8sDqliteArgs:             map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
 	}
 
 	microclusterConfig, err := cfg.ToMicrocluster()

src/k8s/api/v1/join_config.go

@@ -25,6 +25,18 @@ type ControlPlaneNodeJoinConfig struct {
 	KubeletKey        *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeAPIServerArgs         map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+	ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+	ExtraNodeKubeSchedulerArgs         map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+	ExtraNodeKubeProxyArgs             map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs               map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs            map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
 type WorkerNodeJoinConfig struct {
@@ -34,6 +46,15 @@ type WorkerNodeJoinConfig struct {
 	KubeletClientKey    *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
 	KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
 	KubeProxyClientKey  *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
+
+	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+	// Extra args to add to individual services (set any arg to null to delete)
+	ExtraNodeKubeProxyArgs         map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+	ExtraNodeKubeletArgs           map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+	ExtraNodeContainerdArgs        map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+	ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"`
 }
 
 func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string {

src/k8s/cmd/k8s/k8s_bootstrap_test.go

@@ -64,14 +64,22 @@ var testCases = []testCase{
 				},
 				CloudProvider: utils.Pointer("external"),
 			},
-			ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
-			PodCIDR:            utils.Pointer("10.100.0.0/16"),
-			ServiceCIDR:        utils.Pointer("10.200.0.0/16"),
-			DisableRBAC:        utils.Pointer(false),
-			SecurePort:         utils.Pointer(6443),
-			K8sDqlitePort:      utils.Pointer(9090),
-			DatastoreType:      utils.Pointer("k8s-dqlite"),
-			ExtraSANs:          []string{"custom.kubernetes"},
+			ControlPlaneTaints:                 []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
+			PodCIDR:                            utils.Pointer("10.100.0.0/16"),
+			ServiceCIDR:                        utils.Pointer("10.200.0.0/16"),
+			DisableRBAC:                        utils.Pointer(false),
+			SecurePort:                         utils.Pointer(6443),
+			K8sDqlitePort:                      utils.Pointer(9090),
+			DatastoreType:                      utils.Pointer("k8s-dqlite"),
+			ExtraSANs:                          []string{"custom.kubernetes"},
+			ExtraNodeConfigFiles:               map[string]string{"extra-node-config-file.yaml": "test-file-content"},
+			ExtraNodeKubeAPIServerArgs:         map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+			ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+			ExtraNodeKubeSchedulerArgs:         map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+			ExtraNodeKubeProxyArgs:             map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+			ExtraNodeKubeletArgs:               map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+			ExtraNodeContainerdArgs:            map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+			ExtraNodeK8sDqliteArgs:             map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
 		},
 	},
 	{

src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml

@@ -31,3 +31,19 @@ k8s-dqlite-port: 9090
 datastore-type: k8s-dqlite
 extra-sans:
 - custom.kubernetes
+extra-node-config-files:
+  extra-node-config-file.yaml: test-file-content
+extra-node-kube-apiserver-args:
+  --extra-kube-apiserver-arg: extra-kube-apiserver-value
+extra-node-kube-controller-manager-args:
+  --extra-kube-controller-manager-arg: extra-kube-controller-manager-value
+extra-node-kube-scheduler-args:
+  --extra-kube-scheduler-arg: extra-kube-scheduler-value
+extra-node-kube-proxy-args:
+  --extra-kube-proxy-arg: extra-kube-proxy-value
+extra-node-kubelet-args:
+  --extra-kubelet-arg: extra-kubelet-value
+extra-node-containerd-args:
+  --extra-containerd-arg: extra-containerd-value
+extra-node-k8s-dqlite-args:
+  --extra-k8s-dqlite-arg: extra-k8s-dqlite-value

src/k8s/pkg/k8sd/app/cluster_util.go

@@ -3,12 +3,10 @@ package app
 import (
 	"context"
 	"fmt"
-	"net"
 	"path"
 
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
-	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
 	"github.com/canonical/microcluster/state"
@@ -35,29 +33,6 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki
 
 }
 
-func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.ClusterConfig, nodeIP net.IP) error {
-	// Configure services
-	if err := setup.Containerd(snap, nil); err != nil {
-		return fmt.Errorf("failed to configure containerd: %w", err)
-	}
-	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints()); err != nil {
-		return fmt.Errorf("failed to configure kubelet: %w", err)
-	}
-	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR()); err != nil {
-		return fmt.Errorf("failed to configure kube-proxy: %w", err)
-	}
-	if err := setup.KubeControllerManager(snap); err != nil {
-		return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
-	}
-	if err := setup.KubeScheduler(snap); err != nil {
-		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
-	}
-	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil {
-		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
-	}
-	return nil
-}
-
 func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error {
 	// Start services
 	switch datastore {