Initial implementation for embedded datastore

build-scripts/components/k8s-dqlite/version

@@ -1 +1 @@
-master
+KU-961/embedded

docs/src/snap/howto/embedded-datastore.md

@@ -0,0 +1,138 @@
+# How to use the embedded datastore
+
+Canonical Kubernetes supports using an embedded datastore such as etcd
+instead of the bundled dqlite datastore.
+This guide walks you through configuring the embedded etcd datastore.
+
+## What you'll need
+
+This guide assumes the following:
+
+- You have root or sudo access to the machine
+- You have installed the Canonical Kubernetes snap
+  (see How-to [Install Canonical Kubernetes from a snap][snap-install-howto]).
+- You have not bootstrapped the Canonical Kubernetes cluster yet
+
+```{warning}
+The selection of the backing datastore can only be changed during the bootstrap process.
+There is no migration path between the bundled dqlite and the embedded datastore.
+```
+
+## Adjust the bootstrap configuration
+
+To use the embedded datastore, a configuration file that contains the required
+datastore parameters needs to be provided to the bootstrap command.
+Create a configuration file and insert the contents below while replacing
+the placeholder values based on the desired configuration.
+
+```yaml
+# required
+datastore-type: embedded
+
+# optional
+datastore-embedded-port: 2379
+datastore-embedded-peer-port: 2380
+datastore-embedded-ca-crt: |
+  <etcd-root-ca-certificate>
+datastore-embedded-ca-key: |
+  <etcd-root-ca-private-key>
+```
+
+* `datastore-type` must be set to `embedded`.
+* `datastore-embedded-port` expects a port number that will be used for the
+  client URLs of the embedded cluster. The default is port `2379`.
+* `datastore-embedded-peer-port` expects a port number that will be used for
+  the peer URLs of the embedded cluster. The default is port `2380`.
+* `datastore-embedded-ca-crt` and `datastore-embedded-ca-key`: an optional
+  custom CA certificate and private key to use for the embedded etcd cluster.
+  If not specified, k8sd will automatically generate a self-signed CA.
+
+```{note}
+the embedded datastore will always be configured with TLS.
+```
+
+## Bootstrap the cluster
+
+The next step is to bootstrap the cluster with our configuration file:
+
+```
+sudo k8s bootstrap --file /path/to/config.yaml
+```
+
+```{note}
+The datastore can only be configured through the `--file` file option,
+and is not available in interactive mode.
+```
+
+## Confirm the cluster is ready
+
+It is recommended to ensure that the cluster initialises properly and is
+running without issues. Run the command:
+
+```
+sudo k8s status --wait-ready
+```
+
+This command will wait until the cluster is ready and then display
+the current status. The command will time-out if the cluster does not reach a
+ready state.
+
+## Clustering
+
+Control plane nodes that join with `k8s join-cluster` will be added as members
+to the embedded datastore. Nodes that join the cluster start by registering
+themselves in the cluster.
+
+During the transition from 1 to 2 control plane nodes, the embedded cluster
+will temporarily freeze and reject write operations, since after moving to a
+2-node quorum, both nodes must be available for the raft protocol to proceed.
+
+When removing a node with `k8s remove-node`, the node will also be removed
+from the embedded datastore.
+
+## Interacting with the embedded datastore
+
+Under normal operation, you should not have to interact with the embedded
+datastore directly. However, in case it is needed for debugging problems, or
+operations like creating a cluster backup, you can use either `etcdctl` or
+the built-in `k8s-dqlite embeddedctl` commands.
+
+### Using etcdctl
+
+You can interact with the embedded datastore using the standard `etcdctl` CLI
+tool. `etcdctl` is not included in Canonical Kubernetes and needs to be
+installed separately if needed. To point `etcdctl` to the embedded cluster, you
+need to set the following arguments:
+
+```bash
+sudo ETCDCTL_API=3 etcdctl \
+  --endpoints https://${nodeip}:2379 \
+  --cacert /etc/kubernetes/pki/etcd/ca.crt \
+  --cert /etc/kubernetes/pki/apiserver-etcd-client.crt \
+  --key /etc/kubernetes/pki/apiserver-etcd-client.key \
+  member list
+```
+
+### Using k8s-dqlite embeddedctl
+
+There is a `k8s-dqlite embeddedctl` subcommand that can be used from control
+plane nodes to directly interact with the datastore if required. This tool is
+supposed to be a lightweight alternative to common `etcdctl` commands, and
+comes with the following subcommands:
+
+- `k8s-dqlite embeddedctl member {add,list,remove}`: list the current cluster
+  members, and add or remove a cluster member.
+- `k8s-dqlite embeddedctl snapshot save "file.db"`: create a backup snapshot of
+  the current database state and save it in `file.db`. this backup can then be
+  used to do a point-in-time restoration of the database using `etcdutl`.
+
+You can access the `k8s-dqlite embeddedctl` commands as shown below. Specify
+the `--help` argument to see all available commands and supported arguments:
+
+```bash
+sudo /snap/k8s/current/bin/k8s-dqlite embeddedctl --help
+```
+
+<!-- LINKS -->
+
+[snap-install-howto]: ./install/snap

docs/src/snap/howto/external-datastore.md

@@ -27,7 +27,7 @@ Create a configuration file and insert the contents below while replacing
 the placeholder values based on the configuration of your etcd cluster.
 
 ```yaml
-datastore: external
+datastore-type: external
 datastore-url: "<etcd-member-addresses>"
 datastore-ca-crt: |
   <etcd-root-ca-certificate>

docs/src/snap/howto/index.md

@@ -18,6 +18,7 @@ install/index
 networking/index
 storage
 external-datastore
+embedded-datastore
 proxy
 contribute
 support

src/k8s/api/v1/bootstrap_config.go

@@ -11,17 +11,20 @@ type BootstrapConfig struct {
 	ClusterConfig UserFacingClusterConfig `json:"cluster-config,omitempty" yaml:"cluster-config,omitempty"`
 
 	// Seed configuration for the control plane (flat on purpose). Empty values are ignored
-	ControlPlaneTaints  []string `json:"control-plane-taints,omitempty" yaml:"control-plane-taints,omitempty"`
-	PodCIDR             *string  `json:"pod-cidr,omitempty" yaml:"pod-cidr,omitempty"`
-	ServiceCIDR         *string  `json:"service-cidr,omitempty" yaml:"service-cidr,omitempty"`
-	DisableRBAC         *bool    `json:"disable-rbac,omitempty" yaml:"disable-rbac,omitempty"`
-	SecurePort          *int     `json:"secure-port,omitempty" yaml:"secure-port,omitempty"`
-	K8sDqlitePort       *int     `json:"k8s-dqlite-port,omitempty" yaml:"k8s-dqlite-port,omitempty"`
-	DatastoreType       *string  `json:"datastore-type,omitempty" yaml:"datastore-type,omitempty"`
-	DatastoreServers    []string `json:"datastore-servers,omitempty" yaml:"datastore-servers,omitempty"`
-	DatastoreCACert     *string  `json:"datastore-ca-crt,omitempty" yaml:"datastore-ca-crt,omitempty"`
-	DatastoreClientCert *string  `json:"datastore-client-crt,omitempty" yaml:"datastore-client-crt,omitempty"`
-	DatastoreClientKey  *string  `json:"datastore-client-key,omitempty" yaml:"datastore-client-key,omitempty"`
+	ControlPlaneTaints []string `json:"control-plane-taints,omitempty" yaml:"control-plane-taints,omitempty"`
+	PodCIDR            *string  `json:"pod-cidr,omitempty" yaml:"pod-cidr,omitempty"`
+	ServiceCIDR        *string  `json:"service-cidr,omitempty" yaml:"service-cidr,omitempty"`
+	DisableRBAC        *bool    `json:"disable-rbac,omitempty" yaml:"disable-rbac,omitempty"`
+	SecurePort         *int     `json:"secure-port,omitempty" yaml:"secure-port,omitempty"`
+
+	K8sDqlitePort             *int     `json:"k8s-dqlite-port,omitempty" yaml:"k8s-dqlite-port,omitempty"`
+	DatastoreType             *string  `json:"datastore-type,omitempty" yaml:"datastore-type,omitempty"`
+	DatastoreServers          []string `json:"datastore-servers,omitempty" yaml:"datastore-servers,omitempty"`
+	DatastoreCACert           *string  `json:"datastore-ca-crt,omitempty" yaml:"datastore-ca-crt,omitempty"`
+	DatastoreClientCert       *string  `json:"datastore-client-crt,omitempty" yaml:"datastore-client-crt,omitempty"`
+	DatastoreClientKey        *string  `json:"datastore-client-key,omitempty" yaml:"datastore-client-key,omitempty"`
+	DatastoreEmbeddedPort     *int     `json:"datastore-embedded-port,omitempty" yaml:"datastore-embedded-port,omitempty"`
+	DatastoreEmbeddedPeerPort *int     `json:"datastore-embedded-peer-port,omitempty" yaml:"datastore-embedded-peer-port,omitempty"`
 
 	// Seed configuration for certificates
 	ExtraSANs []string `json:"extra-sans,omitempty" yaml:"extra-sans,omitempty"`
@@ -47,6 +50,16 @@ type BootstrapConfig struct {
 	KubeControllerManagerClientKey  *string `json:"kube-controller-manager-client-key,omitempty" yaml:"kube-ControllerManager-client-key,omitempty"`
 	ServiceAccountKey               *string `json:"service-account-key,omitempty" yaml:"service-account-key,omitempty"`
 
+	// Seed configuration for embedded datastore
+	EmbeddedCACert              *string `json:"embedded-ca-crt,omitempty" yaml:"embedded-ca-crt,omitempty"`
+	EmbeddedCAKey               *string `json:"embedded-ca-key,omitempty" yaml:"embedded-ca-key,omitempty"`
+	EmbeddedServerCert          *string `json:"embedded-server-crt,omitempty" yaml:"embedded-server-crt,omitempty"`
+	EmbeddedServerKey           *string `json:"embedded-server-key,omitempty" yaml:"embedded-server-key,omitempty"`
+	EmbeddedServerPeerCert      *string `json:"embedded-peer-crt,omitempty" yaml:"embedded-peer-crt,omitempty"`
+	EmbeddedServerPeerKey       *string `json:"embedded-peer-key,omitempty" yaml:"embedded-peer-key,omitempty"`
+	EmbeddedAPIServerClientCert *string `json:"embedded-apiserver-client-crt,omitempty" yaml:"embedded-apiserver-client-crt,omitempty"`
+	EmbeddedAPIServerClientKey  *string `json:"embedded-apiserver-client-key,omitempty" yaml:"embedded-apiserver-client-key,omitempty"`
+
 	// Seed configuration for external certificates (node-specific)
 	APIServerCert     *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"`
 	APIServerKey      *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"`
@@ -68,10 +81,14 @@ type BootstrapConfig struct {
 	ExtraNodeK8sDqliteArgs             map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
 }
 
-func (b *BootstrapConfig) GetDatastoreType() string        { return getField(b.DatastoreType) }
-func (b *BootstrapConfig) GetDatastoreCACert() string      { return getField(b.DatastoreCACert) }
-func (b *BootstrapConfig) GetDatastoreClientCert() string  { return getField(b.DatastoreClientCert) }
-func (b *BootstrapConfig) GetDatastoreClientKey() string   { return getField(b.DatastoreClientKey) }
+func (b *BootstrapConfig) GetDatastoreType() string       { return getField(b.DatastoreType) }
+func (b *BootstrapConfig) GetDatastoreCACert() string     { return getField(b.DatastoreCACert) }
+func (b *BootstrapConfig) GetDatastoreClientCert() string { return getField(b.DatastoreClientCert) }
+func (b *BootstrapConfig) GetDatastoreClientKey() string  { return getField(b.DatastoreClientKey) }
+func (b *BootstrapConfig) GetDatastoreEmbeddedPort() int  { return getField(b.DatastoreEmbeddedPort) }
+func (b *BootstrapConfig) GetDatastoreEmbeddedPeerPort() int {
+	return getField(b.DatastoreEmbeddedPeerPort)
+}
 func (b *BootstrapConfig) GetK8sDqlitePort() int           { return getField(b.K8sDqlitePort) }
 func (b *BootstrapConfig) GetCACert() string               { return getField(b.CACert) }
 func (b *BootstrapConfig) GetCAKey() string                { return getField(b.CAKey) }
@@ -103,7 +120,21 @@ func (b *BootstrapConfig) GetKubeControllerManagerClientCert() string {
 func (b *BootstrapConfig) GetKubeControllerManagerClientKey() string {
 	return getField(b.KubeControllerManagerClientKey)
 }
-func (b *BootstrapConfig) GetServiceAccountKey() string { return getField(b.ServiceAccountKey) }
+func (b *BootstrapConfig) GetServiceAccountKey() string  { return getField(b.ServiceAccountKey) }
+func (b *BootstrapConfig) GetEmbeddedCACert() string     { return getField(b.EmbeddedCACert) }
+func (b *BootstrapConfig) GetEmbeddedCAKey() string      { return getField(b.EmbeddedCAKey) }
+func (b *BootstrapConfig) GetEmbeddedServerCert() string { return getField(b.EmbeddedServerCert) }
+func (b *BootstrapConfig) GetEmbeddedServerKey() string  { return getField(b.EmbeddedServerKey) }
+func (b *BootstrapConfig) GetEmbeddedServerPeerCert() string {
+	return getField(b.EmbeddedServerPeerCert)
+}
+func (b *BootstrapConfig) GetEmbeddedServerPeerKey() string { return getField(b.EmbeddedServerPeerKey) }
+func (b *BootstrapConfig) GetEmbeddedAPIServerClientCert() string {
+	return getField(b.EmbeddedAPIServerClientCert)
+}
+func (b *BootstrapConfig) GetEmbeddedAPIServerClientKey() string {
+	return getField(b.EmbeddedAPIServerClientKey)
+}
 func (b *BootstrapConfig) GetAPIServerCert() string     { return getField(b.APIServerCert) }
 func (b *BootstrapConfig) GetAPIServerKey() string      { return getField(b.APIServerKey) }
 func (b *BootstrapConfig) GetKubeletCert() string       { return getField(b.KubeletCert) }

src/k8s/api/v1/join_config.go

@@ -26,6 +26,11 @@ type ControlPlaneNodeJoinConfig struct {
 	KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
 	KubeletClientKey  *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
 
+	EmbeddedServerCert     *string `json:"embedded-server-crt,omitempty" yaml:"embedded-server-crt,omitempty"`
+	EmbeddedServerKey      *string `json:"embedded-server-key,omitempty" yaml:"embedded-server-key,omitempty"`
+	EmbeddedServerPeerCert *string `json:"embedded-peer-crt,omitempty" yaml:"embedded-peer-crt,omitempty"`
+	EmbeddedServerPeerKey  *string `json:"embedded-peer-key,omitempty" yaml:"embedded-peer-key,omitempty"`
+
 	// ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
 	ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
 
@@ -91,6 +96,18 @@ func (c *ControlPlaneNodeJoinConfig) GetKubeletClientCert() string {
 func (c *ControlPlaneNodeJoinConfig) GetKubeletClientKey() string {
 	return getField(c.KubeletClientKey)
 }
+func (b *ControlPlaneNodeJoinConfig) GetEmbeddedServerCert() string {
+	return getField(b.EmbeddedServerCert)
+}
+func (b *ControlPlaneNodeJoinConfig) GetEmbeddedServerKey() string {
+	return getField(b.EmbeddedServerKey)
+}
+func (b *ControlPlaneNodeJoinConfig) GetEmbeddedServerPeerCert() string {
+	return getField(b.EmbeddedServerPeerCert)
+}
+func (b *ControlPlaneNodeJoinConfig) GetEmbeddedServerPeerKey() string {
+	return getField(b.EmbeddedServerPeerKey)
+}
 
 func (w *WorkerNodeJoinConfig) GetKubeletCert() string       { return getField(w.KubeletCert) }
 func (w *WorkerNodeJoinConfig) GetKubeletKey() string        { return getField(w.KubeletKey) }

src/k8s/pkg/client/embedded/external.go

@@ -0,0 +1,27 @@
+package embedded
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os/exec"
+)
+
+// externalClient implements Client using `k8s-dqlite embeddedctl` commands.
+type externalClient struct {
+	binary     string
+	storageDir string
+}
+
+func NewExternalClient(binary string, storageDir string) *externalClient {
+	return &externalClient{binary: binary, storageDir: storageDir}
+}
+
+func (c *externalClient) RemoveNodeByAddress(ctx context.Context, peerURL string) error {
+	command := []string{c.binary, "embeddedctl", "member", "remove", "--storage-dir", c.storageDir, "--peer-url", peerURL}
+	cmd := exec.CommandContext(ctx, command[0], command[1:]...)
+	if b, err := cmd.CombinedOutput(); err != nil && !bytes.Contains(b, []byte("cluster member not found")) {
+		return fmt.Errorf("command failed, rc=%v command=%v output=%q", cmd.ProcessState.ExitCode(), command, string(b))
+	}
+	return nil
+}

src/k8s/pkg/client/embedded/interface.go

@@ -0,0 +1,9 @@
+package embedded
+
+import "context"
+
+// Client handles the interaction with an embedded cluster database.
+type Client interface {
+	// RemoveNodeByAddress removes the member with the specified name from the cluster.
+	RemoveNodeByAddress(ctx context.Context, peerURL string) error
+}

src/k8s/pkg/k8sd/app/cluster_util.go

@@ -36,9 +36,9 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki
 func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error {
 	// Start services
 	switch datastore {
-	case "k8s-dqlite":
+	case "k8s-dqlite", "embedded":
 		if err := snaputil.StartK8sDqliteServices(ctx, snap); err != nil {
-			return fmt.Errorf("failed to start control plane services: %w", err)
+			return fmt.Errorf("failed to start k8s-dqlite services: %w", err)
 		}
 	case "external":
 	default: