Server Token Rotation (#8577)

* Consolidate NewCertCommands
* Add support for user defined new token
* Add E2E testlets



* Ensure agent token also changes

Signed-off-by: Derek Nola <derek.nola@suse.com>

cmd/cert/main.go

@@ -15,11 +15,9 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewCertCommand(
-			cmds.NewCertSubcommands(
-				cert.Rotate,
-				cert.RotateCA,
-			),
+		cmds.NewCertCommands(
+			cert.Rotate,
+			cert.RotateCA,
 		),
 	}
 

cmd/k3s/main.go

@@ -57,6 +57,7 @@ func main() {
 			tokenCommand,
 			tokenCommand,
 			tokenCommand,
+			tokenCommand,
 		),
 		cmds.NewEtcdSnapshotCommands(
 			etcdsnapshotCommand,
@@ -73,11 +74,9 @@ func main() {
 			secretsencryptCommand,
 			secretsencryptCommand,
 		),
-		cmds.NewCertCommand(
-			cmds.NewCertSubcommands(
-				certCommand,
-				certCommand,
-			),
+		cmds.NewCertCommands(
+			certCommand,
+			certCommand,
 		),
 		cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)),
 	}

cmd/server/main.go

@@ -54,6 +54,7 @@ func main() {
 			token.Delete,
 			token.Generate,
 			token.List,
+			token.Rotate,
 		),
 		cmds.NewEtcdSnapshotCommands(
 			etcdsnapshot.Run,
@@ -70,11 +71,9 @@ func main() {
 			secretsencrypt.Rotate,
 			secretsencrypt.Reencrypt,
 		),
-		cmds.NewCertCommand(
-			cmds.NewCertSubcommands(
-				cert.Rotate,
-				cert.RotateCA,
-			),
+		cmds.NewCertCommands(
+			cert.Rotate,
+			cert.RotateCA,
 		),
 		cmds.NewCompletionCommand(completion.Run),
 	}

cmd/token/main.go

@@ -20,6 +20,7 @@ func main() {
 			token.Delete,
 			token.Generate,
 			token.List,
+			token.Rotate,
 		),
 	}
 

main.go

@@ -47,11 +47,9 @@ func main() {
 			secretsencrypt.Rotate,
 			secretsencrypt.Reencrypt,
 		),
-		cmds.NewCertCommand(
-			cmds.NewCertSubcommands(
-				cert.Rotate,
-				cert.RotateCA,
-			),
+		cmds.NewCertCommands(
+			cert.Rotate,
+			cert.RotateCA,
 		),
 		cmds.NewCompletionCommand(completion.Run),
 	}

pkg/cli/cmds/certs.go

@@ -54,33 +54,29 @@ var (
 	}
 )
 
-func NewCertCommand(subcommands []cli.Command) cli.Command {
+func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:            CertCommand,
 		Usage:           "Manage K3s certificates",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
-		Subcommands:     subcommands,
-	}
-}
-
-func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command {
-	return []cli.Command{
-		{
-			Name:            "rotate",
-			Usage:           "Rotate " + version.Program + " component certificates on disk",
-			SkipFlagParsing: false,
-			SkipArgReorder:  true,
-			Action:          rotate,
-			Flags:           CertRotateCommandFlags,
-		},
-		{
-			Name:            "rotate-ca",
-			Usage:           "Write updated " + version.Program + " CA certificates to the datastore",
-			SkipFlagParsing: false,
-			SkipArgReorder:  true,
-			Action:          rotateCA,
-			Flags:           CertRotateCACommandFlags,
+		Subcommands: []cli.Command{
+			{
+				Name:            "rotate",
+				Usage:           "Rotate " + version.Program + " component certificates on disk",
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          rotate,
+				Flags:           CertRotateCommandFlags,
+			},
+			{
+				Name:            "rotate-ca",
+				Usage:           "Write updated " + version.Program + " CA certificates to the datastore",
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          rotateCA,
+				Flags:           CertRotateCACommandFlags,
+			},
 		},
 	}
 }

pkg/cli/cmds/token.go

@@ -3,6 +3,7 @@ package cmds
 import (
 	"time"
 
+	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/urfave/cli"
 )
 
@@ -12,7 +13,9 @@ const TokenCommand = "token"
 type Token struct {
 	Description string
 	Kubeconfig  string
+	ServerURL   string
 	Token       string
+	NewToken    string
 	Output      string
 	Groups      cli.StringSlice
 	Usages      cli.StringSlice
@@ -32,7 +35,7 @@ var (
 	}
 )
 
-func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) error) cli.Command {
+func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:            TokenCommand,
 		Usage:           "Manage bootstrap tokens",
@@ -92,6 +95,32 @@ func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) erro
 				SkipArgReorder:  true,
 				Action:          list,
 			},
+			{
+				Name:  "rotate",
+				Usage: "Rotate original server token with a new bootstrap token",
+				Flags: append(TokenFlags,
+					&cli.StringFlag{
+						Name:        "token,t",
+						Usage:       "Existing token used to join a server or agent to a cluster",
+						Destination: &TokenConfig.Token,
+						EnvVar:      version.ProgramUpper + "_TOKEN",
+					},
+					&cli.StringFlag{
+						Name:        "server, s",
+						Usage:       "(cluster) Server to connect to",
+						Destination: &TokenConfig.ServerURL,
+						EnvVar:      version.ProgramUpper + "_URL",
+						Value:       "https://127.0.0.1:6443",
+					},
+					&cli.StringFlag{
+						Name:        "new-token",
+						Usage:       "New token that replaces existing token",
+						Destination: &TokenConfig.NewToken,
+					}),
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          rotate,
+			},
 		},
 	}
 }

pkg/cli/token/token.go

@@ -1,18 +1,23 @@
 package token
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/tabwriter"
 	"time"
 
+	"github.com/erikdubbelboer/gspt"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/kubeadm"
+	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"gopkg.in/yaml.v2"
@@ -22,6 +27,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	bootstraputil "k8s.io/cluster-bootstrap/token/util"
+	"k8s.io/utils/pointer"
 )
 
 func Create(app *cli.Context) error {
@@ -139,6 +145,50 @@ func generate(app *cli.Context, cfg *cmds.Token) error {
 	return nil
 }
 
+func Rotate(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	fmt.Println("\033[33mWARNING\033[0m: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.")
+	info, err := serverAccess(&cmds.TokenConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.TokenRotateRequest{
+		NewToken: pointer.String(cmds.TokenConfig.NewToken),
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/token", b); err != nil {
+		return err
+	}
+	// wait for etcd db propagation delay
+	time.Sleep(1 * time.Second)
+	fmt.Println("Token rotated, restart k3s with new token")
+	return nil
+}
+
+func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) {
+	// hide process arguments from ps output, since they likely contain tokens.
+	gspt.SetProcTitle(os.Args[0] + " token")
+
+	dataDir, err := server.ResolveDataDir("")
+	if err != nil {
+		return nil, err
+	}
+
+	if cfg.Token == "" {
+		fp := filepath.Join(dataDir, "token")
+		tokenByte, err := os.ReadFile(fp)
+		if err != nil {
+			return nil, err
+		}
+		cfg.Token = string(bytes.TrimRight(tokenByte, "\n"))
+	}
+	return clientaccess.ParseAndValidateToken(cfg.ServerURL, cfg.Token, clientaccess.WithUser("server"))
+}
+
 func List(app *cli.Context) error {
 	if err := cmds.InitLogging(); err != nil {
 		return err

pkg/cluster/storage.go

@@ -21,6 +21,49 @@ import (
 // After this many attempts, the lock is deleted and the counter reset.
 const maxBootstrapWaitAttempts = 5
 
+func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
+
+	token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
+	if err != nil {
+		return err
+	}
+
+	normalizedToken, err := normalizeToken(token)
+	if err != nil {
+		return err
+	}
+
+	storageClient, err := client.New(config.Runtime.EtcdConfig)
+	if err != nil {
+		return err
+	}
+	defer storageClient.Close()
+
+	tokenKey := storageKey(normalizedToken)
+
+	var bootstrapList []client.Value
+	if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) {
+		bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}); err != nil {
+		return err
+	}
+
+	normalizedOldToken, err := normalizeToken(oldToken)
+	if err != nil {
+		return err
+	}
+	// reuse the existing migration function to reencrypt bootstrap data with new token
+	if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
 // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
 // This is used when bootstrapping a cluster from a managed database or external etcd cluster.
@@ -225,7 +268,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
 		logrus.Warn("found multiple bootstrap keys in storage")
 	}
 	// check for empty string key and for old token format with k10 prefix
-	if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
+	if err := migrateTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
 		return nil, false, err
 	}
 
@@ -236,6 +279,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
 	}
 	for _, bootstrapKV := range bootstrapList {
 		// ensure bootstrap is stored in the current token's key
+		logrus.Debugf("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey)
 		if string(bootstrapKV.Key) == tokenKey {
 			return &bootstrapKV, false, nil
 		}
@@ -277,21 +321,24 @@ func normalizeToken(token string) (string, error) {
 	return password, nil
 }
 
-// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
+// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and keys that is hashed with old token format before normalizing
 // then migrate those and resave only with the normalized token
-func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
+func migrateTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
 	oldTokenKey := storageKey(oldToken)
 
 	for _, bootstrapKV := range bootstrapList {
 		// checking for empty string bootstrap key
+		logrus.Debug("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey)
 		if string(bootstrapKV.Key) == emptyStringKey {
 			logrus.Warn("Bootstrap data encrypted with empty string, deleting and resaving with token")
 			if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil {
 				return err
 			}
 		} else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey {
-			logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+			if emptyStringKey != "" {
+				logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+			}
 			if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil {
 				return err
 			}

pkg/daemons/control/deps/deps.go

@@ -243,6 +243,7 @@ func genUsers(config *config.Control) error {
 		return err
 	}
 
+	// if no token is provided on bootstrap, we generate a random token
 	serverPass, err := getServerPass(passwd, config)
 	if err != nil {
 		return err

pkg/server/router.go

@@ -85,6 +85,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
 	serverAuthed.Path("/db/info").Handler(nodeAuthed)
 	serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
+	serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig))
 
 	systemAuthed := mux.NewRouter().SkipClean(true)
 	systemAuthed.NotFoundHandler = serverAuthed