From 08f42e1900ea01ea6f6029e8cbfa34d69ea160ed Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 9 Aug 2023 10:38:21 -0700 Subject: [PATCH 01/13] Consolidate NewCertCommands Signed-off-by: Derek Nola --- cmd/cert/main.go | 8 +++----- cmd/k3s/main.go | 8 +++----- cmd/server/main.go | 8 +++----- main.go | 8 +++----- pkg/cli/cmds/certs.go | 40 ++++++++++++++++++---------------------- 5 files changed, 30 insertions(+), 42 deletions(-) diff --git a/cmd/cert/main.go b/cmd/cert/main.go index 1053056eebbc..85d71f07cba5 100644 --- a/cmd/cert/main.go +++ b/cmd/cert/main.go @@ -15,11 +15,9 @@ import ( func main() { app := cmds.NewApp() app.Commands = []cli.Command{ - cmds.NewCertCommand( - cmds.NewCertSubcommands( - cert.Rotate, - cert.RotateCA, - ), + cmds.NewCertCommands( + cert.Rotate, + cert.RotateCA, ), } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 319bdf553902..69117864984b 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -73,11 +73,9 @@ func main() { secretsencryptCommand, secretsencryptCommand, ), - cmds.NewCertCommand( - cmds.NewCertSubcommands( - certCommand, - certCommand, - ), + cmds.NewCertCommands( + certCommand, + certCommand, ), cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)), } diff --git a/cmd/server/main.go b/cmd/server/main.go index 5b9cfd6b74c4..30b742018891 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -70,11 +70,9 @@ func main() { secretsencrypt.Reencrypt, secretsencrypt.RotateKeys, ), - cmds.NewCertCommand( - cmds.NewCertSubcommands( - cert.Rotate, - cert.RotateCA, - ), + cmds.NewCertCommands( + cert.Rotate, + cert.RotateCA, ), cmds.NewCompletionCommand(completion.Run), } diff --git a/main.go b/main.go index 312e66abc0ad..8857094b87a2 100644 --- a/main.go +++ b/main.go @@ -47,11 +47,9 @@ func main() { secretsencrypt.Reencrypt, secretsencrypt.RotateKeys, ), - cmds.NewCertCommand( - cmds.NewCertSubcommands( - cert.Rotate, - cert.RotateCA, - ), + cmds.NewCertCommands( + cert.Rotate, + cert.RotateCA, ), cmds.NewCompletionCommand(completion.Run), } diff --git a/pkg/cli/cmds/certs.go b/pkg/cli/cmds/certs.go index 2bb1652f6680..192dbfa6b788 100644 --- a/pkg/cli/cmds/certs.go +++ b/pkg/cli/cmds/certs.go @@ -54,33 +54,29 @@ var ( } ) -func NewCertCommand(subcommands []cli.Command) cli.Command { +func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: CertCommand, Usage: "Manage K3s certificates", SkipFlagParsing: false, SkipArgReorder: true, - Subcommands: subcommands, - } -} - -func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command { - return []cli.Command{ - { - Name: "rotate", - Usage: "Rotate " + version.Program + " component certificates on disk", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: rotate, - Flags: CertRotateCommandFlags, - }, - { - Name: "rotate-ca", - Usage: "Write updated " + version.Program + " CA certificates to the datastore", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: rotateCA, - Flags: CertRotateCACommandFlags, + Subcommands: []cli.Command{ + { + Name: "rotate", + Usage: "Rotate " + version.Program + " component certificates on disk", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotate, + Flags: CertRotateCommandFlags, + }, + { + Name: "rotate-ca", + Usage: "Write updated " + version.Program + " CA certificates to the datastore", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotateCA, + Flags: CertRotateCACommandFlags, + }, }, } } From 958a5aaf86dbbdf2016dc458dba9d0df65cce9c8 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 9 Aug 2023 13:26:20 -0700 Subject: [PATCH 02/13] Rotate front end Signed-off-by: Derek Nola --- cmd/k3s/main.go | 1 + cmd/server/main.go | 1 + cmd/token/main.go | 1 + pkg/cli/cmds/token.go | 25 ++++++++++++++++++++++- pkg/cli/token/token.go | 45 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 1 deletion(-) diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 69117864984b..23eafa2185e3 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -57,6 +57,7 @@ func main() { tokenCommand, tokenCommand, tokenCommand, + tokenCommand, ), cmds.NewEtcdSnapshotCommands( etcdsnapshotCommand, diff --git a/cmd/server/main.go b/cmd/server/main.go index 30b742018891..c95ba98783f7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -54,6 +54,7 @@ func main() { token.Delete, token.Generate, token.List, + token.Rotate, ), cmds.NewEtcdSnapshotCommands( etcdsnapshot.Delete, diff --git a/cmd/token/main.go b/cmd/token/main.go index 26d069fc926f..3edaf99fb47e 100644 --- a/cmd/token/main.go +++ b/cmd/token/main.go @@ -20,6 +20,7 @@ func main() { token.Delete, token.Generate, token.List, + token.Rotate, ), } diff --git a/pkg/cli/cmds/token.go b/pkg/cli/cmds/token.go index 2e3cfe574b58..c9dc86d72563 100644 --- a/pkg/cli/cmds/token.go +++ b/pkg/cli/cmds/token.go @@ -3,6 +3,7 @@ package cmds import ( "time" + "github.com/k3s-io/k3s/pkg/version" "github.com/urfave/cli" ) @@ -12,6 +13,7 @@ const TokenCommand = "token" type Token struct { Description string Kubeconfig string + ServerURL string Token string Output string Groups cli.StringSlice @@ -32,7 +34,7 @@ var ( } ) -func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) error) cli.Command { +func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: TokenCommand, Usage: "Manage bootstrap tokens", @@ -92,6 +94,27 @@ func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) erro SkipArgReorder: true, Action: list, }, + { + Name: "rotate", + Usage: "Rotate original server token with a new bootstrap token", + Flags: append(TokenFlags, + &cli.StringFlag{ + Name: "token,t", + Usage: "(cluster) Shared secret used to join a server or agent to a cluster", + Destination: &TokenConfig.Token, + EnvVar: version.ProgramUpper + "_TOKEN", + }, + &cli.StringFlag{ + Name: "server, s", + Usage: "(cluster) Server to connect to", + Destination: &TokenConfig.ServerURL, + EnvVar: version.ProgramUpper + "_URL", + Value: "https://127.0.0.1:6443", + }), + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotate, + }, }, } } diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 44e7eb99badc..dacd8658fad3 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -1,18 +1,23 @@ package token import ( + "bytes" "context" "encoding/json" "fmt" "os" + "path/filepath" "strings" "text/tabwriter" "time" + "github.com/erikdubbelboer/gspt" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/kubeadm" + "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" "github.com/urfave/cli" "gopkg.in/yaml.v2" @@ -22,6 +27,7 @@ import ( "k8s.io/client-go/tools/clientcmd" bootstrapapi "k8s.io/cluster-bootstrap/token/api" bootstraputil "k8s.io/cluster-bootstrap/token/util" + "k8s.io/utils/pointer" ) func Create(app *cli.Context) error { @@ -139,6 +145,45 @@ func generate(app *cli.Context, cfg *cmds.Token) error { return nil } +func Rotate(app *cli.Context) error { + if err := cmds.InitLogging(); err != nil { + return err + } + info, err := serverAccess(&cmds.TokenConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.ServerTokenRequest{Rotate: pointer.String("rotate")}) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/token/rotate", b); err != nil { + return err + } + fmt.Println("rotating token") + return nil +} + +func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) { + // hide process arguments from ps output, since they likely contain tokens. + gspt.SetProcTitle(os.Args[0] + " token") + + dataDir, err := server.ResolveDataDir("") + if err != nil { + return nil, err + } + + if cfg.Token == "" { + fp := filepath.Join(dataDir, "token") + tokenByte, err := os.ReadFile(fp) + if err != nil { + return nil, err + } + cfg.Token = string(bytes.TrimRight(tokenByte, "\n")) + } + return clientaccess.ParseAndValidateToken(cfg.ServerURL, cfg.Token, clientaccess.WithUser("server")) +} + func List(app *cli.Context) error { if err := cmds.InitLogging(); err != nil { return err From c8b765ad90cdba77fe5c71e1c3e091cec0978bad Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 10 Aug 2023 13:56:36 -0700 Subject: [PATCH 03/13] Initial POC Signed-off-by: Derek Nola --- pkg/cli/token/token.go | 4 +- pkg/cluster/storage.go | 64 ++++++++++++++++- pkg/daemons/control/deps/deps.go | 6 ++ pkg/server/router.go | 1 + pkg/server/token.go | 114 +++++++++++++++++++++++++++++++ 5 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 pkg/server/token.go diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index dacd8658fad3..651d7b2b2a4b 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -153,11 +153,11 @@ func Rotate(app *cli.Context) error { if err != nil { return err } - b, err := json.Marshal(server.ServerTokenRequest{Rotate: pointer.String("rotate")}) + b, err := json.Marshal(server.ServerTokenRequest{Action: pointer.String("rotate")}) if err != nil { return err } - if err = info.Put("/v1-"+version.Program+"/token/rotate", b); err != nil { + if err = info.Put("/v1-"+version.Program+"/token", b); err != nil { return err } fmt.Println("rotating token") diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index b10fe4fc75e1..6e470ace8a04 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -21,6 +21,61 @@ import ( // After this many attempts, the lock is deleted and the counter reset. const maxBootstrapWaitAttempts = 5 +func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { + logrus.Info("Rotating bootstrap token") + + token := config.Token + logrus.Info("SAVE") + logrus.Info("Using token: ", token) + if token == "" { + tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + if err != nil { + return err + } + token = tokenFromFile + logrus.Info("Token from file: ", token) + } + normalizedToken, err := normalizeToken(token) + if err != nil { + return err + } + logrus.Info("Normalized token from file: ", normalizedToken) + + storageClient, err := client.New(config.Runtime.EtcdConfig) + if err != nil { + return err + } + defer storageClient.Close() + + emptyStringKey := storageKey("") + tokenKey := storageKey(normalizedToken) + + var bootstrapList []client.Value + if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) { + bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0) + if err != nil { + if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) { + return false, nil + } + return false, err + } + return true, nil + }); err != nil { + return err + } + + normalizedOldToken, err := normalizeToken(oldToken) + if err != nil { + return err + } + // check for empty string key and for old token format with k10 prefix + if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, normalizedOldToken); err != nil { + return err + } + + return nil +} + // Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token. // This is used when bootstrapping a cluster from a managed database or external etcd cluster. @@ -32,17 +87,21 @@ func Save(ctx context.Context, config *config.Control, override bool) error { return err } token := config.Token + logrus.Info("SAVE") + logrus.Info("Using token: ", token) if token == "" { tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } token = tokenFromFile + logrus.Info("Token from file: ", token) } normalizedToken, err := normalizeToken(token) if err != nil { return err } + logrus.Info("Normalized token from file: ", normalizedToken) data, err := encrypt(normalizedToken, buf.Bytes()) if err != nil { @@ -59,7 +118,7 @@ func Save(ctx context.Context, config *config.Control, override bool) error { if err != nil { return err } - + // I think I need to inject a new token via Save // If there's an empty bootstrap key, then we've locked it and can override. if currentKey != nil && len(currentKey.Data) == 0 { logrus.Info("Bootstrap key lock is held") @@ -236,6 +295,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client } for _, bootstrapKV := range bootstrapList { // ensure bootstrap is stored in the current token's key + logrus.Infof("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey) if string(bootstrapKV.Key) == tokenKey { return &bootstrapKV, false, nil } @@ -283,6 +343,7 @@ func normalizeToken(token string) (string, error) { func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { oldTokenKey := storageKey(oldToken) + logrus.Info("migrateOldTokens: t: ", token, " tK: ", tokenKey, " ot: ", oldToken, " otK: ", oldTokenKey) for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key if string(bootstrapKV.Key) == emptyStringKey { @@ -296,6 +357,7 @@ func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storage return err } } + logrus.Info("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey) } return nil diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index ea74cc94cada..66ec3843faf9 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -239,17 +239,23 @@ func genUsers(config *config.Control) error { return err } + logrus.Info("HELP ", passwd) + if err := migratePassword(passwd); err != nil { return err } + // if no token is provided on bootstrap, we generate a random token serverPass, err := getServerPass(passwd, config) if err != nil { return err } + logrus.Info("HELP 1 ", serverPass) nodePass := getNodePass(config, serverPass) + logrus.Info("HELP 2 ", nodePass) + if err := passwd.EnsureUser("node", version.Program+":agent", nodePass); err != nil { return err } diff --git a/pkg/server/router.go b/pkg/server/router.go index 2232158b00f0..8c05339d9a83 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -85,6 +85,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig)) serverAuthed.Path("/db/info").Handler(nodeAuthed) serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime)) + serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig)) systemAuthed := mux.NewRouter().SkipClean(true) systemAuthed.NotFoundHandler = serverAuthed diff --git a/pkg/server/token.go b/pkg/server/token.go new file mode 100644 index 000000000000..4bc23193fd4b --- /dev/null +++ b/pkg/server/token.go @@ -0,0 +1,114 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "github.com/k3s-io/k3s/pkg/cluster" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/passwd" + "github.com/k3s-io/k3s/pkg/util" + "github.com/sirupsen/logrus" +) + +type ServerTokenRequest struct { + Action *string `json:"stage,omitempty"` +} + +func getServerTokenRequest(req *http.Request) (ServerTokenRequest, error) { + b, err := io.ReadAll(req.Body) + if err != nil { + return ServerTokenRequest{}, err + } + result := ServerTokenRequest{} + err = json.Unmarshal(b, &result) + return result, err +} + +func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil || req.Method != http.MethodPut { + resp.WriteHeader(http.StatusBadRequest) + return + } + logrus.Info("Received token request") + var err error + b := []byte{} + var token string + sTokenReq, err := getServerTokenRequest(req) + if err != nil { + resp.WriteHeader(http.StatusBadRequest) + resp.Write([]byte(err.Error())) + return + } + + if *sTokenReq.Action == "rotate" { + token, err = tokenRotate(ctx, server) + b = []byte(token) + } else { + err = fmt.Errorf("unknown action %s requested", *sTokenReq.Action) + } + + if err != nil { + genErrorMessage(resp, http.StatusInternalServerError, err, "token") + return + } + resp.WriteHeader(http.StatusOK) + resp.Write(b) + }) +} + +func tokenRotate(ctx context.Context, server *config.Control) (string, error) { + passwd, err := passwd.Read(server.Runtime.PasswdFile) + if err != nil { + return "", err + } + + logrus.Info("BEFORE ", passwd) + if err != nil { + return "", err + } + oldToken, found := passwd.Pass("server") + if !found { + return "", fmt.Errorf("server token not found") + } + newToken, err := util.Random(16) + if err != nil { + return "", err + } + // if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil { + // return "", err + // } + // logrus.Info("AFTER ", passwd) + + // if err := passwd.Write(server.Runtime.PasswdFile); err != nil { + // return "", err + // } + + serverTokenFile := filepath.Join(server.DataDir, "token") + b, err := os.ReadFile(serverTokenFile) + if err != nil { + return "", err + } + logrus.Info("OLD TOKEN ", string(b)) + if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil { + return "", err + } + + b, err = os.ReadFile(serverTokenFile) + if err != nil { + return "", err + } + logrus.Info("NEW TOKEN ", string(b)) + + cluster.RotateBootstrapToken(ctx, server, oldToken) + // if err := cluster.Save(ctx, server, true); err != nil { + // return "", err + // } + return newToken, nil +} From 4308d83e72b0a709358bc90ff62b9d41a24b4981 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 28 Aug 2023 15:59:16 -0700 Subject: [PATCH 04/13] Add support for user defined new token Signed-off-by: Derek Nola --- pkg/cli/cmds/token.go | 8 ++++- pkg/cli/token/token.go | 6 +++- pkg/cluster/storage.go | 16 +++------- pkg/server/token.go | 58 +++++++++++++++++------------------ tests/e2e/token/Vagrantfile | 14 ++++++++- tests/e2e/token/token_test.go | 11 +++---- 6 files changed, 64 insertions(+), 49 deletions(-) diff --git a/pkg/cli/cmds/token.go b/pkg/cli/cmds/token.go index c9dc86d72563..beced6426420 100644 --- a/pkg/cli/cmds/token.go +++ b/pkg/cli/cmds/token.go @@ -15,6 +15,7 @@ type Token struct { Kubeconfig string ServerURL string Token string + NewToken string Output string Groups cli.StringSlice Usages cli.StringSlice @@ -100,7 +101,7 @@ func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Conte Flags: append(TokenFlags, &cli.StringFlag{ Name: "token,t", - Usage: "(cluster) Shared secret used to join a server or agent to a cluster", + Usage: "Existing token used to join a server or agent to a cluster", Destination: &TokenConfig.Token, EnvVar: version.ProgramUpper + "_TOKEN", }, @@ -110,6 +111,11 @@ func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Conte Destination: &TokenConfig.ServerURL, EnvVar: version.ProgramUpper + "_URL", Value: "https://127.0.0.1:6443", + }, + &cli.StringFlag{ + Name: "new-token", + Usage: "New token that replaces existing token", + Destination: &TokenConfig.NewToken, }), SkipFlagParsing: false, SkipArgReorder: true, diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 651d7b2b2a4b..95d794ac7549 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -153,7 +153,11 @@ func Rotate(app *cli.Context) error { if err != nil { return err } - b, err := json.Marshal(server.ServerTokenRequest{Action: pointer.String("rotate")}) + b, err := json.Marshal(server.ServerTokenRequest{ + Action: pointer.String("rotate"), + NewToken: pointer.String(cmds.TokenConfig.NewToken), + }) + fmt.Println(cmds.TokenConfig.NewToken) if err != nil { return err } diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 6e470ace8a04..96f11222b8c9 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -22,19 +22,13 @@ import ( const maxBootstrapWaitAttempts = 5 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { - logrus.Info("Rotating bootstrap token") - token := config.Token - logrus.Info("SAVE") - logrus.Info("Using token: ", token) - if token == "" { - tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) - if err != nil { - return err - } - token = tokenFromFile - logrus.Info("Token from file: ", token) + logrus.Info("RotateBootstrapToken") + token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + if err != nil { + return err } + logrus.Info("Token from file: ", token) normalizedToken, err := normalizeToken(token) if err != nil { return err diff --git a/pkg/server/token.go b/pkg/server/token.go index 4bc23193fd4b..68ad102cb179 100644 --- a/pkg/server/token.go +++ b/pkg/server/token.go @@ -17,7 +17,8 @@ import ( ) type ServerTokenRequest struct { - Action *string `json:"stage,omitempty"` + Action *string `json:"stage,omitempty"` + NewToken *string `json:"newToken,omitempty"` } func getServerTokenRequest(req *http.Request) (ServerTokenRequest, error) { @@ -36,20 +37,16 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl resp.WriteHeader(http.StatusBadRequest) return } - logrus.Info("Received token request") var err error - b := []byte{} - var token string sTokenReq, err := getServerTokenRequest(req) + logrus.Info("Received token request: ", *sTokenReq.Action, *sTokenReq.NewToken) if err != nil { resp.WriteHeader(http.StatusBadRequest) resp.Write([]byte(err.Error())) return } - if *sTokenReq.Action == "rotate" { - token, err = tokenRotate(ctx, server) - b = []byte(token) + err = tokenRotate(ctx, server, *sTokenReq.NewToken) } else { err = fmt.Errorf("unknown action %s requested", *sTokenReq.Action) } @@ -59,56 +56,59 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl return } resp.WriteHeader(http.StatusOK) - resp.Write(b) }) } -func tokenRotate(ctx context.Context, server *config.Control) (string, error) { +func tokenRotate(ctx context.Context, server *config.Control, newToken string) error { passwd, err := passwd.Read(server.Runtime.PasswdFile) if err != nil { - return "", err + return err } logrus.Info("BEFORE ", passwd) if err != nil { - return "", err + return err } oldToken, found := passwd.Pass("server") if !found { - return "", fmt.Errorf("server token not found") + return fmt.Errorf("server token not found") } - newToken, err := util.Random(16) - if err != nil { - return "", err + if newToken == "" { + newToken, err = util.Random(16) + if err != nil { + return err + } } + // if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil { - // return "", err + // return err // } // logrus.Info("AFTER ", passwd) - // if err := passwd.Write(server.Runtime.PasswdFile); err != nil { - // return "", err - // } + if err := passwd.Write(server.Runtime.PasswdFile); err != nil { + return err + } serverTokenFile := filepath.Join(server.DataDir, "token") b, err := os.ReadFile(serverTokenFile) if err != nil { - return "", err + return err } - logrus.Info("OLD TOKEN ", string(b)) + logrus.Debug("Old Token File: ", string(b)) + if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil { - return "", err + return err } b, err = os.ReadFile(serverTokenFile) if err != nil { - return "", err + return err } - logrus.Info("NEW TOKEN ", string(b)) + logrus.Debug("New Token File: ", string(b)) - cluster.RotateBootstrapToken(ctx, server, oldToken) - // if err := cluster.Save(ctx, server, true); err != nil { - // return "", err - // } - return newToken, nil + if err := cluster.RotateBootstrapToken(ctx, server, oldToken); err != nil { + return err + } + server.Token = newToken + return cluster.Save(ctx, server, true) } diff --git a/tests/e2e/token/Vagrantfile b/tests/e2e/token/Vagrantfile index f56800c2fcd5..b9d1bf678de7 100644 --- a/tests/e2e/token/Vagrantfile +++ b/tests/e2e/token/Vagrantfile @@ -1,6 +1,6 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || - ["server-0", "agent-0", "agent-1" ]) + ["server-0", "server-1", "server-2", "agent-0", "agent-1" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") @@ -40,6 +40,18 @@ def provision(vm, roles, role_num, node_num) YAML k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] end + elsif roles.include?("server") && role_num != 0 + vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "server" + k3s.config = <<~YAML + server: "https://#{NETWORK_PREFIX}.100:6443" + token: vagrant + node-external-ip: #{node_ip} + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end end if roles.include?("agent") vm.provision :k3s, run: 'once' do |k3s| diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 29778bae10f5..8e359d64dd19 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -3,7 +3,6 @@ package snapshotrestore import ( "flag" "fmt" - "os" "strings" "testing" "time" @@ -17,7 +16,7 @@ import ( // generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.4.x86_64 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") -var serverCount = flag.Int("serverCount", 1, "number of server nodes") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") @@ -42,7 +41,7 @@ var _ = ReportAfterEach(e2e.GenReport) var _ = Describe("Use the token CLI to create and join agents", Ordered, func() { Context("Agent joins with permanent token:", func() { - It("Starts up with no issues", func() { + FIt("Starts up with no issues", func() { var err error if *local { serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) @@ -58,7 +57,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Expect(err).NotTo(HaveOccurred()) }) - It("Checks Node and Pod Status", func() { + FIt("Checks Node and Pod Status", func() { fmt.Printf("\nFetching node status\n") Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) @@ -163,7 +162,7 @@ var _ = AfterSuite(func() { fmt.Println("FAILED!") } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) - Expect(e2e.DestroyCluster()).To(Succeed()) - Expect(os.Remove(kubeConfigFile)).To(Succeed()) + // Expect(e2e.DestroyCluster()).To(Succeed()) + // Expect(os.Remove(kubeConfigFile)).To(Succeed()) } }) From 0d8d187842410274dbf1e13e1b36c0c40e9554d9 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 29 Aug 2023 10:56:34 -0700 Subject: [PATCH 05/13] Add E2E testlets Signed-off-by: Derek Nola --- pkg/cli/token/token.go | 3 ++- tests/e2e/token/token_test.go | 39 +++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 95d794ac7549..44dc77c1d2f8 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -164,7 +164,8 @@ func Rotate(app *cli.Context) error { if err = info.Put("/v1-"+version.Program+"/token", b); err != nil { return err } - fmt.Println("rotating token") + time.Sleep(1 * time.Second) + fmt.Println("token rotated, restart k3s with new token") return nil } diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 8e359d64dd19..226ae35b6f80 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -3,6 +3,7 @@ package snapshotrestore import ( "flag" "fmt" + "os" "strings" "testing" "time" @@ -41,7 +42,7 @@ var _ = ReportAfterEach(e2e.GenReport) var _ = Describe("Use the token CLI to create and join agents", Ordered, func() { Context("Agent joins with permanent token:", func() { - FIt("Starts up with no issues", func() { + It("Starts up with no issues", func() { var err error if *local { serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) @@ -57,7 +58,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Expect(err).NotTo(HaveOccurred()) }) - FIt("Checks Node and Pod Status", func() { + It("Checks Node and Pod Status", func() { fmt.Printf("\nFetching node status\n") Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) @@ -103,7 +104,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).Should(Equal(2)) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 1)) for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } @@ -143,7 +144,33 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).Should(Equal(3)) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "60s", "5s").Should(Succeed()) + }) + }) + Context("Rotate server bootstrap token", func() { + serverToken := "1234" + It("Creates a new server token", func() { + Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])). + To(ContainSubstring("token rotated, restart k3s with new token")) + }) + It("Restarts servers with the new token", func() { + cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) + for _, node := range serverNodeNames { + _, err := e2e.RunCmdOnNode(cmd, node) + Expect(err).NotTo(HaveOccurred()) + } + for _, node := range serverNodeNames { + _, err := e2e.RunCmdOnNode("systemctl restart k3s", node) + Expect(err).NotTo(HaveOccurred()) + } + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } @@ -162,7 +189,7 @@ var _ = AfterSuite(func() { fmt.Println("FAILED!") } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) - // Expect(e2e.DestroyCluster()).To(Succeed()) - // Expect(os.Remove(kubeConfigFile)).To(Succeed()) + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) } }) From d9597b3152f5e5eed81f498cb5e5458437dd22e3 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 29 Aug 2023 11:05:35 -0700 Subject: [PATCH 06/13] Cleanup debug messages Signed-off-by: Derek Nola --- pkg/cluster/storage.go | 13 +++---------- pkg/server/token.go | 15 --------------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 96f11222b8c9..2177067b5615 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -23,17 +23,15 @@ const maxBootstrapWaitAttempts = 5 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { - logrus.Info("RotateBootstrapToken") token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } - logrus.Info("Token from file: ", token) + normalizedToken, err := normalizeToken(token) if err != nil { return err } - logrus.Info("Normalized token from file: ", normalizedToken) storageClient, err := client.New(config.Runtime.EtcdConfig) if err != nil { @@ -41,7 +39,6 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken } defer storageClient.Close() - emptyStringKey := storageKey("") tokenKey := storageKey(normalizedToken) var bootstrapList []client.Value @@ -62,8 +59,8 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken if err != nil { return err } - // check for empty string key and for old token format with k10 prefix - if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, normalizedOldToken); err != nil { + // resuse the existing migration function to reencrypt bootstrap data with new token + if err := migrateOldTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil { return err } @@ -81,21 +78,17 @@ func Save(ctx context.Context, config *config.Control, override bool) error { return err } token := config.Token - logrus.Info("SAVE") - logrus.Info("Using token: ", token) if token == "" { tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } token = tokenFromFile - logrus.Info("Token from file: ", token) } normalizedToken, err := normalizeToken(token) if err != nil { return err } - logrus.Info("Normalized token from file: ", normalizedToken) data, err := encrypt(normalizedToken, buf.Bytes()) if err != nil { diff --git a/pkg/server/token.go b/pkg/server/token.go index 68ad102cb179..c95b69511378 100644 --- a/pkg/server/token.go +++ b/pkg/server/token.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "os" "path/filepath" "github.com/k3s-io/k3s/pkg/cluster" @@ -65,7 +64,6 @@ func tokenRotate(ctx context.Context, server *config.Control, newToken string) e return err } - logrus.Info("BEFORE ", passwd) if err != nil { return err } @@ -83,29 +81,16 @@ func tokenRotate(ctx context.Context, server *config.Control, newToken string) e // if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil { // return err // } - // logrus.Info("AFTER ", passwd) if err := passwd.Write(server.Runtime.PasswdFile); err != nil { return err } serverTokenFile := filepath.Join(server.DataDir, "token") - b, err := os.ReadFile(serverTokenFile) - if err != nil { - return err - } - logrus.Debug("Old Token File: ", string(b)) - if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil { return err } - b, err = os.ReadFile(serverTokenFile) - if err != nil { - return err - } - logrus.Debug("New Token File: ", string(b)) - if err := cluster.RotateBootstrapToken(ctx, server, oldToken); err != nil { return err } From 47a37ef490608ada277a72306b0056f11790dd52 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 29 Aug 2023 11:25:44 -0700 Subject: [PATCH 07/13] More dev logging cleanup Signed-off-by: Derek Nola --- pkg/cluster/storage.go | 7 +++---- pkg/daemons/control/deps/deps.go | 5 ----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 2177067b5615..659fd4a1c484 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -105,7 +105,7 @@ func Save(ctx context.Context, config *config.Control, override bool) error { if err != nil { return err } - // I think I need to inject a new token via Save + // If there's an empty bootstrap key, then we've locked it and can override. if currentKey != nil && len(currentKey.Data) == 0 { logrus.Info("Bootstrap key lock is held") @@ -282,7 +282,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client } for _, bootstrapKV := range bootstrapList { // ensure bootstrap is stored in the current token's key - logrus.Infof("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey) + logrus.Debugf("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey) if string(bootstrapKV.Key) == tokenKey { return &bootstrapKV, false, nil } @@ -330,9 +330,9 @@ func normalizeToken(token string) (string, error) { func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { oldTokenKey := storageKey(oldToken) - logrus.Info("migrateOldTokens: t: ", token, " tK: ", tokenKey, " ot: ", oldToken, " otK: ", oldTokenKey) for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key + logrus.Debug("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey) if string(bootstrapKV.Key) == emptyStringKey { logrus.Warn("Bootstrap data encrypted with empty string, deleting and resaving with token") if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil { @@ -344,7 +344,6 @@ func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storage return err } } - logrus.Info("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey) } return nil diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 66ec3843faf9..5af285c086a9 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -239,8 +239,6 @@ func genUsers(config *config.Control) error { return err } - logrus.Info("HELP ", passwd) - if err := migratePassword(passwd); err != nil { return err } @@ -250,12 +248,9 @@ func genUsers(config *config.Control) error { if err != nil { return err } - logrus.Info("HELP 1 ", serverPass) nodePass := getNodePass(config, serverPass) - logrus.Info("HELP 2 ", nodePass) - if err := passwd.EnsureUser("node", version.Program+":agent", nodePass); err != nil { return err } From 2a366f75a30f58915fab774d2f8622070a441d92 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 29 Aug 2023 12:43:46 -0700 Subject: [PATCH 08/13] Ensure agent token also changes Signed-off-by: Derek Nola --- pkg/cluster/storage.go | 12 +++++++----- pkg/server/token.go | 16 ++++++++++++---- tests/e2e/token/Vagrantfile | 2 +- tests/e2e/token/token_test.go | 18 +++++++++++++++++- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 659fd4a1c484..8f19859db55a 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -60,7 +60,7 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken return err } // resuse the existing migration function to reencrypt bootstrap data with new token - if err := migrateOldTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil { + if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil { return err } @@ -271,7 +271,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client logrus.Warn("found multiple bootstrap keys in storage") } // check for empty string key and for old token format with k10 prefix - if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { + if err := migrateTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { return nil, false, err } @@ -324,10 +324,10 @@ func normalizeToken(token string) (string, error) { return password, nil } -// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is +// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and keys that is hashed with old token format before normalizing // then migrate those and resave only with the normalized token -func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { +func migrateTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { oldTokenKey := storageKey(oldToken) for _, bootstrapKV := range bootstrapList { @@ -339,7 +339,9 @@ func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storage return err } } else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey { - logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token") + if emptyStringKey != "" { + logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token") + } if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil { return err } diff --git a/pkg/server/token.go b/pkg/server/token.go index c95b69511378..27b82d1025de 100644 --- a/pkg/server/token.go +++ b/pkg/server/token.go @@ -12,6 +12,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/passwd" "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/sirupsen/logrus" ) @@ -38,7 +39,7 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl } var err error sTokenReq, err := getServerTokenRequest(req) - logrus.Info("Received token request: ", *sTokenReq.Action, *sTokenReq.NewToken) + logrus.Debug("Received token request") if err != nil { resp.WriteHeader(http.StatusBadRequest) resp.Write([]byte(err.Error())) @@ -78,9 +79,16 @@ func tokenRotate(ctx context.Context, server *config.Control, newToken string) e } } - // if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil { - // return err - // } + if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil { + return err + } + + // If the agent token is the same a server, we need to change both + if agentToken, found := passwd.Pass("node"); found && agentToken == oldToken && server.AgentToken == "" { + if err := passwd.EnsureUser("node", version.Program+":agent", newToken); err != nil { + return err + } + } if err := passwd.Write(server.Runtime.PasswdFile); err != nil { return err diff --git a/tests/e2e/token/Vagrantfile b/tests/e2e/token/Vagrantfile index b9d1bf678de7..0a61b7ac57e3 100644 --- a/tests/e2e/token/Vagrantfile +++ b/tests/e2e/token/Vagrantfile @@ -1,6 +1,6 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || - ["server-0", "server-1", "server-2", "agent-0", "agent-1" ]) + ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 226ae35b6f80..1be715df9a2e 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -122,7 +122,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() It("Cleans up 20s token automatically", func() { Eventually(func() (string, error) { return e2e.RunCmdOnNode("k3s token list", serverNodeNames[0]) - }, "20s", "5s").ShouldNot(ContainSubstring("20sect")) + }, "25s", "5s").ShouldNot(ContainSubstring("20sect")) }) var tempToken string It("Creates a 10m agent token", func() { @@ -176,6 +176,22 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() } }, "60s", "5s").Should(Succeed()) }) + It("Rejoins an agent with the new server token", func() { + cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) + _, err := e2e.RunCmdOnNode(cmd, agentNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + _, err = e2e.RunCmdOnNode("systemctl restart k3s-agent", agentNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "60s", "5s").Should(Succeed()) + }) }) }) From 43793c8eba00b93ac072038fb4ae064e88b0f7b3 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 1 Sep 2023 11:38:19 -0700 Subject: [PATCH 09/13] More dev log removal Signed-off-by: Derek Nola --- pkg/cli/token/token.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 44dc77c1d2f8..c37883a08130 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -157,13 +157,13 @@ func Rotate(app *cli.Context) error { Action: pointer.String("rotate"), NewToken: pointer.String(cmds.TokenConfig.NewToken), }) - fmt.Println(cmds.TokenConfig.NewToken) if err != nil { return err } if err = info.Put("/v1-"+version.Program+"/token", b); err != nil { return err } + // wait for etcd db propagation delay time.Sleep(1 * time.Second) fmt.Println("token rotated, restart k3s with new token") return nil From d5ebf702fa55987bcf2ec4b2e24eb5d740f1f3c7 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 1 Sep 2023 12:37:58 -0700 Subject: [PATCH 10/13] Add a user prompt warning Signed-off-by: Derek Nola --- pkg/cli/cmds/token.go | 4 ++++ pkg/cli/token/token.go | 31 ++++++++++++++++++++++++++++++- tests/e2e/token/token_test.go | 4 ++-- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/pkg/cli/cmds/token.go b/pkg/cli/cmds/token.go index beced6426420..72326b4c8667 100644 --- a/pkg/cli/cmds/token.go +++ b/pkg/cli/cmds/token.go @@ -116,6 +116,10 @@ func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Conte Name: "new-token", Usage: "New token that replaces existing token", Destination: &TokenConfig.NewToken, + }, + &cli.BoolFlag{ + Name: "f,force", + Usage: "Bypass user prompt warnings", }), SkipFlagParsing: false, SkipArgReorder: true, diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index c37883a08130..c0a09e9e35b3 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -1,6 +1,7 @@ package token import ( + "bufio" "bytes" "context" "encoding/json" @@ -19,6 +20,7 @@ import ( "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "gopkg.in/yaml.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -149,6 +151,13 @@ func Rotate(app *cli.Context) error { if err := cmds.InitLogging(); err != nil { return err } + if !app.Bool("force") { + warning := fmt.Sprint("\033[33mWARNING\033[0m: This will replace the existing token with a new one.", + "Recommend keeping a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.") + if !askContinue(warning) { + return nil + } + } info, err := serverAccess(&cmds.TokenConfig) if err != nil { return err @@ -165,10 +174,30 @@ func Rotate(app *cli.Context) error { } // wait for etcd db propagation delay time.Sleep(1 * time.Second) - fmt.Println("token rotated, restart k3s with new token") + fmt.Println("Token rotated, restart k3s with new token") return nil } +func askContinue(prefix string) bool { + reader := bufio.NewReader(os.Stdin) + + for i := 0; i < 3; i++ { + fmt.Printf("%s Continue: [y/n]: \n", prefix) + + resp, err := reader.ReadString('\n') + if err != nil { + logrus.Fatal(err) + } + resp = strings.ToLower(strings.TrimSpace(resp)) + if resp == "y" || resp == "yes" { + return true + } else if resp == "n" || resp == "no" { + return false + } + } + return false +} + func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) { // hide process arguments from ps output, since they likely contain tokens. gspt.SetProcTitle(os.Args[0] + " token") diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 1be715df9a2e..3578b23b8ee6 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -154,8 +154,8 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Context("Rotate server bootstrap token", func() { serverToken := "1234" It("Creates a new server token", func() { - Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])). - To(ContainSubstring("token rotated, restart k3s with new token")) + Expect(e2e.RunCmdOnNode("k3s token rotate -f -t vagrant --new-token="+serverToken, serverNodeNames[0])). + To(ContainSubstring("Token rotated, restart k3s with new token")) }) It("Restarts servers with the new token", func() { cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) From 28a8c7f10f89d44dbcf5ba524514115730ab521d Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 19 Sep 2023 16:02:49 -0700 Subject: [PATCH 11/13] Remove user input, just print the warning Signed-off-by: Derek Nola --- pkg/cli/cmds/token.go | 4 ---- pkg/cli/token/token.go | 30 +----------------------------- tests/e2e/token/token_test.go | 2 +- 3 files changed, 2 insertions(+), 34 deletions(-) diff --git a/pkg/cli/cmds/token.go b/pkg/cli/cmds/token.go index 72326b4c8667..beced6426420 100644 --- a/pkg/cli/cmds/token.go +++ b/pkg/cli/cmds/token.go @@ -116,10 +116,6 @@ func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Conte Name: "new-token", Usage: "New token that replaces existing token", Destination: &TokenConfig.NewToken, - }, - &cli.BoolFlag{ - Name: "f,force", - Usage: "Bypass user prompt warnings", }), SkipFlagParsing: false, SkipArgReorder: true, diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index c0a09e9e35b3..56b8cd8921b5 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -1,7 +1,6 @@ package token import ( - "bufio" "bytes" "context" "encoding/json" @@ -20,7 +19,6 @@ import ( "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/urfave/cli" "gopkg.in/yaml.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -151,13 +149,7 @@ func Rotate(app *cli.Context) error { if err := cmds.InitLogging(); err != nil { return err } - if !app.Bool("force") { - warning := fmt.Sprint("\033[33mWARNING\033[0m: This will replace the existing token with a new one.", - "Recommend keeping a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.") - if !askContinue(warning) { - return nil - } - } + fmt.Println("\033[33mWARNING\033[0m: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.") info, err := serverAccess(&cmds.TokenConfig) if err != nil { return err @@ -178,26 +170,6 @@ func Rotate(app *cli.Context) error { return nil } -func askContinue(prefix string) bool { - reader := bufio.NewReader(os.Stdin) - - for i := 0; i < 3; i++ { - fmt.Printf("%s Continue: [y/n]: \n", prefix) - - resp, err := reader.ReadString('\n') - if err != nil { - logrus.Fatal(err) - } - resp = strings.ToLower(strings.TrimSpace(resp)) - if resp == "y" || resp == "yes" { - return true - } else if resp == "n" || resp == "no" { - return false - } - } - return false -} - func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) { // hide process arguments from ps output, since they likely contain tokens. gspt.SetProcTitle(os.Args[0] + " token") diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 3578b23b8ee6..09ab793838aa 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -154,7 +154,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Context("Rotate server bootstrap token", func() { serverToken := "1234" It("Creates a new server token", func() { - Expect(e2e.RunCmdOnNode("k3s token rotate -f -t vagrant --new-token="+serverToken, serverNodeNames[0])). + Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])). To(ContainSubstring("Token rotated, restart k3s with new token")) }) It("Restarts servers with the new token", func() { From c27afe80e7a296c426bf2a1ee2c8e07f1029cb38 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 4 Oct 2023 09:57:30 -0700 Subject: [PATCH 12/13] Address feedback Signed-off-by: Derek Nola --- pkg/cluster/storage.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 8f19859db55a..70e3961fdd23 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -45,9 +45,6 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) { bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0) if err != nil { - if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) { - return false, nil - } return false, err } return true, nil @@ -59,7 +56,7 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken if err != nil { return err } - // resuse the existing migration function to reencrypt bootstrap data with new token + // reuse the existing migration function to reencrypt bootstrap data with new token if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil { return err } From a250c6a6d361dff6c8da91ca3c41dd021ae2ae1f Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 5 Oct 2023 09:58:31 -0700 Subject: [PATCH 13/13] Remove action request field Signed-off-by: Derek Nola --- pkg/cli/token/token.go | 3 +-- pkg/server/token.go | 17 +++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 56b8cd8921b5..cdb1a56a8754 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -154,8 +154,7 @@ func Rotate(app *cli.Context) error { if err != nil { return err } - b, err := json.Marshal(server.ServerTokenRequest{ - Action: pointer.String("rotate"), + b, err := json.Marshal(server.TokenRotateRequest{ NewToken: pointer.String(cmds.TokenConfig.NewToken), }) if err != nil { diff --git a/pkg/server/token.go b/pkg/server/token.go index 27b82d1025de..d107bbd0ed7c 100644 --- a/pkg/server/token.go +++ b/pkg/server/token.go @@ -16,17 +16,16 @@ import ( "github.com/sirupsen/logrus" ) -type ServerTokenRequest struct { - Action *string `json:"stage,omitempty"` +type TokenRotateRequest struct { NewToken *string `json:"newToken,omitempty"` } -func getServerTokenRequest(req *http.Request) (ServerTokenRequest, error) { +func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) { b, err := io.ReadAll(req.Body) if err != nil { - return ServerTokenRequest{}, err + return TokenRotateRequest{}, err } - result := ServerTokenRequest{} + result := TokenRotateRequest{} err = json.Unmarshal(b, &result) return result, err } @@ -45,13 +44,7 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl resp.Write([]byte(err.Error())) return } - if *sTokenReq.Action == "rotate" { - err = tokenRotate(ctx, server, *sTokenReq.NewToken) - } else { - err = fmt.Errorf("unknown action %s requested", *sTokenReq.Action) - } - - if err != nil { + if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil { genErrorMessage(resp, http.StatusInternalServerError, err, "token") return }