From e590e389a352ff410bace321ddfc2bc40572fdaa Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 9 Oct 2023 10:58:49 -0700 Subject: [PATCH] Server Token Rotation (#8265) * Consolidate NewCertCommands * Add support for user defined new token * Add E2E testlets Signed-off-by: Derek Nola * Ensure agent token also changes Signed-off-by: Derek Nola --- cmd/cert/main.go | 8 +-- cmd/k3s/main.go | 9 ++- cmd/server/main.go | 9 ++- cmd/token/main.go | 1 + main.go | 8 +-- pkg/cli/cmds/certs.go | 40 ++++++------- pkg/cli/cmds/token.go | 31 +++++++++- pkg/cli/token/token.go | 50 ++++++++++++++++ pkg/cluster/storage.go | 55 +++++++++++++++-- pkg/daemons/control/deps/deps.go | 1 + pkg/server/router.go | 1 + pkg/server/token.go | 100 +++++++++++++++++++++++++++++++ tests/e2e/token/Vagrantfile | 14 ++++- tests/e2e/token/token_test.go | 50 ++++++++++++++-- 14 files changed, 325 insertions(+), 52 deletions(-) create mode 100644 pkg/server/token.go diff --git a/cmd/cert/main.go b/cmd/cert/main.go index 1053056eebbc..85d71f07cba5 100644 --- a/cmd/cert/main.go +++ b/cmd/cert/main.go @@ -15,11 +15,9 @@ import ( func main() { app := cmds.NewApp() app.Commands = []cli.Command{ - cmds.NewCertCommand( - cmds.NewCertSubcommands( - cert.Rotate, - cert.RotateCA, - ), + cmds.NewCertCommands( + cert.Rotate, + cert.RotateCA, ), } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index def64bc9a3ee..e40a295586ed 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -57,6 +57,7 @@ func main() { tokenCommand, tokenCommand, tokenCommand, + tokenCommand, ), cmds.NewEtcdSnapshotCommands( etcdsnapshotCommand, @@ -72,11 +73,9 @@ func main() { secretsencryptCommand, secretsencryptCommand, ), - cmds.NewCertCommand( - cmds.NewCertSubcommands( - certCommand, - certCommand, - ), + cmds.NewCertCommands( + certCommand, + certCommand, ), cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)), } diff --git a/cmd/server/main.go b/cmd/server/main.go index 37bf8d2cd326..a96f0927ed03 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -54,6 +54,7 @@ func main() { token.Delete, token.Generate, token.List, + token.Rotate, ), cmds.NewEtcdSnapshotCommands( etcdsnapshot.Delete, @@ -69,11 +70,9 @@ func main() { secretsencrypt.Rotate, secretsencrypt.Reencrypt, ), - cmds.NewCertCommand( - cmds.NewCertSubcommands( - cert.Rotate, - cert.RotateCA, - ), + cmds.NewCertCommands( + cert.Rotate, + cert.RotateCA, ), cmds.NewCompletionCommand(completion.Run), } diff --git a/cmd/token/main.go b/cmd/token/main.go index 26d069fc926f..3edaf99fb47e 100644 --- a/cmd/token/main.go +++ b/cmd/token/main.go @@ -20,6 +20,7 @@ func main() { token.Delete, token.Generate, token.List, + token.Rotate, ), } diff --git a/main.go b/main.go index ad09773f3ec2..86d167d0e391 100644 --- a/main.go +++ b/main.go @@ -46,11 +46,9 @@ func main() { secretsencrypt.Rotate, secretsencrypt.Reencrypt, ), - cmds.NewCertCommand( - cmds.NewCertSubcommands( - cert.Rotate, - cert.RotateCA, - ), + cmds.NewCertCommands( + cert.Rotate, + cert.RotateCA, ), cmds.NewCompletionCommand(completion.Run), } diff --git a/pkg/cli/cmds/certs.go b/pkg/cli/cmds/certs.go index 2bb1652f6680..192dbfa6b788 100644 --- a/pkg/cli/cmds/certs.go +++ b/pkg/cli/cmds/certs.go @@ -54,33 +54,29 @@ var ( } ) -func NewCertCommand(subcommands []cli.Command) cli.Command { +func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: CertCommand, Usage: "Manage K3s certificates", SkipFlagParsing: false, SkipArgReorder: true, - Subcommands: subcommands, - } -} - -func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command { - return []cli.Command{ - { - Name: "rotate", - Usage: "Rotate " + version.Program + " component certificates on disk", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: rotate, - Flags: CertRotateCommandFlags, - }, - { - Name: "rotate-ca", - Usage: "Write updated " + version.Program + " CA certificates to the datastore", - SkipFlagParsing: false, - SkipArgReorder: true, - Action: rotateCA, - Flags: CertRotateCACommandFlags, + Subcommands: []cli.Command{ + { + Name: "rotate", + Usage: "Rotate " + version.Program + " component certificates on disk", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotate, + Flags: CertRotateCommandFlags, + }, + { + Name: "rotate-ca", + Usage: "Write updated " + version.Program + " CA certificates to the datastore", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotateCA, + Flags: CertRotateCACommandFlags, + }, }, } } diff --git a/pkg/cli/cmds/token.go b/pkg/cli/cmds/token.go index 2e3cfe574b58..beced6426420 100644 --- a/pkg/cli/cmds/token.go +++ b/pkg/cli/cmds/token.go @@ -3,6 +3,7 @@ package cmds import ( "time" + "github.com/k3s-io/k3s/pkg/version" "github.com/urfave/cli" ) @@ -12,7 +13,9 @@ const TokenCommand = "token" type Token struct { Description string Kubeconfig string + ServerURL string Token string + NewToken string Output string Groups cli.StringSlice Usages cli.StringSlice @@ -32,7 +35,7 @@ var ( } ) -func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) error) cli.Command { +func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: TokenCommand, Usage: "Manage bootstrap tokens", @@ -92,6 +95,32 @@ func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) erro SkipArgReorder: true, Action: list, }, + { + Name: "rotate", + Usage: "Rotate original server token with a new bootstrap token", + Flags: append(TokenFlags, + &cli.StringFlag{ + Name: "token,t", + Usage: "Existing token used to join a server or agent to a cluster", + Destination: &TokenConfig.Token, + EnvVar: version.ProgramUpper + "_TOKEN", + }, + &cli.StringFlag{ + Name: "server, s", + Usage: "(cluster) Server to connect to", + Destination: &TokenConfig.ServerURL, + EnvVar: version.ProgramUpper + "_URL", + Value: "https://127.0.0.1:6443", + }, + &cli.StringFlag{ + Name: "new-token", + Usage: "New token that replaces existing token", + Destination: &TokenConfig.NewToken, + }), + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotate, + }, }, } } diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 44e7eb99badc..cdb1a56a8754 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -1,18 +1,23 @@ package token import ( + "bytes" "context" "encoding/json" "fmt" "os" + "path/filepath" "strings" "text/tabwriter" "time" + "github.com/erikdubbelboer/gspt" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/kubeadm" + "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" "github.com/urfave/cli" "gopkg.in/yaml.v2" @@ -22,6 +27,7 @@ import ( "k8s.io/client-go/tools/clientcmd" bootstrapapi "k8s.io/cluster-bootstrap/token/api" bootstraputil "k8s.io/cluster-bootstrap/token/util" + "k8s.io/utils/pointer" ) func Create(app *cli.Context) error { @@ -139,6 +145,50 @@ func generate(app *cli.Context, cfg *cmds.Token) error { return nil } +func Rotate(app *cli.Context) error { + if err := cmds.InitLogging(); err != nil { + return err + } + fmt.Println("\033[33mWARNING\033[0m: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.") + info, err := serverAccess(&cmds.TokenConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.TokenRotateRequest{ + NewToken: pointer.String(cmds.TokenConfig.NewToken), + }) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/token", b); err != nil { + return err + } + // wait for etcd db propagation delay + time.Sleep(1 * time.Second) + fmt.Println("Token rotated, restart k3s with new token") + return nil +} + +func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) { + // hide process arguments from ps output, since they likely contain tokens. + gspt.SetProcTitle(os.Args[0] + " token") + + dataDir, err := server.ResolveDataDir("") + if err != nil { + return nil, err + } + + if cfg.Token == "" { + fp := filepath.Join(dataDir, "token") + tokenByte, err := os.ReadFile(fp) + if err != nil { + return nil, err + } + cfg.Token = string(bytes.TrimRight(tokenByte, "\n")) + } + return clientaccess.ParseAndValidateToken(cfg.ServerURL, cfg.Token, clientaccess.WithUser("server")) +} + func List(app *cli.Context) error { if err := cmds.InitLogging(); err != nil { return err diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index b10fe4fc75e1..70e3961fdd23 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -21,6 +21,49 @@ import ( // After this many attempts, the lock is deleted and the counter reset. const maxBootstrapWaitAttempts = 5 +func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { + + token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + if err != nil { + return err + } + + normalizedToken, err := normalizeToken(token) + if err != nil { + return err + } + + storageClient, err := client.New(config.Runtime.EtcdConfig) + if err != nil { + return err + } + defer storageClient.Close() + + tokenKey := storageKey(normalizedToken) + + var bootstrapList []client.Value + if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) { + bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0) + if err != nil { + return false, err + } + return true, nil + }); err != nil { + return err + } + + normalizedOldToken, err := normalizeToken(oldToken) + if err != nil { + return err + } + // reuse the existing migration function to reencrypt bootstrap data with new token + if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil { + return err + } + + return nil +} + // Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token. // This is used when bootstrapping a cluster from a managed database or external etcd cluster. @@ -225,7 +268,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client logrus.Warn("found multiple bootstrap keys in storage") } // check for empty string key and for old token format with k10 prefix - if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { + if err := migrateTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { return nil, false, err } @@ -236,6 +279,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client } for _, bootstrapKV := range bootstrapList { // ensure bootstrap is stored in the current token's key + logrus.Debugf("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey) if string(bootstrapKV.Key) == tokenKey { return &bootstrapKV, false, nil } @@ -277,21 +321,24 @@ func normalizeToken(token string) (string, error) { return password, nil } -// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is +// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and keys that is hashed with old token format before normalizing // then migrate those and resave only with the normalized token -func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { +func migrateTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { oldTokenKey := storageKey(oldToken) for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key + logrus.Debug("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey) if string(bootstrapKV.Key) == emptyStringKey { logrus.Warn("Bootstrap data encrypted with empty string, deleting and resaving with token") if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil { return err } } else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey { - logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token") + if emptyStringKey != "" { + logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token") + } if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil { return err } diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 60ed40241d39..557706fab39e 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -243,6 +243,7 @@ func genUsers(config *config.Control) error { return err } + // if no token is provided on bootstrap, we generate a random token serverPass, err := getServerPass(passwd, config) if err != nil { return err diff --git a/pkg/server/router.go b/pkg/server/router.go index 2232158b00f0..8c05339d9a83 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -85,6 +85,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig)) serverAuthed.Path("/db/info").Handler(nodeAuthed) serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime)) + serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig)) systemAuthed := mux.NewRouter().SkipClean(true) systemAuthed.NotFoundHandler = serverAuthed diff --git a/pkg/server/token.go b/pkg/server/token.go new file mode 100644 index 000000000000..d107bbd0ed7c --- /dev/null +++ b/pkg/server/token.go @@ -0,0 +1,100 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + + "github.com/k3s-io/k3s/pkg/cluster" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/passwd" + "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" + "github.com/sirupsen/logrus" +) + +type TokenRotateRequest struct { + NewToken *string `json:"newToken,omitempty"` +} + +func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) { + b, err := io.ReadAll(req.Body) + if err != nil { + return TokenRotateRequest{}, err + } + result := TokenRotateRequest{} + err = json.Unmarshal(b, &result) + return result, err +} + +func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil || req.Method != http.MethodPut { + resp.WriteHeader(http.StatusBadRequest) + return + } + var err error + sTokenReq, err := getServerTokenRequest(req) + logrus.Debug("Received token request") + if err != nil { + resp.WriteHeader(http.StatusBadRequest) + resp.Write([]byte(err.Error())) + return + } + if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil { + genErrorMessage(resp, http.StatusInternalServerError, err, "token") + return + } + resp.WriteHeader(http.StatusOK) + }) +} + +func tokenRotate(ctx context.Context, server *config.Control, newToken string) error { + passwd, err := passwd.Read(server.Runtime.PasswdFile) + if err != nil { + return err + } + + if err != nil { + return err + } + oldToken, found := passwd.Pass("server") + if !found { + return fmt.Errorf("server token not found") + } + if newToken == "" { + newToken, err = util.Random(16) + if err != nil { + return err + } + } + + if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil { + return err + } + + // If the agent token is the same a server, we need to change both + if agentToken, found := passwd.Pass("node"); found && agentToken == oldToken && server.AgentToken == "" { + if err := passwd.EnsureUser("node", version.Program+":agent", newToken); err != nil { + return err + } + } + + if err := passwd.Write(server.Runtime.PasswdFile); err != nil { + return err + } + + serverTokenFile := filepath.Join(server.DataDir, "token") + if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil { + return err + } + + if err := cluster.RotateBootstrapToken(ctx, server, oldToken); err != nil { + return err + } + server.Token = newToken + return cluster.Save(ctx, server, true) +} diff --git a/tests/e2e/token/Vagrantfile b/tests/e2e/token/Vagrantfile index f56800c2fcd5..0a61b7ac57e3 100644 --- a/tests/e2e/token/Vagrantfile +++ b/tests/e2e/token/Vagrantfile @@ -1,6 +1,6 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || - ["server-0", "agent-0", "agent-1" ]) + ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") @@ -40,6 +40,18 @@ def provision(vm, roles, role_num, node_num) YAML k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] end + elsif roles.include?("server") && role_num != 0 + vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s| + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + k3s.args = "server" + k3s.config = <<~YAML + server: "https://#{NETWORK_PREFIX}.100:6443" + token: vagrant + node-external-ip: #{node_ip} + flannel-iface: eth1 + YAML + k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type] + end end if roles.include?("agent") vm.provision :k3s, run: 'once' do |k3s| diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 29778bae10f5..09ab793838aa 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -17,7 +17,7 @@ import ( // generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.4.x86_64 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") -var serverCount = flag.Int("serverCount", 1, "number of server nodes") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") @@ -104,7 +104,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).Should(Equal(2)) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 1)) for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } @@ -122,7 +122,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() It("Cleans up 20s token automatically", func() { Eventually(func() (string, error) { return e2e.RunCmdOnNode("k3s token list", serverNodeNames[0]) - }, "20s", "5s").ShouldNot(ContainSubstring("20sect")) + }, "25s", "5s").ShouldNot(ContainSubstring("20sect")) }) var tempToken string It("Creates a 10m agent token", func() { @@ -144,7 +144,49 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).Should(Equal(3)) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "60s", "5s").Should(Succeed()) + }) + }) + Context("Rotate server bootstrap token", func() { + serverToken := "1234" + It("Creates a new server token", func() { + Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])). + To(ContainSubstring("Token rotated, restart k3s with new token")) + }) + It("Restarts servers with the new token", func() { + cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) + for _, node := range serverNodeNames { + _, err := e2e.RunCmdOnNode(cmd, node) + Expect(err).NotTo(HaveOccurred()) + } + for _, node := range serverNodeNames { + _, err := e2e.RunCmdOnNode("systemctl restart k3s", node) + Expect(err).NotTo(HaveOccurred()) + } + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "60s", "5s").Should(Succeed()) + }) + It("Rejoins an agent with the new server token", func() { + cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) + _, err := e2e.RunCmdOnNode(cmd, agentNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + _, err = e2e.RunCmdOnNode("systemctl restart k3s-agent", agentNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) }