From 08bd281897af48ff9883c3a426302bbef7d5b389 Mon Sep 17 00:00:00 2001 From: Liang Deng <283304489@qq.com> Date: Wed, 29 Mar 2023 20:09:51 +0800 Subject: [PATCH] feat: add yurtadm renew certificate command (#1314) Signed-off-by: Liang Deng <283304489@qq.com> --- go.mod | 1 + go.sum | 6 +- pkg/node-servant/components/yurthub.go | 44 ++++- pkg/yurtadm/cmd/cmd.go | 2 + pkg/yurtadm/cmd/join/phases/postcheck.go | 49 +---- pkg/yurtadm/cmd/join/phases/prepare.go | 53 +----- .../cmd/renew/certificate/certificate.go | 164 +++++++++++++++++ pkg/yurtadm/cmd/renew/renew.go | 67 +++++++ pkg/yurtadm/constants/constants.go | 7 +- pkg/yurtadm/util/error/error.go | 114 ++++++++++++ pkg/yurtadm/util/error/error_test.go | 59 ++++++ pkg/yurtadm/util/kubernetes/kubernetes.go | 13 ++ pkg/yurtadm/util/yurthub/yurthub.go | 169 ++++++++++++++++++ 13 files changed, 652 insertions(+), 96 deletions(-) create mode 100644 pkg/yurtadm/cmd/renew/certificate/certificate.go create mode 100644 pkg/yurtadm/cmd/renew/renew.go create mode 100644 pkg/yurtadm/util/error/error.go create mode 100644 pkg/yurtadm/util/error/error_test.go create mode 100644 pkg/yurtadm/util/yurthub/yurthub.go diff --git a/go.mod b/go.mod index df8b9b19361..41b7e416f68 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,7 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spf13/afero v1.6.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect go.opentelemetry.io/contrib v0.20.0 // indirect diff --git a/go.sum b/go.sum index ca5676cc97b..6626920d985 100644 --- a/go.sum +++ b/go.sum @@ -400,6 +400,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -508,6 +509,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -572,8 +574,9 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -705,6 +708,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go index f363deda7d1..7114ee87a11 100644 --- a/pkg/node-servant/components/yurthub.go +++ b/pkg/node-servant/components/yurthub.go @@ -27,14 +27,16 @@ import ( "strings" "time" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -42,6 +44,8 @@ import ( const ( hubHealthzCheckFrequency = 10 * time.Second fileMode = 0666 + DefaultRootDir = "/var/lib" + DefaultCaPath = "/etc/kubernetes/pki/ca.crt" ) type yurtHubOperator struct { @@ -79,7 +83,7 @@ func (op *yurtHubOperator) Install() error { "yurthubServerAddr": constants.DefaultYurtHubServerAddr, "kubernetesServerAddr": op.apiServerAddr, "image": op.yurthubImage, - "joinToken": op.joinToken, + "bootstrapFile": constants.YurtHubBootstrapConfig, "workingMode": string(op.workingMode), "enableDummyIf": strconv.FormatBool(op.enableDummyIf), "enableNodePool": strconv.FormatBool(op.enableNodePool), @@ -88,7 +92,16 @@ func (op *yurtHubOperator) Install() error { return err } - // 1-2. create yurthub.yaml + // 1-2. create /var/lib/yurthub/bootstrap-hub.conf + if err := enutil.EnsureDir(constants.YurtHubWorkdir); err != nil { + return err + } + if err := setHubBootstrapConfig(op.apiServerAddr, op.joinToken); err != nil { + return err + } + klog.Infof("create the %s", constants.YurtHubBootstrapConfig) + + // 1-3. create yurthub.yaml podManifestPath := enutil.GetPodManifestPath() if err := enutil.EnsureDir(podManifestPath); err != nil { return err @@ -151,7 +164,7 @@ func getYurthubYaml(podManifestPath string) string { } func getYurthubConf() string { - return filepath.Join(token.DefaultRootDir, projectinfo.GetHubName()) + return filepath.Join(DefaultRootDir, projectinfo.GetHubName()) } func getYurthubCacheDir() string { @@ -221,3 +234,26 @@ func pingClusterHealthz(client *http.Client, addr string) (bool, error) { return true, nil } + +func setHubBootstrapConfig(serverAddr string, joinToken string) error { + caData, err := os.ReadFile(DefaultCaPath) + if err != nil { + return err + } + tlsBootstrapCfg := kubeconfigutil.CreateWithToken( + serverAddr, + "openyurt-e2e-test", + "token-bootstrap-client", + caData, + joinToken, + ) + content, err := clientcmd.Write(*tlsBootstrapCfg) + if err != nil { + return err + } + if err := os.WriteFile(constants.YurtHubBootstrapConfig, content, fileMode); err != nil { + return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk") + } + + return nil +} diff --git a/pkg/yurtadm/cmd/cmd.go b/pkg/yurtadm/cmd/cmd.go index 2af8ffc80af..761310725a6 100644 --- a/pkg/yurtadm/cmd/cmd.go +++ b/pkg/yurtadm/cmd/cmd.go @@ -28,6 +28,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/docs" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join" + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/reset" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/token" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/yurtinit" @@ -50,6 +51,7 @@ func NewYurtadmCommand() *cobra.Command { cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, os.Stderr)) cmds.AddCommand(token.NewCmdToken(os.Stdin, os.Stdout, os.Stderr)) cmds.AddCommand(docs.NewDocsCmd(cmds)) + cmds.AddCommand(renew.NewCmdRenew(os.Stdin, os.Stdout, os.Stderr)) klog.InitFlags(nil) // goflag.Parse() diff --git a/pkg/yurtadm/cmd/join/phases/postcheck.go b/pkg/yurtadm/cmd/join/phases/postcheck.go index aaa77e17dcc..a70cae1cb52 100644 --- a/pkg/yurtadm/cmd/join/phases/postcheck.go +++ b/pkg/yurtadm/cmd/join/phases/postcheck.go @@ -17,64 +17,31 @@ limitations under the License. package phases import ( - "fmt" - "io" - "net/http" - "time" - - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" - "github.com/openyurtio/openyurt/pkg/yurtadm/constants" - "github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" ) -// RunPostCheck executes the node health check process. +// RunPostCheck executes the node health check and clean process. func RunPostCheck(data joindata.YurtJoinData) error { klog.V(1).Infof("check kubelet status.") - if err := checkKubeletStatus(); err != nil { + if err := kubernetes.CheckKubeletStatus(); err != nil { return err } klog.V(1).Infof("kubelet service is active") klog.V(1).Infof("waiting hub agent ready.") - if err := checkYurthubHealthz(data); err != nil { + if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil { return err } klog.V(1).Infof("hub agent is ready") - return nil -} - -// checkKubeletStatus check if kubelet is healthy. -func checkKubeletStatus() error { - initSystem, err := initsystem.GetInitSystem() - if err != nil { + if err := yurthub.CleanHubBootstrapConfig(); err != nil { return err } - if ok := initSystem.ServiceIsActive("kubelet"); !ok { - return fmt.Errorf("kubelet is not active. ") - } - return nil -} + klog.V(1).Infof("clean yurthub bootstrap config file success") -// checkYurthubHealthz check if YurtHub is healthy. -func checkYurthubHealthz(joinData joindata.YurtJoinData) error { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", joinData.YurtHubServer()), constants.ServerHealthzURLPath), nil) - if err != nil { - return err - } - client := &http.Client{} - return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { - resp, err := client.Do(req) - if err != nil { - return false, nil - } - ok, err := io.ReadAll(resp.Body) - if err != nil { - return false, nil - } - return string(ok) == "OK", nil - }) + return nil } diff --git a/pkg/yurtadm/cmd/join/phases/prepare.go b/pkg/yurtadm/cmd/join/phases/prepare.go index acc46c4f2a1..6d21954b642 100644 --- a/pkg/yurtadm/cmd/join/phases/prepare.go +++ b/pkg/yurtadm/cmd/join/phases/prepare.go @@ -17,18 +17,16 @@ limitations under the License. package phases import ( - "fmt" "os" "path/filepath" - "strings" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" "github.com/openyurtio/openyurt/pkg/yurtadm/util/system" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" ) // RunPrepare executes the node initialization process. @@ -66,58 +64,17 @@ func RunPrepare(data joindata.YurtJoinData) error { if err := yurtadmutil.SetKubeletConfigForNode(); err != nil { return err } - if err := addYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil { + if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil { return err } - if err := yurtadmutil.SetDiscoveryConfig(data); err != nil { + if err := yurthub.AddYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil { return err } - if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil { - return err - } - return nil -} - -// addYurthubStaticYaml generate YurtHub static yaml for worker node. -func addYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error { - klog.Info("[join-node] Adding edge hub static yaml") - if _, err := os.Stat(podManifestPath); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(podManifestPath, os.ModePerm) - if err != nil { - return err - } - } else { - klog.Errorf("Describe dir %s fail: %v", podManifestPath, err) - return err - } - } - - // There can be multiple master IP addresses - serverAddrs := strings.Split(data.ServerAddr(), ",") - for i := 0; i < len(serverAddrs); i++ { - serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i]) - } - - kubernetesServerAddrs := strings.Join(serverAddrs, ",") - - ctx := map[string]string{ - "kubernetesServerAddr": kubernetesServerAddrs, - "image": data.YurtHubImage(), - "joinToken": data.JoinToken(), - "workingMode": data.NodeRegistration().WorkingMode, - "organizations": data.NodeRegistration().Organizations, - "yurthubServerAddr": data.YurtHubServer(), - } - - yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx) - if err != nil { + if err := yurtadmutil.SetDiscoveryConfig(data); err != nil { return err } - - if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil { + if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil { return err } - klog.Info("[join-node] Add hub agent static yaml is ok") return nil } diff --git a/pkg/yurtadm/cmd/renew/certificate/certificate.go b/pkg/yurtadm/cmd/renew/certificate/certificate.go new file mode 100644 index 00000000000..172f7d1face --- /dev/null +++ b/pkg/yurtadm/cmd/renew/certificate/certificate.go @@ -0,0 +1,164 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "fmt" + "net/url" + "strings" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + yurtconstants "github.com/openyurtio/openyurt/pkg/yurtadm/constants" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" +) + +type certificateOptions struct { + token string + caCertHashes []string + unsafeSkipCAVerification bool + serverAddr string + yurthubServer string +} + +// newCertificateOptions returns a struct ready for being used for creating cmd renew flags. +func newCertificateOptions() *certificateOptions { + return &certificateOptions{ + caCertHashes: make([]string, 0), + unsafeSkipCAVerification: true, + yurthubServer: yurtconstants.DefaultYurtHubServerAddr, + } +} + +// NewCmdCertificate returns "yurtadm renew certificate" command. +func NewCmdCertificate() *cobra.Command { + o := newCertificateOptions() + + certificateCmd := &cobra.Command{ + Use: "certificate", + Short: "Create bootstrap file for yurthub to update certificate", + RunE: func(certificateCmd *cobra.Command, args []string) error { + if err := o.validate(); err != nil { + klog.Fatalf("validate options: %v", err) + } + + // Check if YurtHub's certificate is ready or not. + // No update if ready. + if ok := yurthub.CheckYurthubReadyzOnce(o.yurthubServer); ok { + klog.Infoln("The certificate is still valid and does not need to be renewed") + return nil + } + + us, err := parseRemoteServers(o.serverAddr) + if err != nil { + return err + } + + // 1.Create a temporary bootstrap file and set to /var/lib/yurthub directory. + if err := yurthub.SetHubBootstrapConfig(us[0].Host, o.token, o.caCertHashes); err != nil { + return err + } + + // 2.Check if YurtHub's certificates is ready or not. + if err := yurthub.CheckYurthubReadyz(o.yurthubServer); err != nil { + return err + } + + // 3.Delete temporary bootstrap file. + if err := yurthub.CleanHubBootstrapConfig(); err != nil { + return err + } + + klog.Infoln("Certificate renewed successfully") + + return nil + }, + } + + addCertificateConfigFlags(certificateCmd.Flags(), o) + return certificateCmd +} + +func (options *certificateOptions) validate() error { + if len(options.serverAddr) == 0 { + return fmt.Errorf("server-address is empty") + } + + if len(options.token) == 0 { + return fmt.Errorf("bootstrap token is empty") + } + + if len(options.caCertHashes) == 0 && !options.unsafeSkipCAVerification { + return fmt.Errorf("set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue") + } + + return nil +} + +func parseRemoteServers(serverAddr string) ([]*url.URL, error) { + servers := strings.Split(serverAddr, ",") + us := make([]*url.URL, 0, len(servers)) + remoteServers := make([]string, 0, len(servers)) + for _, server := range servers { + u, err := url.Parse(server) + if err != nil { + klog.Errorf("failed to parse server address %s, %v", servers, err) + return us, err + } + if u.Scheme == "" { + u.Scheme = "https" + } else if u.Scheme != "https" { + return us, fmt.Errorf("only https scheme is supported for server address(%s)", serverAddr) + } + us = append(us, u) + remoteServers = append(remoteServers, u.String()) + } + + if len(us) < 1 { + return us, fmt.Errorf("no server address is set, can not connect remote server") + } + klog.Infof("%s would connect remote servers: %s", projectinfo.GetHubName(), strings.Join(remoteServers, ",")) + + return us, nil +} + +// addCertificateConfigFlags adds certificate flags bound to the config to the specified flagset +func addCertificateConfigFlags(flagSet *flag.FlagSet, certificateOptions *certificateOptions) { + flagSet.StringVar( + &certificateOptions.token, yurtconstants.TokenStr, certificateOptions.token, + "Use this token for bootstrapping yurthub.", + ) + flagSet.StringSliceVar( + &certificateOptions.caCertHashes, yurtconstants.TokenDiscoveryCAHash, certificateOptions.caCertHashes, + "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").", + ) + flagSet.BoolVar( + &certificateOptions.unsafeSkipCAVerification, yurtconstants.TokenDiscoverySkipCAHash, certificateOptions.unsafeSkipCAVerification, + "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.", + ) + flagSet.StringVar( + &certificateOptions.serverAddr, yurtconstants.ServerAddr, certificateOptions.serverAddr, + "The address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"", + ) + flagSet.StringVar( + &certificateOptions.yurthubServer, yurtconstants.YurtHubServerAddr, certificateOptions.yurthubServer, + "Sets the address for yurthub server addr", + ) +} diff --git a/pkg/yurtadm/cmd/renew/renew.go b/pkg/yurtadm/cmd/renew/renew.go new file mode 100644 index 00000000000..8d2cd9ba4c1 --- /dev/null +++ b/pkg/yurtadm/cmd/renew/renew.go @@ -0,0 +1,67 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package renew + +import ( + "fmt" + "io" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew/certificate" + util "github.com/openyurtio/openyurt/pkg/yurtadm/util/error" +) + +// NewCmdRenew returns "yurtadm renew" command. +func NewCmdRenew(in io.Reader, out io.Writer, outErr io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "renew", + Short: "Renew something like certificate", + // Without this callback, if a user runs just the "renew" + // command without a subcommand, or with an invalid subcommand, + // cobra will print usage information, but still exit cleanly. + // We want to return an error code in these cases so that the + // user knows that their command was invalid. + Run: subCmdRun(), + } + + cmd.AddCommand(certificate.NewCmdCertificate()) + return cmd +} + +// subCmdRun returns a function that handles a case where a subcommand must be specified +// Without this callback, if a user runs just the command without a subcommand, +// or with an invalid subcommand, cobra will print usage information, but still exit cleanly. +func subCmdRun() func(c *cobra.Command, args []string) { + return func(c *cobra.Command, args []string) { + if len(args) > 0 { + util.CheckErr(usageErrorf(c, "invalid subcommand %q", strings.Join(args, " "))) + } + err := c.Help() + if err != nil { + return + } + util.CheckErr(util.ErrExit) + } +} + +func usageErrorf(c *cobra.Command, format string, args ...interface{}) error { + msg := fmt.Sprintf(format, args...) + return errors.Errorf("%s\nSee '%s -h' for help and examples", msg, c.CommandPath()) +} diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go index 6aac12e21f4..1b73f798229 100644 --- a/pkg/yurtadm/constants/constants.go +++ b/pkg/yurtadm/constants/constants.go @@ -23,10 +23,10 @@ const ( KubeletConfigureDir = "/etc/kubernetes" KubeletWorkdir = "/var/lib/kubelet" YurtHubWorkdir = "/var/lib/yurthub" + YurtHubBootstrapConfig = "/var/lib/yurthub/bootstrap-hub.conf" OpenyurtDir = "/var/lib/openyurt" YurttunnelAgentWorkdir = "/var/lib/yurttunnel-agent" YurttunnelServerWorkdir = "/var/lib/yurttunnel-server" - KubeConfigPath = "/etc/kubernetes/kubelet.conf" KubeCniDir = "/opt/cni/bin" KubeCniVersion = "v0.8.0" KubeletServiceFilepath = "/etc/systemd/system/kubelet.service" @@ -97,11 +97,14 @@ const ( YurtHubImage = "yurthub-image" // YurtHubServerAddr flag set the address of yurthub server (not proxy server!) YurtHubServerAddr = "yurthub-server-addr" + // ServerAddr flag set the address of kubernetes kube-apiserver + ServerAddr = "server-addr" // ReuseCNIBin flag sets whether to reuse local CNI binaries or not. ReuseCNIBin = "reuse-cni-bin" ServerHealthzServer = "127.0.0.1:10267" ServerHealthzURLPath = "/v1/healthz" + ServerReadyzURLPath = "/v1/readyz" DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt" DefaultOpenYurtVersion = "latest" Yurthub = "yurthub" @@ -219,7 +222,7 @@ spec: - --bind-address={{.yurthubServerAddr}} - --server-addr={{.kubernetesServerAddr}} - --node-name=$(NODE_NAME) - - --join-token={{.joinToken}} + - --bootstrap-file={{.bootstrapFile}} - --working-mode={{.workingMode}} {{if .enableDummyIf }} - --enable-dummy-if={{.enableDummyIf}} diff --git a/pkg/yurtadm/util/error/error.go b/pkg/yurtadm/util/error/error.go new file mode 100644 index 00000000000..dc0aa7a55f8 --- /dev/null +++ b/pkg/yurtadm/util/error/error.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The OpenYurt Authors. +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/pkg/errors" + errorsutil "k8s.io/apimachinery/pkg/util/errors" +) + +const ( + // DefaultErrorExitCode defines exit the code for failed action generally + DefaultErrorExitCode = 1 + // PreFlightExitCode defines exit the code for preflight checks + PreFlightExitCode = 2 + // ValidationExitCode defines the exit code validation checks + ValidationExitCode = 3 +) + +var ( + ErrInvalidSubCommandMsg = "invalid subcommand" + ErrExit = errors.New("exit") +) + +// fatal prints the message if set and then exits. +func fatal(msg string, code int) { + if len(msg) > 0 { + // add newline if needed + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + + fmt.Fprint(os.Stderr, msg) + } + os.Exit(code) +} + +// CheckErr prints a user friendly error to STDERR and exits with a non-zero +// exit code. Unrecognized errors will be printed with an "error: " prefix. +// +// This method is generic to the command in use and may be used by non-Kubectl +// commands. +func CheckErr(err error) { + checkErr(err, fatal) +} + +// preflightError allows us to know if the error is a preflight error or not +// defining the interface here avoids an import cycle of pulling in preflight into the util package +type preflightError interface { + Preflight() bool +} + +// checkErr formats a given error as a string and calls the passed handleErr +// func with that string and an exit code. +func checkErr(err error, handleErr func(string, int)) { + + var msg string + if err != nil { + msg = fmt.Sprintf("%s\nTo see the stack trace of this error execute with --v=5 or higher", err.Error()) + // check if the verbosity level in klog is high enough and print a stack trace. + f := flag.CommandLine.Lookup("v") + if f != nil { + // assume that the "v" flag contains a parseable Int32 as per klog's "Level" type alias, + // thus no error from ParseInt is handled here. + if v, e := strconv.ParseInt(f.Value.String(), 10, 32); e == nil { + // https://git.k8s.io/community/contributors/devel/sig-instrumentation/logging.md + // klog.V(5) - Trace level verbosity + if v > 4 { + msg = fmt.Sprintf("%+v", err) + } + } + } + } + + if err == nil { + return + } + switch { + case err == ErrExit: + handleErr("", DefaultErrorExitCode) + case strings.Contains(err.Error(), ErrInvalidSubCommandMsg): + handleErr(err.Error(), DefaultErrorExitCode) + default: + switch err.(type) { + case preflightError: + handleErr(msg, PreFlightExitCode) + case errorsutil.Aggregate: + handleErr(msg, ValidationExitCode) + + default: + handleErr(msg, DefaultErrorExitCode) + } + } +} diff --git a/pkg/yurtadm/util/error/error_test.go b/pkg/yurtadm/util/error/error_test.go new file mode 100644 index 00000000000..3c53a302c43 --- /dev/null +++ b/pkg/yurtadm/util/error/error_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The OpenYurt Authors. +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/pkg/errors" +) + +type pferror struct{} + +func (p *pferror) Preflight() bool { return true } +func (p *pferror) Error() string { return "" } +func TestCheckErr(t *testing.T) { + var codeReturned int + errHandle := func(err string, code int) { + codeReturned = code + } + + var tests = []struct { + name string + e error + expected int + }{ + {"error is nil", nil, 0}, + {"empty error", errors.New(""), DefaultErrorExitCode}, + {"preflight error", &pferror{}, PreFlightExitCode}, + } + + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + codeReturned = 0 + checkErr(rt.e, errHandle) + if codeReturned != rt.expected { + t.Errorf( + "failed checkErr:\n\texpected: %d\n\t actual: %d", + rt.expected, + codeReturned, + ) + } + }) + } +} diff --git a/pkg/yurtadm/util/kubernetes/kubernetes.go b/pkg/yurtadm/util/kubernetes/kubernetes.go index d9f26646aec..1ef2ab89f5c 100644 --- a/pkg/yurtadm/util/kubernetes/kubernetes.go +++ b/pkg/yurtadm/util/kubernetes/kubernetes.go @@ -47,6 +47,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtadm/constants" "github.com/openyurtio/openyurt/pkg/yurtadm/util" "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem" ) const ( @@ -475,3 +476,15 @@ func RetrieveBootstrapConfig(data joindata.YurtJoinData) (*clientcmdapi.Config, data.JoinToken(), ), nil } + +// CheckKubeletStatus check if kubelet is healthy. +func CheckKubeletStatus() error { + initSystem, err := initsystem.GetInitSystem() + if err != nil { + return err + } + if ok := initSystem.ServiceIsActive("kubelet"); !ok { + return fmt.Errorf("kubelet is not active. ") + } + return nil +} diff --git a/pkg/yurtadm/util/yurthub/yurthub.go b/pkg/yurtadm/util/yurthub/yurthub.go new file mode 100644 index 00000000000..27539ce9ed8 --- /dev/null +++ b/pkg/yurtadm/util/yurthub/yurthub.go @@ -0,0 +1,169 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurthub + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" + "github.com/openyurtio/openyurt/pkg/util/templates" + "github.com/openyurtio/openyurt/pkg/util/token" + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" + "github.com/openyurtio/openyurt/pkg/yurtadm/constants" +) + +// AddYurthubStaticYaml generate YurtHub static yaml for worker node. +func AddYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error { + klog.Info("[join-node] Adding edge hub static yaml") + if _, err := os.Stat(podManifestPath); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(podManifestPath, os.ModePerm) + if err != nil { + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", podManifestPath, err) + return err + } + } + + // There can be multiple master IP addresses + serverAddrs := strings.Split(data.ServerAddr(), ",") + for i := 0; i < len(serverAddrs); i++ { + serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i]) + } + + kubernetesServerAddrs := strings.Join(serverAddrs, ",") + + ctx := map[string]string{ + "kubernetesServerAddr": kubernetesServerAddrs, + "image": data.YurtHubImage(), + "bootstrapFile": constants.YurtHubBootstrapConfig, + "workingMode": data.NodeRegistration().WorkingMode, + "organizations": data.NodeRegistration().Organizations, + "yurthubServerAddr": data.YurtHubServer(), + } + + yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx) + if err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil { + return err + } + klog.Info("[join-node] Add hub agent static yaml is ok") + return nil +} + +func SetHubBootstrapConfig(serverAddr string, joinToken string, caCertHashes []string) error { + if cfg, err := token.RetrieveValidatedConfigInfo(nil, &token.BootstrapData{ + ServerAddr: serverAddr, + JoinToken: joinToken, + CaCertHashes: caCertHashes, + }); err != nil { + return errors.Wrap(err, "couldn't retrieve bootstrap config info") + } else { + clusterInfo := kubeconfigutil.GetClusterFromKubeConfig(cfg) + tlsBootstrapCfg := kubeconfigutil.CreateWithToken( + fmt.Sprintf("https://%s", serverAddr), + "kubernetes", + "token-bootstrap-client", + clusterInfo.CertificateAuthorityData, + joinToken, + ) + if err = kubeconfigutil.WriteToDisk(constants.YurtHubBootstrapConfig, tlsBootstrapCfg); err != nil { + return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk") + } + } + + return nil +} + +// CheckYurthubHealthz check if YurtHub is healthy. +func CheckYurthubHealthz(yurthubServer string) error { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerHealthzURLPath), nil) + if err != nil { + return err + } + client := &http.Client{} + return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + resp, err := client.Do(req) + if err != nil { + return false, nil + } + ok, err := io.ReadAll(resp.Body) + if err != nil { + return false, nil + } + return string(ok) == "OK", nil + }) +} + +// CheckYurthubReadyz check if YurtHub's certificates are ready or not +func CheckYurthubReadyz(yurthubServer string) error { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerReadyzURLPath), nil) + if err != nil { + return err + } + client := &http.Client{} + return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + resp, err := client.Do(req) + if err != nil { + return false, nil + } + ok, err := io.ReadAll(resp.Body) + if err != nil { + return false, nil + } + return string(ok) == "OK", nil + }) +} + +func CheckYurthubReadyzOnce(yurthubServer string) bool { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", yurthubServer), constants.ServerReadyzURLPath), nil) + if err != nil { + return false + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return false + } + ok, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + return string(ok) == "OK" +} + +func CleanHubBootstrapConfig() error { + if err := os.RemoveAll(constants.YurtHubBootstrapConfig); err != nil { + klog.Warningf("Clean file %s fail: %v, please clean it manually.", constants.YurtHubBootstrapConfig, err) + } + return nil +}