feat: add yurtadm renew certificate command (#1314)

Signed-off-by: Liang Deng <283304489@qq.com>

go.mod

@@ -115,6 +115,7 @@ require (
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
 	go.opentelemetry.io/contrib v0.20.0 // indirect

go.sum

@@ -400,6 +400,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -508,6 +509,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -572,8 +574,9 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
@@ -705,6 +708,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

pkg/node-servant/components/yurthub.go

@@ -27,21 +27,25 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig"
 	"github.com/openyurtio/openyurt/pkg/util/templates"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
 	enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode"
-	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
 	"github.com/openyurtio/openyurt/pkg/yurthub/storage/disk"
 	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 const (
 	hubHealthzCheckFrequency = 10 * time.Second
 	fileMode                 = 0666
+	DefaultRootDir           = "/var/lib"
+	DefaultCaPath            = "/etc/kubernetes/pki/ca.crt"
 )
 
 type yurtHubOperator struct {
@@ -79,7 +83,7 @@ func (op *yurtHubOperator) Install() error {
 		"yurthubServerAddr":    constants.DefaultYurtHubServerAddr,
 		"kubernetesServerAddr": op.apiServerAddr,
 		"image":                op.yurthubImage,
-		"joinToken":            op.joinToken,
+		"bootstrapFile":        constants.YurtHubBootstrapConfig,
 		"workingMode":          string(op.workingMode),
 		"enableDummyIf":        strconv.FormatBool(op.enableDummyIf),
 		"enableNodePool":       strconv.FormatBool(op.enableNodePool),
@@ -88,7 +92,16 @@ func (op *yurtHubOperator) Install() error {
 		return err
 	}
 
-	// 1-2. create yurthub.yaml
+	// 1-2. create /var/lib/yurthub/bootstrap-hub.conf
+	if err := enutil.EnsureDir(constants.YurtHubWorkdir); err != nil {
+		return err
+	}
+	if err := setHubBootstrapConfig(op.apiServerAddr, op.joinToken); err != nil {
+		return err
+	}
+	klog.Infof("create the %s", constants.YurtHubBootstrapConfig)
+
+	// 1-3. create yurthub.yaml
 	podManifestPath := enutil.GetPodManifestPath()
 	if err := enutil.EnsureDir(podManifestPath); err != nil {
 		return err
@@ -151,7 +164,7 @@ func getYurthubYaml(podManifestPath string) string {
 }
 
 func getYurthubConf() string {
-	return filepath.Join(token.DefaultRootDir, projectinfo.GetHubName())
+	return filepath.Join(DefaultRootDir, projectinfo.GetHubName())
 }
 
 func getYurthubCacheDir() string {
@@ -221,3 +234,26 @@ func pingClusterHealthz(client *http.Client, addr string) (bool, error) {
 
 	return true, nil
 }
+
+func setHubBootstrapConfig(serverAddr string, joinToken string) error {
+	caData, err := os.ReadFile(DefaultCaPath)
+	if err != nil {
+		return err
+	}
+	tlsBootstrapCfg := kubeconfigutil.CreateWithToken(
+		serverAddr,
+		"openyurt-e2e-test",
+		"token-bootstrap-client",
+		caData,
+		joinToken,
+	)
+	content, err := clientcmd.Write(*tlsBootstrapCfg)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(constants.YurtHubBootstrapConfig, content, fileMode); err != nil {
+		return errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk")
+	}
+
+	return nil
+}

pkg/yurtadm/cmd/cmd.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/docs"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/renew"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/reset"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/token"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/yurtinit"
@@ -50,6 +51,7 @@ func NewYurtadmCommand() *cobra.Command {
 	cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, os.Stderr))
 	cmds.AddCommand(token.NewCmdToken(os.Stdin, os.Stdout, os.Stderr))
 	cmds.AddCommand(docs.NewDocsCmd(cmds))
+	cmds.AddCommand(renew.NewCmdRenew(os.Stdin, os.Stdout, os.Stderr))
 
 	klog.InitFlags(nil)
 	// goflag.Parse()

pkg/yurtadm/cmd/join/phases/postcheck.go

@@ -17,64 +17,31 @@ limitations under the License.
 package phases
 
 import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
-	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
-	"github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
 )
 
-// RunPostCheck executes the node health check process.
+// RunPostCheck executes the node health check and clean process.
 func RunPostCheck(data joindata.YurtJoinData) error {
 	klog.V(1).Infof("check kubelet status.")
-	if err := checkKubeletStatus(); err != nil {
+	if err := kubernetes.CheckKubeletStatus(); err != nil {
 		return err
 	}
 	klog.V(1).Infof("kubelet service is active")
 
 	klog.V(1).Infof("waiting hub agent ready.")
-	if err := checkYurthubHealthz(data); err != nil {
+	if err := yurthub.CheckYurthubHealthz(data.YurtHubServer()); err != nil {
 		return err
 	}
 	klog.V(1).Infof("hub agent is ready")
 
-	return nil
-}
-
-// checkKubeletStatus check if kubelet is healthy.
-func checkKubeletStatus() error {
-	initSystem, err := initsystem.GetInitSystem()
-	if err != nil {
+	if err := yurthub.CleanHubBootstrapConfig(); err != nil {
 		return err
 	}
-	if ok := initSystem.ServiceIsActive("kubelet"); !ok {
-		return fmt.Errorf("kubelet is not active. ")
-	}
-	return nil
-}
+	klog.V(1).Infof("clean yurthub bootstrap config file success")
 
-// checkYurthubHealthz check if YurtHub is healthy.
-func checkYurthubHealthz(joinData joindata.YurtJoinData) error {
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", fmt.Sprintf("%s:10267", joinData.YurtHubServer()), constants.ServerHealthzURLPath), nil)
-	if err != nil {
-		return err
-	}
-	client := &http.Client{}
-	return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) {
-		resp, err := client.Do(req)
-		if err != nil {
-			return false, nil
-		}
-		ok, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return false, nil
-		}
-		return string(ok) == "OK", nil
-	})
+	return nil
 }

pkg/yurtadm/cmd/join/phases/prepare.go

@@ -17,18 +17,16 @@ limitations under the License.
 package phases
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"k8s.io/klog/v2"
 
-	"github.com/openyurtio/openyurt/pkg/util/templates"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/constants"
 	yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes"
 	"github.com/openyurtio/openyurt/pkg/yurtadm/util/system"
+	"github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub"
 )
 
 // RunPrepare executes the node initialization process.
@@ -66,58 +64,17 @@ func RunPrepare(data joindata.YurtJoinData) error {
 	if err := yurtadmutil.SetKubeletConfigForNode(); err != nil {
 		return err
 	}
-	if err := addYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil {
+	if err := yurthub.SetHubBootstrapConfig(data.ServerAddr(), data.JoinToken(), data.CaCertHashes()); err != nil {
 		return err
 	}
-	if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
+	if err := yurthub.AddYurthubStaticYaml(data, filepath.Join(constants.KubeletConfigureDir, constants.ManifestsSubDirName)); err != nil {
 		return err
 	}
-	if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil {
-		return err
-	}
-	return nil
-}
-
-// addYurthubStaticYaml generate YurtHub static yaml for worker node.
-func addYurthubStaticYaml(data joindata.YurtJoinData, podManifestPath string) error {
-	klog.Info("[join-node] Adding edge hub static yaml")
-	if _, err := os.Stat(podManifestPath); err != nil {
-		if os.IsNotExist(err) {
-			err = os.MkdirAll(podManifestPath, os.ModePerm)
-			if err != nil {
-				return err
-			}
-		} else {
-			klog.Errorf("Describe dir %s fail: %v", podManifestPath, err)
-			return err
-		}
-	}
-
-	// There can be multiple master IP addresses
-	serverAddrs := strings.Split(data.ServerAddr(), ",")
-	for i := 0; i < len(serverAddrs); i++ {
-		serverAddrs[i] = fmt.Sprintf("https://%s", serverAddrs[i])
-	}
-
-	kubernetesServerAddrs := strings.Join(serverAddrs, ",")
-
-	ctx := map[string]string{
-		"kubernetesServerAddr": kubernetesServerAddrs,
-		"image":                data.YurtHubImage(),
-		"joinToken":            data.JoinToken(),
-		"workingMode":          data.NodeRegistration().WorkingMode,
-		"organizations":        data.NodeRegistration().Organizations,
-		"yurthubServerAddr":    data.YurtHubServer(),
-	}
-
-	yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, ctx)
-	if err != nil {
+	if err := yurtadmutil.SetDiscoveryConfig(data); err != nil {
 		return err
 	}
-
-	if err := os.WriteFile(filepath.Join(podManifestPath, constants.YurthubStaticPodFileName), []byte(yurthubTemplate), 0600); err != nil {
+	if err := yurtadmutil.SetKubeadmJoinConfig(data); err != nil {
 		return err
 	}
-	klog.Info("[join-node] Add hub agent static yaml is ok")
 	return nil
 }