staging for cgroupdriver

Signed-off-by: oldthreefeng <louisehong4168@gmail.com>
labring · Jan 17, 2021 · 5edfd53 · 5edfd53
1 parent 9ec95b9
commit 5edfd53
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 17 deletions.
diff --git a/install/constants.go b/install/constants.go
@@ -27,6 +27,8 @@ const (
 	// CriSocket
 	DefaultDockerCRISocket     = "/var/run/dockershim.sock"
 	DefaultContainerdCRISocket = "/run/containerd/containerd.sock"
+	DefaultCgroupDriver        = "cgroupfs"
+	DefaultSystemdCgroupDriver = "systemd"
 )
 
 const InitTemplateTextV1beta1 = string(`apiVersion: kubeadm.k8s.io/v1beta1
@@ -88,7 +90,11 @@ kind: KubeProxyConfiguration
 mode: "ipvs"
 ipvs:
   excludeCIDRs:
-  - "{{.VIP}}/32"`)
+  - "{{.VIP}}/32"
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: {{ .CgroupDriver}}`)
 
 const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
 caCertPath: /etc/kubernetes/pki/ca.crt
@@ -111,7 +117,11 @@ controlPlane:
     bindPort: 6443
 {{- end}}
 nodeRegistration:
-  criSocket: {{.CriSocket}}`)
+  criSocket: {{.CriSocket}}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: {{ .CgroupDriver}}`)
 
 const InitTemplateTextV1bate2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
@@ -174,4 +184,20 @@ kind: KubeProxyConfiguration
 mode: "ipvs"
 ipvs:
   excludeCIDRs:
-  - "{{.VIP}}/32"`)
+  - "{{.VIP}}/32"
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: {{ .CgroupDriver}}`)
+
+const (
+	ContainerdShell = `if grep "SystemdCgroup = true"  /etc/containerd/config.toml &> /dev/nul; then  
+driver=systemd
+else
+driver=cgroupfs
+fi
+echo ${driver}`
+	DockerShell = `cgroupDriver=$(docker info|grep Cg)
+	driver=${cgroupDriver##*: }
+	echo "${driver}"`
+)
diff --git a/install/generator.go b/install/generator.go
@@ -54,11 +54,11 @@ func Template() []byte {
 }
 
 // JoinTemplate is generate JoinCP nodes configuration by master ip.
-func JoinTemplate(ip string) []byte {
-	return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip)
+func JoinTemplate(ip string, cgroup string) []byte {
+	return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip, cgroup)
 }
 
-func JoinTemplateFromTemplateContent(templateContent, ip string) []byte {
+func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte {
 	tmpl, err := template.New("text").Parse(templateContent)
 	defer func() {
 		if r := recover(); r != nil {
@@ -80,6 +80,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip string) []byte {
 		CriSocket = DefaultDockerCRISocket
 	}
 	envMap["CriSocket"] = CriSocket
+	envMap["CgroupDriver"] = cgroup
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()
@@ -110,6 +111,7 @@ func TemplateFromTemplateContent(templateContent string) []byte {
 	envMap["SvcCIDR"] = SvcCIDR
 	envMap["Repo"] = Repo
 	envMap["Master0"] = IpFormat(MasterIPs[0])
+	envMap["CgroupDriver"] = CgroupDriver
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()

diff --git a/install/generator_test.go b/install/generator_test.go
@@ -115,21 +115,20 @@ func TestJoinTemplate(t *testing.T) {
 	config := sshutil.SSH{
 		User:     "louis",
 		Password: "210010",
-		PkFile: "/home/louis/.ssh/id_rsa",
+		PkFile:   "/home/louis/.ssh/id_rsa",
 	}
 	Version = "v1.20.0"
 	MasterIPs = masters
 	JoinToken = "1y6yyl.ramfafiy99vz3tbw"
 	TokenCaCertHash = "sha256:a68c79c87368ff794ae50c5fd6a8ce13fdb2778764f1080614ddfeaa0e2b9d14"
 
-
 	VIP = vip
-	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0])))+"\" > ~/aa")
-	t.Log(string(JoinTemplate(IpFormat(masters[0]))))
-	
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0]), "systemd"))+"\" > ~/aa")
+	t.Log(string(JoinTemplate(IpFormat(masters[0]), "cgroupfs")))
+
 	Version = "v1.19.0"
-	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(""))+"\" > ~/aa")
-	t.Log(string(JoinTemplate("")))
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate("", "systemd"))+"\" > ~/aa")
+	t.Log(string(JoinTemplate("", "cgroupfs")))
 }
 
 var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2

diff --git a/install/init.go b/install/init.go
@@ -54,9 +54,24 @@ func BuildInit() {
 	i.PrintFinish()
 }
 
+func (s *SealosInstaller) getCgroupDriverFromShell(h string) string {
+	var output string
+	if For120(Version) {
+		cmd := ContainerdShell
+		output = SSHConfig.CmdToString(h, cmd, " ")
+	} else {
+		cmd := DockerShell
+		output = SSHConfig.CmdToString(h, cmd, " ")
+	}
+	output = strings.TrimSpace(output)
+	logger.Info("cgroup driver is %s", output)
+	return output 
+}
+
 //KubeadmConfigInstall is
 func (s *SealosInstaller) KubeadmConfigInstall() {
 	var templateData string
+	CgroupDriver = s.getCgroupDriverFromShell(s.Masters[0])
 	if KubeadmFile == "" {
 		templateData = string(Template())
 	} else {

diff --git a/install/join.go b/install/join.go
@@ -84,13 +84,15 @@ func getApiserverHost(ipAddr string) (host string) {
 }
 
 // sendJoinCPConfig send join CP nodes configuration
-func sendJoinCPConfig(joinMaster []string) {
+func (s *SealosInstaller) sendJoinCPConfig(joinMaster []string) {
 	var wg sync.WaitGroup
 	for _, master := range joinMaster {
 		wg.Add(1)
 		go func(master string) {
 			defer wg.Done()
-			templateData := string(JoinTemplate(IpFormat(master)))
+			var cgroup string
+			cgroup = s.getCgroupDriverFromShell(master)
+			templateData := string(JoinTemplate(IpFormat(master), cgroup))
 			cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(master, cmd)
 		}(master)
@@ -105,7 +107,7 @@ func (s *SealosInstaller) JoinMasters(masters []string) {
 	s.SendJoinMasterKubeConfigs(masters)
 	s.sendNewCertAndKey(masters)
 	// send CP nodes configuration
-	sendJoinCPConfig(masters)
+	s.sendJoinCPConfig(masters)
 
 	//join master do sth
 	cmd := s.Command(Version, JoinMaster)
@@ -140,12 +142,14 @@ func (s *SealosInstaller) JoinNodes() {
 		masters += fmt.Sprintf(" --rs %s:6443", IpFormat(master))
 	}
 	ipvsCmd := fmt.Sprintf("sealos ipvs --vs %s:6443 %s --health-path /healthz --health-schem https --run-once", VIP, masters)
-	templateData := string(JoinTemplate(""))
 	for _, node := range s.Nodes {
 		wg.Add(1)
 		go func(node string) {
 			defer wg.Done()
 			// send join node config
+			var cgroup string
+			cgroup = s.getCgroupDriverFromShell(node)
+			templateData := string(JoinTemplate("", cgroup))
 			cmdJoinConfig := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(node, cmdJoinConfig)
 

diff --git a/install/vars.go b/install/vars.go
@@ -30,6 +30,7 @@ var (
 
 	//criSocket
 	CriSocket string
+	CgroupDriver string
 
 	VIP     string
 	PkgUrl  string