staging for cgroupdriver , fix #582

README.md

@@ -40,6 +40,7 @@
 - Ubuntu 16.04, 18.04, 20.04,  x86_64/ arm64
 - Centos/RHEL 7.6+,  x86_64/ arm64
 - 其他支持 systemd 的系统环境.  x86_64/ arm64
+- Kylin arm64
 
 ## kubernetes 版本
 

README_en.md

@@ -18,6 +18,7 @@ Build a production kubernetes HA cluster.
 - Ubuntu 16.04， 18.04， 20.04 ,  x86_64/ arm64
 - Centos/RHEL 7.6+,  x86_64/ arm64
 - 99% systemd manage linux system。 x86_64/ arm64
+- Kylin arm64
 
 ## kubernetes Versions
 

README_zh.md

@@ -40,6 +40,7 @@
 - Ubuntu 16.04， 18.04， 20.04 ,  x86_64/ arm64
 - Centos/RHEL 7.6+,  x86_64/ arm64
 - 其他支持 systemd 的系统环境.  x86_64/ arm64
+- Kylin arm64
 
 ## kubernetes 版本
 

install/constants.go

@@ -27,6 +27,8 @@ const (
 	// CriSocket
 	DefaultDockerCRISocket     = "/var/run/dockershim.sock"
 	DefaultContainerdCRISocket = "/run/containerd/containerd.sock"
+	DefaultCgroupDriver        = "cgroupfs"
+	DefaultSystemdCgroupDriver = "systemd"
 )
 
 const InitTemplateTextV1beta1 = string(`apiVersion: kubeadm.k8s.io/v1beta1
@@ -91,7 +93,9 @@ kind: KubeProxyConfiguration
 mode: "ipvs"
 ipvs:
   excludeCIDRs:
-  - "{{.VIP}}/32"`)
+  - "{{.VIP}}/32"
+---
+` + kubeletConfigDefault)
 
 const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
 caCertPath: /etc/kubernetes/pki/ca.crt
@@ -114,7 +118,9 @@ controlPlane:
     bindPort: 6443
 {{- end}}
 nodeRegistration:
-  criSocket: {{.CriSocket}}`)
+  criSocket: {{.CriSocket}}
+---
+` + kubeletConfigDefault)
 
 const InitTemplateTextV1bate2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
@@ -180,4 +186,87 @@ kind: KubeProxyConfiguration
 mode: "ipvs"
 ipvs:
   excludeCIDRs:
-  - "{{.VIP}}/32"`)
+  - "{{.VIP}}/32"
+---
+` + kubeletConfigDefault)
+
+const (
+	ContainerdShell = `if grep "SystemdCgroup = true"  /etc/containerd/config.toml &> /dev/null; then  
+driver=systemd
+else
+driver=cgroupfs
+fi
+echo ${driver}`
+	DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}")
+	echo "${driver}"`
+
+  kubeletConfigDefault = `apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    cacheTTL: 2m0s
+    enabled: true
+  x509:
+    clientCAFile: /etc/kubernetes/pki/ca.crt
+authorization:
+  mode: Webhook
+  webhook:
+    cacheAuthorizedTTL: 5m0s
+    cacheUnauthorizedTTL: 30s
+cgroupDriver: {{ .CgroupDriver}}
+cgroupsPerQOS: true
+clusterDomain: cluster.local
+configMapAndSecretChangeDetectionStrategy: Watch
+containerLogMaxFiles: 5
+containerLogMaxSize: 10Mi
+contentType: application/vnd.kubernetes.protobuf
+cpuCFSQuota: true
+cpuCFSQuotaPeriod: 100ms
+cpuManagerPolicy: none
+cpuManagerReconcilePeriod: 10s
+enableControllerAttachDetach: true
+enableDebuggingHandlers: true
+enforceNodeAllocatable:
+- pods
+eventBurst: 10
+eventRecordQPS: 5
+evictionHard:
+  imagefs.available: 15%
+  memory.available: 100Mi
+  nodefs.available: 10%
+  nodefs.inodesFree: 5%
+evictionPressureTransitionPeriod: 5m0s
+failSwapOn: true
+fileCheckFrequency: 20s
+hairpinMode: promiscuous-bridge
+healthzBindAddress: 127.0.0.1
+healthzPort: 10248
+httpCheckFrequency: 20s
+imageGCHighThresholdPercent: 85
+imageGCLowThresholdPercent: 80
+imageMinimumGCAge: 2m0s
+iptablesDropBit: 15
+iptablesMasqueradeBit: 14
+kubeAPIBurst: 10
+kubeAPIQPS: 5
+makeIPTablesUtilChains: true
+maxOpenFiles: 1000000
+maxPods: 110
+nodeLeaseDurationSeconds: 40
+nodeStatusReportFrequency: 10s
+nodeStatusUpdateFrequency: 10s
+oomScoreAdj: -999
+podPidsLimit: -1
+port: 10250
+registryBurst: 10
+registryPullQPS: 5
+rotateCertificates: true
+runtimeRequestTimeout: 2m0s
+serializeImagePulls: true
+staticPodPath: /etc/kubernetes/manifests
+streamingConnectionIdleTimeout: 4h0m0s
+syncFrequency: 1m0s
+volumeStatsAggPeriod: 1m0s`
+)

install/generator.go

@@ -54,11 +54,11 @@ func Template() []byte {
 }
 
 // JoinTemplate is generate JoinCP nodes configuration by master ip.
-func JoinTemplate(ip string) []byte {
-	return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip)
+func JoinTemplate(ip string, cgroup string) []byte {
+	return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip, cgroup)
 }
 
-func JoinTemplateFromTemplateContent(templateContent, ip string) []byte {
+func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte {
 	tmpl, err := template.New("text").Parse(templateContent)
 	defer func() {
 		if r := recover(); r != nil {
@@ -80,6 +80,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip string) []byte {
 		CriSocket = DefaultDockerCRISocket
 	}
 	envMap["CriSocket"] = CriSocket
+	envMap["CgroupDriver"] = cgroup
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()
@@ -111,6 +112,7 @@ func TemplateFromTemplateContent(templateContent string) []byte {
 	envMap["Repo"] = Repo
 	envMap["Master0"] = IpFormat(MasterIPs[0])
 	envMap["Network"] = Network
+	envMap["CgroupDriver"] = CgroupDriver
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()

install/generator_test.go

@@ -141,12 +141,12 @@ func TestJoinTemplate(t *testing.T) {
 	TokenCaCertHash = "sha256:a68c79c87368ff794ae50c5fd6a8ce13fdb2778764f1080614ddfeaa0e2b9d14"
 
 	VIP = vip
-	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0])))+"\" > ~/aa")
-	t.Log(string(JoinTemplate(IpFormat(masters[0]))))
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0]), "systemd"))+"\" > ~/aa")
+	t.Log(string(JoinTemplate(IpFormat(masters[0]), "cgroupfs")))
 
 	Version = "v1.19.0"
-	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(""))+"\" > ~/aa")
-	t.Log(string(JoinTemplate("")))
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate("", "systemd"))+"\" > ~/aa")
+	t.Log(string(JoinTemplate("", "cgroupfs")))
 }
 
 var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2

install/init.go

@@ -56,9 +56,24 @@ func BuildInit() {
 	i.PrintFinish()
 }
 
+func (s *SealosInstaller) getCgroupDriverFromShell(h string) string {
+	var output string
+	if For120(Version) {
+		cmd := ContainerdShell
+		output = SSHConfig.CmdToString(h, cmd, " ")
+	} else {
+		cmd := DockerShell
+		output = SSHConfig.CmdToString(h, cmd, " ")
+	}
+	output = strings.TrimSpace(output)
+	logger.Info("cgroup driver is %s", output)
+	return output 
+}
+
 //KubeadmConfigInstall is
 func (s *SealosInstaller) KubeadmConfigInstall() {
 	var templateData string
+	CgroupDriver = s.getCgroupDriverFromShell(s.Masters[0])
 	if KubeadmFile == "" {
 		templateData = string(Template())
 	} else {

install/join.go

@@ -86,13 +86,15 @@ func getApiserverHost(ipAddr string) (host string) {
 }
 
 // sendJoinCPConfig send join CP nodes configuration
-func sendJoinCPConfig(joinMaster []string) {
+func (s *SealosInstaller) sendJoinCPConfig(joinMaster []string) {
 	var wg sync.WaitGroup
 	for _, master := range joinMaster {
 		wg.Add(1)
 		go func(master string) {
 			defer wg.Done()
-			templateData := string(JoinTemplate(IpFormat(master)))
+			var cgroup string
+			cgroup = s.getCgroupDriverFromShell(master)
+			templateData := string(JoinTemplate(IpFormat(master), cgroup))
 			cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(master, cmd)
 		}(master)
@@ -107,7 +109,7 @@ func (s *SealosInstaller) JoinMasters(masters []string) {
 	s.SendJoinMasterKubeConfigs(masters)
 	s.sendNewCertAndKey(masters)
 	// send CP nodes configuration
-	sendJoinCPConfig(masters)
+	s.sendJoinCPConfig(masters)
 
 	//join master do sth
 	cmd := s.Command(Version, JoinMaster)
@@ -142,12 +144,14 @@ func (s *SealosInstaller) JoinNodes() {
 		masters += fmt.Sprintf(" --rs %s:6443", IpFormat(master))
 	}
 	ipvsCmd := fmt.Sprintf("sealos ipvs --vs %s:6443 %s --health-path /healthz --health-schem https --run-once", VIP, masters)
-	templateData := string(JoinTemplate(""))
 	for _, node := range s.Nodes {
 		wg.Add(1)
 		go func(node string) {
 			defer wg.Done()
 			// send join node config
+			var cgroup string
+			cgroup = s.getCgroupDriverFromShell(node)
+			templateData := string(JoinTemplate("", cgroup))
 			cmdJoinConfig := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(node, cmdJoinConfig)
 

install/vars.go

@@ -30,6 +30,7 @@ var (
 
 	//criSocket
 	CriSocket string
+	CgroupDriver string
 
 	VIP     string
 	PkgUrl  string