diff --git a/README.md b/README.md index 8cc4d03e0a3..9ffa9dd91bc 100755 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ - Ubuntu 16.04, 18.04, 20.04, x86_64/ arm64 - Centos/RHEL 7.6+, x86_64/ arm64 - 其他支持 systemd 的系统环境. x86_64/ arm64 +- Kylin arm64 ## kubernetes 版本 diff --git a/README_en.md b/README_en.md index 4f3f275b0e7..ebd9c98e525 100644 --- a/README_en.md +++ b/README_en.md @@ -18,6 +18,7 @@ Build a production kubernetes HA cluster. - Ubuntu 16.04， 18.04， 20.04 , x86_64/ arm64 - Centos/RHEL 7.6+, x86_64/ arm64 - 99% systemd manage linux system。 x86_64/ arm64 +- Kylin arm64 ## kubernetes Versions diff --git a/README_zh.md b/README_zh.md index 36c39216dd6..9a47bda23e2 100644 --- a/README_zh.md +++ b/README_zh.md @@ -40,6 +40,7 @@ - Ubuntu 16.04， 18.04， 20.04 , x86_64/ arm64 - Centos/RHEL 7.6+, x86_64/ arm64 - 其他支持 systemd 的系统环境. x86_64/ arm64 +- Kylin arm64 ## kubernetes 版本 diff --git a/install/constants.go b/install/constants.go index 64c5b57c011..df1f671df23 100644 --- a/install/constants.go +++ b/install/constants.go @@ -27,6 +27,8 @@ const ( // CriSocket DefaultDockerCRISocket = "/var/run/dockershim.sock" DefaultContainerdCRISocket = "/run/containerd/containerd.sock" + DefaultCgroupDriver = "cgroupfs" + DefaultSystemdCgroupDriver = "systemd" ) const InitTemplateTextV1beta1 = string(`apiVersion: kubeadm.k8s.io/v1beta1 @@ -91,7 +93,9 @@ kind: KubeProxyConfiguration mode: "ipvs" ipvs: excludeCIDRs: - - "{{.VIP}}/32"`) + - "{{.VIP}}/32" +--- +` + kubeletConfigDefault) const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2 caCertPath: /etc/kubernetes/pki/ca.crt @@ -114,7 +118,9 @@ controlPlane: bindPort: 6443 {{- end}} nodeRegistration: - criSocket: {{.CriSocket}}`) + criSocket: {{.CriSocket}} +--- +` + kubeletConfigDefault) const InitTemplateTextV1bate2 = string(`apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration @@ -180,4 +186,87 @@ kind: KubeProxyConfiguration mode: "ipvs" ipvs: excludeCIDRs: - - "{{.VIP}}/32"`) + - "{{.VIP}}/32" +--- +` + kubeletConfigDefault) + +const ( + ContainerdShell = `if grep "SystemdCgroup = true" /etc/containerd/config.toml &> /dev/null; then +driver=systemd +else +driver=cgroupfs +fi +echo ${driver}` + DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}") + echo "${driver}"` + + kubeletConfigDefault = `apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +cgroupDriver: {{ .CgroupDriver}} +cgroupsPerQOS: true +clusterDomain: cluster.local +configMapAndSecretChangeDetectionStrategy: Watch +containerLogMaxFiles: 5 +containerLogMaxSize: 10Mi +contentType: application/vnd.kubernetes.protobuf +cpuCFSQuota: true +cpuCFSQuotaPeriod: 100ms +cpuManagerPolicy: none +cpuManagerReconcilePeriod: 10s +enableControllerAttachDetach: true +enableDebuggingHandlers: true +enforceNodeAllocatable: +- pods +eventBurst: 10 +eventRecordQPS: 5 +evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% +evictionPressureTransitionPeriod: 5m0s +failSwapOn: true +fileCheckFrequency: 20s +hairpinMode: promiscuous-bridge +healthzBindAddress: 127.0.0.1 +healthzPort: 10248 +httpCheckFrequency: 20s +imageGCHighThresholdPercent: 85 +imageGCLowThresholdPercent: 80 +imageMinimumGCAge: 2m0s +iptablesDropBit: 15 +iptablesMasqueradeBit: 14 +kubeAPIBurst: 10 +kubeAPIQPS: 5 +makeIPTablesUtilChains: true +maxOpenFiles: 1000000 +maxPods: 110 +nodeLeaseDurationSeconds: 40 +nodeStatusReportFrequency: 10s +nodeStatusUpdateFrequency: 10s +oomScoreAdj: -999 +podPidsLimit: -1 +port: 10250 +registryBurst: 10 +registryPullQPS: 5 +rotateCertificates: true +runtimeRequestTimeout: 2m0s +serializeImagePulls: true +staticPodPath: /etc/kubernetes/manifests +streamingConnectionIdleTimeout: 4h0m0s +syncFrequency: 1m0s +volumeStatsAggPeriod: 1m0s` +) diff --git a/install/generator.go b/install/generator.go index 627d52a2abc..2e5e4434d66 100644 --- a/install/generator.go +++ b/install/generator.go @@ -54,11 +54,11 @@ func Template() []byte { } // JoinTemplate is generate JoinCP nodes configuration by master ip. -func JoinTemplate(ip string) []byte { - return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip) +func JoinTemplate(ip string, cgroup string) []byte { + return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip, cgroup) } -func JoinTemplateFromTemplateContent(templateContent, ip string) []byte { +func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte { tmpl, err := template.New("text").Parse(templateContent) defer func() { if r := recover(); r != nil { @@ -80,6 +80,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip string) []byte { CriSocket = DefaultDockerCRISocket } envMap["CriSocket"] = CriSocket + envMap["CgroupDriver"] = cgroup var buffer bytes.Buffer _ = tmpl.Execute(&buffer, envMap) return buffer.Bytes() @@ -111,6 +112,7 @@ func TemplateFromTemplateContent(templateContent string) []byte { envMap["Repo"] = Repo envMap["Master0"] = IpFormat(MasterIPs[0]) envMap["Network"] = Network + envMap["CgroupDriver"] = CgroupDriver var buffer bytes.Buffer _ = tmpl.Execute(&buffer, envMap) return buffer.Bytes() diff --git a/install/generator_test.go b/install/generator_test.go index 000de49f1eb..371dfc5420c 100644 --- a/install/generator_test.go +++ b/install/generator_test.go @@ -141,12 +141,12 @@ func TestJoinTemplate(t *testing.T) { TokenCaCertHash = "sha256:a68c79c87368ff794ae50c5fd6a8ce13fdb2778764f1080614ddfeaa0e2b9d14" VIP = vip - config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0])))+"\" > ~/aa") - t.Log(string(JoinTemplate(IpFormat(masters[0])))) + config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0]), "systemd"))+"\" > ~/aa") + t.Log(string(JoinTemplate(IpFormat(masters[0]), "cgroupfs"))) Version = "v1.19.0" - config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(""))+"\" > ~/aa") - t.Log(string(JoinTemplate(""))) + config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate("", "systemd"))+"\" > ~/aa") + t.Log(string(JoinTemplate("", "cgroupfs"))) } var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2 diff --git a/install/init.go b/install/init.go index 5cd89c7439d..8a54d631805 100644 --- a/install/init.go +++ b/install/init.go @@ -56,9 +56,24 @@ func BuildInit() { i.PrintFinish() } +func (s *SealosInstaller) getCgroupDriverFromShell(h string) string { + var output string + if For120(Version) { + cmd := ContainerdShell + output = SSHConfig.CmdToString(h, cmd, " ") + } else { + cmd := DockerShell + output = SSHConfig.CmdToString(h, cmd, " ") + } + output = strings.TrimSpace(output) + logger.Info("cgroup driver is %s", output) + return output +} + //KubeadmConfigInstall is func (s *SealosInstaller) KubeadmConfigInstall() { var templateData string + CgroupDriver = s.getCgroupDriverFromShell(s.Masters[0]) if KubeadmFile == "" { templateData = string(Template()) } else { diff --git a/install/join.go b/install/join.go index 5e8731c1506..87a773956c8 100644 --- a/install/join.go +++ b/install/join.go @@ -86,13 +86,15 @@ func getApiserverHost(ipAddr string) (host string) { } // sendJoinCPConfig send join CP nodes configuration -func sendJoinCPConfig(joinMaster []string) { +func (s *SealosInstaller) sendJoinCPConfig(joinMaster []string) { var wg sync.WaitGroup for _, master := range joinMaster { wg.Add(1) go func(master string) { defer wg.Done() - templateData := string(JoinTemplate(IpFormat(master))) + var cgroup string + cgroup = s.getCgroupDriverFromShell(master) + templateData := string(JoinTemplate(IpFormat(master), cgroup)) cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData) _ = SSHConfig.CmdAsync(master, cmd) }(master) @@ -107,7 +109,7 @@ func (s *SealosInstaller) JoinMasters(masters []string) { s.SendJoinMasterKubeConfigs(masters) s.sendNewCertAndKey(masters) // send CP nodes configuration - sendJoinCPConfig(masters) + s.sendJoinCPConfig(masters) //join master do sth cmd := s.Command(Version, JoinMaster) @@ -142,12 +144,14 @@ func (s *SealosInstaller) JoinNodes() { masters += fmt.Sprintf(" --rs %s:6443", IpFormat(master)) } ipvsCmd := fmt.Sprintf("sealos ipvs --vs %s:6443 %s --health-path /healthz --health-schem https --run-once", VIP, masters) - templateData := string(JoinTemplate("")) for _, node := range s.Nodes { wg.Add(1) go func(node string) { defer wg.Done() // send join node config + var cgroup string + cgroup = s.getCgroupDriverFromShell(node) + templateData := string(JoinTemplate("", cgroup)) cmdJoinConfig := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData) _ = SSHConfig.CmdAsync(node, cmdJoinConfig) diff --git a/install/vars.go b/install/vars.go index 2ac34d101f1..e68cf9a2d89 100644 --- a/install/vars.go +++ b/install/vars.go @@ -30,6 +30,7 @@ var ( //criSocket CriSocket string + CgroupDriver string VIP string PkgUrl string