fix type error

add requirements and recommendations
fix #596. add default kubelet.conf

Signed-off-by: oldthreefeng <louisehong4168@gmail.com>

README.md

@@ -36,7 +36,8 @@
 
 ## Linux 发行版, CPU架构
 
-- Ubuntu 16.04， 18.04， 20.04 ,  x86_64/ arm64
+- Debian 9+,  x86_64/ arm64
+- Ubuntu 16.04, 18.04, 20.04,  x86_64/ arm64
 - Centos/RHEL 7.6+,  x86_64/ arm64
 - 其他支持 systemd 的系统环境.  x86_64/ arm64
 - Kylin arm64
@@ -69,6 +70,8 @@
    - 确保 /etc/resolv.conf 中的 DNS 地址可用。否则，可能会导致群集中coredns异常。 
    - 如果使用阿里云/华为云主机部署。 默认的pod网段会和阿里云的dns网段冲突， 建议自定义修改pod网段, 在init的时候指定`--podcidr` 来修改。
    - sealos 默认会关闭防火墙， 如果需要打开防火墙， 建议手动放行相关的端口。
+ - 内核要求:
+   - cni组件选择cilium时要求内核版本不低于5.4
 
 # 🚀 快速开始
 

cmd/init.go

@@ -110,7 +110,7 @@ func init() {
 	// Here you will define your flags and configuration settings.
 	initCmd.Flags().StringVar(&install.SSHConfig.User, "user", "root", "servers user name for ssh")
 	initCmd.Flags().StringVar(&install.SSHConfig.Password, "passwd", "", "password for ssh")
-	initCmd.Flags().StringVar(&install.SSHConfig.PkFile, "pk", cert.GetUserHomeDir() + "/.ssh/id_rsa", "private key for ssh")
+	initCmd.Flags().StringVar(&install.SSHConfig.PkFile, "pk", cert.GetUserHomeDir()+"/.ssh/id_rsa", "private key for ssh")
 	initCmd.Flags().StringVar(&install.SSHConfig.PkPassword, "pk-passwd", "", "private key password for ssh")
 
 	initCmd.Flags().StringVar(&install.KubeadmFile, "kubeadm-config", "", "kubeadm-config.yaml template file")
@@ -132,7 +132,6 @@ func init() {
 	initCmd.Flags().StringVar(&install.Network, "network", "calico", "cni plugin, calico..")
 	initCmd.Flags().BoolVar(&install.IPIP, "ipip", true, "ipip mode enable, calico..")
 	initCmd.Flags().StringVar(&install.MTU, "mtu", "1440", "mtu of the ipip mode , calico..")
-
 	initCmd.Flags().StringVar(&install.LvscareImage.Image, "lvscare-image", "fanux/lvscare", "lvscare image name")
 	initCmd.Flags().StringVar(&install.LvscareImage.Tag, "lvscare-tag", "latest", "lvscare image tag name")
 

cmd/ipvs.go

@@ -37,7 +37,7 @@ func init() {
 
 	// Here you will define your flags and configuration settings.
 	ipvsCmd.Flags().BoolVar(&install.Ipvs.RunOnce, "run-once", false, "is run once mode")
-	ipvsCmd.Flags().BoolVarP(&install.Ipvs.Clean, "clean","c", true, " clean Vip ipvs rule before join node, if Vip has no ipvs rule do nothing.")
+	ipvsCmd.Flags().BoolVarP(&install.Ipvs.Clean, "clean", "c", true, " clean Vip ipvs rule before join node, if Vip has no ipvs rule do nothing.")
 	ipvsCmd.Flags().StringVar(&install.Ipvs.VirtualServer, "vs", "", "virturl server like 10.54.0.2:6443")
 	ipvsCmd.Flags().StringSliceVar(&install.Ipvs.RealServer, "rs", []string{}, "virturl server like 192.168.0.2:6443")
 

cmd/root.go

@@ -67,7 +67,7 @@ func initConfig() {
 	home := cert.GetUserHomeDir()
 	logFile := fmt.Sprintf("%s/.sealos/sealos.log", home)
 	if !install.FileExist(home + "/.sealos") {
-		err := os.MkdirAll(home + "/.sealos",os.ModePerm)
+		err := os.MkdirAll(home+"/.sealos", os.ModePerm)
 		if err != nil {
 			fmt.Println("create default sealos config dir failed, please create it by your self mkdir -p /root/.sealos && touch /root/.sealos/config.yaml")
 		}

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/fanux/lvscare v0.0.0-00010101000000-000000000000
 	github.com/fanux/sealgate v0.0.5
-	github.com/ghodss/yaml v1.0.0
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/linuxsuren/cobra-extension v0.0.8
 	github.com/pkg/errors v0.9.1
@@ -28,6 +27,7 @@ require (
 	k8s.io/api v0.18.0
 	k8s.io/apimachinery v0.18.0
 	k8s.io/client-go v0.18.0
+	sigs.k8s.io/yaml v1.2.0
 )
 
 replace (

install/check.go

@@ -49,5 +49,15 @@ func (s *SealosInstaller) CheckValid() {
 			}
 			logger.Info("[%s]  ------------ check ok", h)
 		}
+		if s.Network == "cilium" {
+			if err := SSHConfig.CmdAsync(h, "uname -r | grep 5 | awk -F. '{if($2>3)print \"ok\"}' | grep ok && exit 0 || exit 1"); err != nil {
+				logger.Error("[%s] ------------ check kernel version  < 5.3", h)
+				os.Exit(1)
+			}
+			if err := SSHConfig.CmdAsync(h, "mount bpffs -t bpf /sys/fs/bpf && mount | grep /sys/fs/bpf && exit 0 || exit 1"); err != nil {
+				logger.Error("[%s] ------------ mount  bpffs err", h)
+				os.Exit(1)
+			}
+		}
 	}
 }

install/config.go

@@ -28,13 +28,13 @@ type SealConfig struct {
 	PkPassword string
 	//ApiServer ex. apiserver.cluster.local
 	ApiServerDomain string
-
-	VIP     string
-	PkgURL  string
-	Version string
-	Repo    string
-	PodCIDR string
-	SvcCIDR string
+	Network         string
+	VIP             string
+	PkgURL          string
+	Version         string
+	Repo            string
+	PodCIDR         string
+	SvcCIDR         string
 	//certs location
 	CertPath     string
 	CertEtcdPath string
@@ -59,6 +59,7 @@ func (c *SealConfig) Dump(path string) {
 	c.PrivateKey = SSHConfig.PkFile
 	c.PkPassword = SSHConfig.PkPassword
 	c.ApiServerDomain = ApiServer
+	c.Network = Network
 	c.VIP = VIP
 	c.PkgURL = PkgUrl
 	c.Version = Version
@@ -135,13 +136,13 @@ func (c *SealConfig) Load(path string) (err error) {
 	SSHConfig.PkFile = c.PrivateKey
 	SSHConfig.PkPassword = c.PkPassword
 	ApiServer = c.ApiServerDomain
+	Network = c.Network
 	VIP = c.VIP
 	PkgUrl = c.PkgURL
 	Version = c.Version
 	Repo = c.Repo
 	PodCIDR = c.PodCIDR
 	SvcCIDR = c.SvcCIDR
-
 	DnsDomain = c.DnsDomain
 	ApiServerCertSANs = c.ApiServerCertSANs
 	CertPath = c.CertPath
@@ -184,6 +185,7 @@ func (c *SealConfig) ShowDefaultConfig() {
 	c.Passwd = "123456"
 	c.PrivateKey = home + "/.ssh/id_rsa"
 	c.ApiServerDomain = "apiserver.cluster.local"
+	c.Network = "calico"
 	c.VIP = "10.103.97.2"
 	c.PkgURL = home + "/kube1.17.13.tar.gz"
 	c.Version = "v1.17.13"

install/constants.go

@@ -69,6 +69,9 @@ controllerManager:
   extraArgs:
     feature-gates: TTLAfterFinished=true
     experimental-cluster-signing-duration: 876000h
+{{- if eq .Network "cilium" }}
+    allocate-node-cidrs: \"true\"
+{{- end }}
   extraVolumes:
   - hostPath: /etc/localtime
     mountPath: /etc/localtime
@@ -159,6 +162,9 @@ controllerManager:
   extraArgs:
     feature-gates: TTLAfterFinished=true
     experimental-cluster-signing-duration: 876000h
+{{- if eq .Network "cilium" }}
+    allocate-node-cidrs: \"true\"
+{{- end }}
   extraVolumes:
   - hostPath: /etc/localtime
     mountPath: /etc/localtime

install/etcd_save_test.go

@@ -15,8 +15,8 @@ func Test_reFormatHostToIp(t *testing.T) {
 		args args
 		want string
 	}{
-		{"test",args{"192.168.0.22:22"},"192.168.0.22"},
-		{"test02",args{"192.168.0.22"},"192.168.0.22"},
+		{"test", args{"192.168.0.22:22"}, "192.168.0.22"},
+		{"test02", args{"192.168.0.22"}, "192.168.0.22"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -26,7 +26,6 @@ func Test_reFormatHostToIp(t *testing.T) {
 		})
 	}
 
-
 	u := fmt.Sprintf("%v", time.Now().Unix())
 	fmt.Println(u)
-}
+}

install/generator.go

@@ -3,8 +3,8 @@ package install
 import (
 	"bytes"
 	"fmt"
-	"github.com/ghodss/yaml"
 	"github.com/wonderivan/logger"
+	"sigs.k8s.io/yaml"
 	"strings"
 	"text/template"
 )
@@ -112,6 +112,7 @@ func TemplateFromTemplateContent(templateContent string) []byte {
 	envMap["Repo"] = Repo
 	envMap["Master0"] = IpFormat(MasterIPs[0])
 	envMap["CgroupDriver"] = CgroupDriver
+	envMap["Network"] = Network
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()

install/generator_test.go

@@ -24,6 +24,24 @@ func TestTemplate(t *testing.T) {
 	t.Log(string(Template()))
 }
 
+func TestNetCiliumTemplate(t *testing.T) {
+	var masters = []string{"172.20.241.205:22", "172.20.241.206:22", "172.20.241.207:22"}
+	var vip = "10.103.97.1"
+	MasterIPs = masters
+	VIP = vip
+	ApiServer = "apiserver.cluster.local"
+	Version = "1.20.5"
+	Network = "cilium"
+	t.Log(string(Template()))
+	Network = "calico"
+	t.Log(string(Template()))
+	Version = "1.18.5"
+	Network = "cilium"
+	t.Log(string(Template()))
+	Network = "calico"
+	t.Log(string(Template()))
+}
+
 var testYaml = `apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
 kubernetesVersion: v1.18.0
@@ -123,8 +141,8 @@ func TestJoinTemplate(t *testing.T) {
 	TokenCaCertHash = "sha256:a68c79c87368ff794ae50c5fd6a8ce13fdb2778764f1080614ddfeaa0e2b9d14"
 
 	VIP = vip
-	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0]), "systemd"))+"\" > ~/aa")
-	t.Log(string(JoinTemplate(IpFormat(masters[0]), "cgroupfs")))
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0]), ""))+"\" > ~/aa")
+	t.Log(string(JoinTemplate(IpFormat(masters[0]), "")))
 
 	Version = "v1.19.0"
 	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate("", "systemd"))+"\" > ~/aa")
@@ -145,4 +163,4 @@ controlPlane:
   localAPIEndpoint:
     advertiseAddress: {{.Master}}
     bindPort: 6443
-`
+`

install/init.go

@@ -25,9 +25,11 @@ func BuildInit() {
 	nodes := NodeIPs
 	hosts := append(masters, nodes...)
 	i := &SealosInstaller{
-		Hosts:   hosts,
-		Masters: masters,
-		Nodes:   nodes,
+		Hosts:     hosts,
+		Masters:   masters,
+		Nodes:     nodes,
+		Network:   Network,
+		ApiServer: ApiServer,
 	}
 	i.CheckValid()
 	i.Print()
@@ -197,18 +199,20 @@ func (s *SealosInstaller) InstallMaster0() {
 	}
 	//cmd = `kubectl apply -f /root/kube/conf/net/calico.yaml || true`
 
-	// can-reach is used by calico multi network , flannel has nothing to add. just Use it. 
+	// can-reach is used by calico multi network , flannel has nothing to add. just Use it.
 	if k8s.IsIpv4(Interface) && Network == "calico" {
 		Interface = "can-reach=" + Interface
 	} else {
 		Interface = "interface=" + Interface
 	}
 
 	netyaml := net.NewNetwork(Network, net.MetaData{
-		Interface: Interface,
-		CIDR:      PodCIDR,
-		IPIP:      IPIP,
-		MTU:       MTU,
+		Interface:      Interface,
+		CIDR:           PodCIDR,
+		IPIP:           IPIP,
+		MTU:            MTU,
+		CniRepo:        Repo,
+		K8sServiceHost: s.ApiServer,
 	}).Manifests("")
 
 	cmd = fmt.Sprintf(`echo '%s' | kubectl apply -f -`, netyaml)

install/join.go

@@ -24,9 +24,11 @@ func joinMastersFunc(joinMasters []string) {
 	masters := MasterIPs
 	nodes := NodeIPs
 	i := &SealosInstaller{
-		Hosts:   joinMasters,
-		Masters: masters,
-		Nodes:   nodes,
+		Hosts:     joinMasters,
+		Masters:   masters,
+		Nodes:     nodes,
+		Network:   Network,
+		ApiServer: ApiServer,
 	}
 	i.CheckValid()
 	i.SendSealos()
@@ -164,7 +166,6 @@ func (s *SealosInstaller) JoinNodes() {
 				addRouteCmd := fmt.Sprintf("sealos route add --host %s --gateway %s", VIP, IpFormat(node))
 				SSHConfig.CmdToString(node, addRouteCmd, "")
 			}
-
 
 			_ = SSHConfig.CmdAsync(node, ipvsCmd) // create ipvs rules before we join node
 			cmd := s.Command(Version, JoinNode)

install/sealos.go

@@ -40,9 +40,11 @@ var (
 
 //SealosInstaller is
 type SealosInstaller struct {
-	Hosts   []string
-	Masters []string
-	Nodes   []string
+	Hosts     []string
+	Masters   []string
+	Nodes     []string
+	Network   string
+	ApiServer string
 }
 
 type CommandType string
@@ -54,8 +56,8 @@ const JoinNode CommandType = "joinNode"
 
 func (s *SealosInstaller) Command(version string, name CommandType) (cmd string) {
 	cmds := make(map[CommandType]string)
-	// Please convert your v1beta1 configuration files to v1beta2 using the 
-	// "kubeadm config migrate" command of kubeadm v1.15.x, 因此1.14 版本不支持双网卡. 
+	// Please convert your v1beta1 configuration files to v1beta2 using the
+	// "kubeadm config migrate" command of kubeadm v1.15.x, 因此1.14 版本不支持双网卡.
 	cmds = map[CommandType]string{
 		InitMaster: `kubeadm init --config=/root/kubeadm-config.yaml --experimental-upload-certs` + vlogToStr(),
 		JoinMaster: fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s --experimental-control-plane --certificate-key %s"+vlogToStr(), IpFormat(s.Masters[0]), JoinToken, TokenCaCertHash, CertificateKey),
@@ -65,8 +67,21 @@ func (s *SealosInstaller) Command(version string, name CommandType) (cmd string)
 	//todo
 	if VersionToInt(version) >= 115 {
 		cmds[InitMaster] = `kubeadm init --config=/root/kubeadm-config.yaml --upload-certs` + vlogToStr()
-		cmds[JoinMaster] = "kubeadm join --config=/root/kubeadm-join-config.yaml "+vlogToStr()
-		cmds[JoinNode] =   "kubeadm join --config=/root/kubeadm-join-config.yaml "+vlogToStr()
+		cmds[JoinMaster] = "kubeadm join --config=/root/kubeadm-join-config.yaml " + vlogToStr()
+		cmds[JoinNode] = "kubeadm join --config=/root/kubeadm-join-config.yaml " + vlogToStr()
+	}
+
+	// version >= 1.16.x support kubeadm init --skip-phases=addon/kube-proxy
+	// version <= 115
+	// kubectl -n kube-system delete ds kube-proxy
+	// # Run on each node:
+	// iptables-restore <(iptables-save | grep -v KUBE)
+	if s.Network == "cilium" {
+		if VersionToInt(version) >= 116 {
+			cmds[InitMaster] = `kubeadm init --skip-phases=addon/kube-proxy --config=/root/kubeadm-config.yaml --upload-certs` + vlogToStr()
+		} else {
+			cmds[InitMaster] = `kubeadm init --config=/root/kubeadm-config.yaml --upload-certs` + vlogToStr()
+		}
 	}
 
 	v, ok := cmds[name]

install/upgrade.go

@@ -2,7 +2,7 @@ package install
 
 import (
 	"fmt"
-	"os"	
+	"os"
 	"time"
 
 	"k8s.io/client-go/kubernetes"
@@ -143,7 +143,7 @@ func (u *SealosUpgrade) upgradeNodes(hostnames []string, isMaster bool) {
 			time.Sleep(time.Second * 10)
 			k8sNode, _ := k8s.GetNodeByName(u.Client, node)
 			if k8s.IsNodeReady(*k8sNode) {
-				logger.Info("[%s] fourth:  %s nodes is ready", ip,node)
+				logger.Info("[%s] fourth:  %s nodes is ready", ip, node)
 
 				// fifth to uncordon node
 				err = k8s.CordonUnCordon(u.Client, node, false)