Fix some bugs and add fabric (#566)

* add

* fix2

* fix centos version

* add some colors and fix

* fix resgistry error

* fix rm bugs

* fix

* fix by comments

* add fabric

* add fabric

* fix

* add pxe option

* add gpu

* add reboot

* add mac_host

* fix

* fix

* fix mac

* add k8s install

* fix

* add gpu

* add log.sh

* fix logs

* add log ok

* set host ok

* rm not need

* set startpxe

* modify etcd path ok

* change docker path ok

* etcd path ok

* rm not need

* add reame

* fix etcd path

* fix kubelet start bug

* fix by comments

* fix readme

bsroot.sh

@@ -22,14 +22,15 @@
 # bootstrap server when the bootstrapper server is "offline". Or, you can run
 # bsroot.sh directly on the bootstrap server.
 
+alias cp='cp'
+
 SEXTANT_ROOT=${PWD}
+source $SEXTANT_ROOT/scripts/log.sh
 source $SEXTANT_ROOT/scripts/common.sh
 
 check_prerequisites
 check_cluster_desc_file
 
-
-
 echo "Install OS: ${cluster_desc_os_name}"
 if [[ $cluster_desc_os_name == "CentOS" ]]; then
     source $SEXTANT_DIR/scripts/centos.sh
@@ -63,3 +64,4 @@ build_bootstrapper_image
 generate_tls_assets
 prepare_setup_kubectl
 generate_addons_config
+log info "bsroot done!"

docker/Dockerfile

@@ -1,5 +1,4 @@
-FROM alpine
-#FROM golang:alpine
+FROM distribution/registry
 
 # Install required software packages.
 RUN set -ex && \
@@ -9,7 +8,6 @@ apk add dnsmasq openssl
 # Upload Sextant Go programs and retrieve dependencies.
 RUN mkdir -p /go/bin
 COPY cloud-config-server /go/bin
-COPY registry            /go/bin
 
 # NOTICE: change install.sh HTTP server ip:port when running entrypoint.sh
 COPY entrypoint.sh /

docker/entrypoint.sh

@@ -1,9 +1,16 @@
 #!/bin/sh
 
-# start dnsmasq
-mkdir -p /bsroot/dnsmasq
-dnsmasq --log-facility=-  --conf-file=/bsroot/config/dnsmasq.conf \
-  --dhcp-leasefile=/bsroot/dnsmasq/dnsmasq.leases
+if [[ $# != 1 ]]; then
+    echo "need to set start_pxe"
+    exit 1
+fi
+
+if [[ $1 == " y" ]]; then
+    # start dnsmasq
+    mkdir -p /bsroot/dnsmasq
+    dnsmasq --log-facility=-  --conf-file=/bsroot/config/dnsmasq.conf \
+      --dhcp-leasefile=/bsroot/dnsmasq/dnsmasq.leases
+fi
 
 # start cloud-config-server
 /go/bin/cloud-config-server -addr ":80" \

fabric/README_CN.md

@@ -0,0 +1,50 @@
+# 前言
+Sextant设计之初考虑的是在裸机集群中一键式的解决方案。实际使用的过程中，企业内部的集群一般都有了自己的初始化安装环境，如部署了DHCP服务器，有自己的DNS，机器也有自己的hostname，机器之间通过hostname相互也能ping同。这种情况下，同时两个DHCP服务器无疑是有冲突的，需要对Sextant做一些改动以便适应这种的环境。
+
+我们可以把Sextant PXE服务部分设置为可选项，保留资源cache服务部分。由于post_script不能通过kick start的方式启动，所以引入fabric作为集群管理者，方便安装、配置、检查、启动、关闭软件。我们写的如下的步骤，都是在考虑了企业一般的现实情况来做的。
+
+首先，`copy host.template.yaml host.yaml`，然后修改之。
+
+***注意：***
+- 符合要求的步骤可以略过
+- 需要已经安装centos7的基础操作系统
+
+# 步骤一：机器之间可以访问
+我们需要机器都可以通过hostname来相互之间访问。如果企业的网络不支持，需要我们把静态解析写入各个节点`/etc/hosts`中(已经支持的可以忽略)。
+
+```
+# get mac_ip_host
+fab -f get_mac_ip_host.py get_mac_addr
+
+# display all before set them
+fab -f set_hosts.py display
+
+# set hosts
+fab -f set_hosts.py set_mac_hosts
+```
+
+# 步骤二：生成bsroot
+注意设置cluster-desc.yaml中的`start_pxe: n`。
+
+
+# 步骤三：升级kernel
+```
+fab -f upgrade_kernel.py prepare
+fab -f upgrade_kernel.py upgrade
+fab -f upgrade_kernel.py reboot
+```
+
+# 步骤四：安装gpu driver
+```
+fab -f gpu_driver.py prepare
+fab -f gpu_driver.py install
+fab -f gpu_driver.py check
+```
+
+# 步骤五：安装k8s需要的软件
+```
+fab -f k8s.py prepare
+fab -f k8s.py install
+```
+
+# TODO: 启动etcd flannel kubelet等

fabric/get_mac_ip_host.py

@@ -0,0 +1,33 @@
+from __future__ import with_statement
+from fabric.api import *
+from fabric.contrib.console import confirm
+import fabric.operations as op
+import yaml
+import sys
+import re
+
+def get_mac_addr():
+    src_path = "/etc/mac_ip_host"
+
+    cmd = """ default_iface=$(awk '$2 == 00000000 { print $1  }' /proc/net/route | uniq) && 
+    default_iface=`echo ${default_iface} | awk '{ print $1 }'` &&
+    mac_addr=`ip addr show dev ${default_iface} | awk '$1 ~ /^link\// { print $2 }'` && 
+    echo $mac_addr %s $HOSTNAME  > %s
+    """ % (env.host_string, src_path)
+    run(cmd)
+
+    dst_path = env.host_string + "/mac_ip_host"
+    get(src_path, dst_path)
+
+with open("hosts.yaml", 'r') as stream:
+    try:
+        y = yaml.load(stream)
+        env.hosts = y["hosts"]
+        env.user = y["user"]
+        env.password = y["password"]
+    except yaml.YAMLError as exc:
+        print(exc)
+        abort("load yaml error")
+
+
+

fabric/gpu_driver.py

@@ -0,0 +1,46 @@
+from __future__ import with_statement
+from fabric.api import *
+from fabric.contrib.console import confirm
+import fabric.operations as op
+import yaml
+import sys
+
+driver_version=""
+http_gpu_dir=""
+boot_strapper=""
+
+def prepare():
+    cmd = """setenforce 0 
+        && sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config 
+        && cat /etc/selinux/config | grep SELINUX"""
+    run(cmd)
+
+def install():
+    # Imporant: gpu must be installed after the kernel has been installed
+    run("wget -P /root %s/build_centos_gpu_drivers.sh" % http_gpu_dir)
+    cmd = "bash -x /root/build_centos_gpu_drivers.sh %s %s" % (driver_version, http_gpu_dir)
+    run(cmd)
+
+#@parallel
+def check():
+    cmd="ret=`nvidia-smi | grep  \"Driver Version\" | grep %s` ; if [[ -z $ret  ]]; then exit 1; fi " % driver_version
+    result = run(cmd)
+    if result.failed:
+        abort(env.host_string + ": check failed")
+
+with open("hosts.yaml", 'r') as stream:
+    try:
+        y = yaml.load(stream)
+        env.hosts = y["hosts"]
+        env.user = y["user"]
+        env.password = y["password"]
+
+        boot_strapper = y["boot_strapper"]
+        driver_version = y["gpu"]["driver_version"]
+
+        http_gpu_dir="http://%s/static/CentOS7/gpu_drivers" % boot_strapper
+    except yaml.YAMLError as exc:
+        print(exc)
+        abort("load yaml error")
+
+

fabric/hosts.template.yaml

@@ -0,0 +1,26 @@
+user: "root"
+password: "passwd"
+hosts:
+  - 192.168.16.23
+  - 192.168.16.24
+  - 192.168.16.25
+  - 192.168.16.26
+  - 192.168.16.27
+  - 192.168.16.28
+  - 192.168.16.29
+kernel:
+  old_version: "3.10.0-327.el7.x86_64"
+  new_version: "4.4.79-1.el7.elrepo.x86_64"
+gpu:
+  driver_verion: 375.26
+boot_strapper: "192.168.16.23"
+
+# host, mac, all
+set_type: host
+
+# change hostname to mac?
+set_mac_hostname: n
+
+# set "" if not need to change default path
+docker_data_path: "/home/var/lib/docker"
+etcd_data_path: "/home/var/lib/etcd"

fabric/k8s.py

@@ -0,0 +1,63 @@
+from __future__ import with_statement
+from fabric.api import *
+from fabric.contrib.console import confirm
+import fabric.operations as op
+import yaml
+import sys
+import re
+
+
+boot_strapper=""
+set_mac_hostname=""
+docker_data_path=""
+etcd_data_path=""
+
+def prepare():
+    run("systemctl stop firewalld && systemctl disable firewalld")
+    run("wget -O /etc/yum.repos.d/Cloud-init.repo http://%s/static/CentOS7/repo/cloud-init.repo" % boot_strapper)
+    run("wget -O /root/post-process.sh http://%s/centos/post-script/00-00-00-00-00-00" % boot_strapper)
+    run("wget -O /root http://%s/static/CentOS7/post_cloudinit_provision.sh" % boot_strapper)
+
+def install():
+    run("yum --enablerepo=Cloud-init install -y cloud-init docker-engine etcd flannel")
+    run("""cd /root 
+        && export set_mac_hostname=%s 
+        && export docker_data_path=%s 
+        && bash post-process.sh""" % (set_mac_hostname, docker_data_path))
+
+    if len(etcd_data_path) > 0 :
+        run("id -u etcd &>/dev/null || useradd etcd")
+        run("mkdir -p %s && chown etcd -R %s" % (etcd_data_path, etcd_data_path))
+
+    run(""" cd /root
+        && export bootstrapper_ip=%s 
+        && export etcd_data_path=%s 
+        && bash post_cloudinit_provision.sh""" % (boot_strapper, etcd_data_path))
+
+def rm_clouinit_cache():
+    run("rm -rf /var/lib/cloud/instances/iid-local01")
+
+def start_etcd():
+    run("""systemctl daemon-reload
+        && systemctl stop etcd
+        && systemctl enable etcd 
+        && systemctl start etcd""")
+
+with open("hosts.yaml", 'r') as stream:
+    try:
+        y = yaml.load(stream)
+        env.user = y["user"]
+        env.password = y["password"]
+        env.hosts = y["hosts"]
+        boot_strapper = y["boot_strapper"]
+
+        set_mac_hostname = y["set_mac_hostname"]
+        docker_data_path = y["docker_data_path"]
+        etcd_data_path = y["etcd_data_path"]
+    except yaml.YAMLError as exc:
+        print(exc)
+        abort("load yaml error")
+
+
+
+

fabric/set_hosts.py

@@ -0,0 +1,79 @@
+from __future__ import with_statement
+from fabric.api import *
+from fabric.contrib.console import confirm
+import fabric.operations as op
+import yaml
+import sys
+import re
+
+mac_ip={}
+host_ip={}
+set_type=""
+
+def modify_mac_hosts(path, ips):
+    import copy
+    local = copy.deepcopy(ips)
+
+    #hostname->ip
+    hosts = []
+    with open(path, "r") as fp:
+        for line in fp.read().split('\n'):
+            if len(re.sub('\s*', '', line)) and not line.startswith('#'):
+                parts = re.split('\s+', line)
+                ip = parts[0]
+                host_name = " ".join(parts[1:])
+                hosts.append([host_name, ip])
+        fp.close()
+
+    for n in hosts:
+        if n[0] in local:
+            n[1] = local[n[0]]
+            local[n[0]]= ""
+
+    with open(path, "w") as fw:
+        for n in hosts:
+            fw.write("%s %s\n" % (n[1], n[0]) )
+        for n in local:
+            if len(local[n]) > 0:
+                fw.write("%s %s\n" % (local[n], n) )
+        fw.close()
+
+def set_mac_hosts():
+    src_path = "/etc/hosts"
+    dst_path = env.host_string + "/hosts"
+    get(src_path)
+    if set_type == "mac" or set_type == "all":
+        modify_mac_hosts(dst_path, mac_ip)
+    if set_type == "host" or set_type == "all":
+        modify_mac_hosts(dst_path, host_ip)
+    put(dst_path, src_path)
+
+def display():
+    print host_ip
+    print mac_ip
+
+with open("hosts.yaml", 'r') as stream:
+    try:
+        y = yaml.load(stream)
+        env.hosts = y["hosts"]
+        env.user = y["user"]
+        env.password = y["password"]
+
+        set_type = y["set_type"]
+    except yaml.YAMLError as exc:
+        print(exc)
+        abort("load yaml error")
+
+for h in env.hosts:
+    dst_path = h + "/mac_ip_host"
+    with open(dst_path, "r") as fp:
+        for line in fp.read().split('\n'):
+            if len(re.sub('\s*', '', line)) and not line.startswith('#'):
+                parts = re.split('\s+', line)
+                mac = parts[0].replace(":", "-")
+                ip = parts[1]
+                host_name = parts[2]
+
+                mac_ip[mac] = ip
+                host_ip[host_name] = ip
+