diff --git a/bsroot.sh b/bsroot.sh index f33aca2..d8a2489 100755 --- a/bsroot.sh +++ b/bsroot.sh @@ -22,14 +22,15 @@ # bootstrap server when the bootstrapper server is "offline". Or, you can run # bsroot.sh directly on the bootstrap server. +alias cp='cp' + SEXTANT_ROOT=${PWD} +source $SEXTANT_ROOT/scripts/log.sh source $SEXTANT_ROOT/scripts/common.sh check_prerequisites check_cluster_desc_file - - echo "Install OS: ${cluster_desc_os_name}" if [[ $cluster_desc_os_name == "CentOS" ]]; then source $SEXTANT_DIR/scripts/centos.sh @@ -63,3 +64,4 @@ build_bootstrapper_image generate_tls_assets prepare_setup_kubectl generate_addons_config +log info "bsroot done!" diff --git a/docker/Dockerfile b/docker/Dockerfile index 037f8c1..58d19e7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,4 @@ -FROM alpine -#FROM golang:alpine +FROM distribution/registry # Install required software packages. RUN set -ex && \ @@ -9,7 +8,6 @@ apk add dnsmasq openssl # Upload Sextant Go programs and retrieve dependencies. RUN mkdir -p /go/bin COPY cloud-config-server /go/bin -COPY registry /go/bin # NOTICE: change install.sh HTTP server ip:port when running entrypoint.sh COPY entrypoint.sh / diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index c886f77..258ab50 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,9 +1,16 @@ #!/bin/sh -# start dnsmasq -mkdir -p /bsroot/dnsmasq -dnsmasq --log-facility=- --conf-file=/bsroot/config/dnsmasq.conf \ - --dhcp-leasefile=/bsroot/dnsmasq/dnsmasq.leases +if [[ $# != 1 ]]; then + echo "need to set start_pxe" + exit 1 +fi + +if [[ $1 == " y" ]]; then + # start dnsmasq + mkdir -p /bsroot/dnsmasq + dnsmasq --log-facility=- --conf-file=/bsroot/config/dnsmasq.conf \ + --dhcp-leasefile=/bsroot/dnsmasq/dnsmasq.leases +fi # start cloud-config-server /go/bin/cloud-config-server -addr ":80" \ diff --git a/fabric/README_CN.md b/fabric/README_CN.md new file mode 100644 index 0000000..5cfb742 --- /dev/null +++ b/fabric/README_CN.md @@ -0,0 +1,50 @@ +# 前言 +Sextant设计之初考虑的是在裸机集群中一键式的解决方案。实际使用的过程中,企业内部的集群一般都有了自己的初始化安装环境,如部署了DHCP服务器,有自己的DNS,机器也有自己的hostname,机器之间通过hostname相互也能ping同。这种情况下,同时两个DHCP服务器无疑是有冲突的,需要对Sextant做一些改动以便适应这种的环境。 + +我们可以把Sextant PXE服务部分设置为可选项,保留资源cache服务部分。由于post_script不能通过kick start的方式启动,所以引入fabric作为集群管理者,方便安装、配置、检查、启动、关闭软件。我们写的如下的步骤,都是在考虑了企业一般的现实情况来做的。 + +首先,`copy host.template.yaml host.yaml`,然后修改之。 + +***注意:*** +- 符合要求的步骤可以略过 +- 需要已经安装centos7的基础操作系统 + +# 步骤一:机器之间可以访问 +我们需要机器都可以通过hostname来相互之间访问。如果企业的网络不支持,需要我们把静态解析写入各个节点`/etc/hosts`中(已经支持的可以忽略)。 + +``` +# get mac_ip_host +fab -f get_mac_ip_host.py get_mac_addr + +# display all before set them +fab -f set_hosts.py display + +# set hosts +fab -f set_hosts.py set_mac_hosts +``` + +# 步骤二:生成bsroot +注意设置cluster-desc.yaml中的`start_pxe: n`。 + + +# 步骤三:升级kernel +``` +fab -f upgrade_kernel.py prepare +fab -f upgrade_kernel.py upgrade +fab -f upgrade_kernel.py reboot +``` + +# 步骤四:安装gpu driver +``` +fab -f gpu_driver.py prepare +fab -f gpu_driver.py install +fab -f gpu_driver.py check +``` + +# 步骤五:安装k8s需要的软件 +``` +fab -f k8s.py prepare +fab -f k8s.py install +``` + +# TODO: 启动etcd flannel kubelet等 \ No newline at end of file diff --git a/fabric/get_mac_ip_host.py b/fabric/get_mac_ip_host.py new file mode 100644 index 0000000..fd3d905 --- /dev/null +++ b/fabric/get_mac_ip_host.py @@ -0,0 +1,33 @@ +from __future__ import with_statement +from fabric.api import * +from fabric.contrib.console import confirm +import fabric.operations as op +import yaml +import sys +import re + +def get_mac_addr(): + src_path = "/etc/mac_ip_host" + + cmd = """ default_iface=$(awk '$2 == 00000000 { print $1 }' /proc/net/route | uniq) && + default_iface=`echo ${default_iface} | awk '{ print $1 }'` && + mac_addr=`ip addr show dev ${default_iface} | awk '$1 ~ /^link\// { print $2 }'` && + echo $mac_addr %s $HOSTNAME > %s + """ % (env.host_string, src_path) + run(cmd) + + dst_path = env.host_string + "/mac_ip_host" + get(src_path, dst_path) + +with open("hosts.yaml", 'r') as stream: + try: + y = yaml.load(stream) + env.hosts = y["hosts"] + env.user = y["user"] + env.password = y["password"] + except yaml.YAMLError as exc: + print(exc) + abort("load yaml error") + + + diff --git a/fabric/gpu_driver.py b/fabric/gpu_driver.py new file mode 100644 index 0000000..6539f80 --- /dev/null +++ b/fabric/gpu_driver.py @@ -0,0 +1,46 @@ +from __future__ import with_statement +from fabric.api import * +from fabric.contrib.console import confirm +import fabric.operations as op +import yaml +import sys + +driver_version="" +http_gpu_dir="" +boot_strapper="" + +def prepare(): + cmd = """setenforce 0 + && sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config + && cat /etc/selinux/config | grep SELINUX""" + run(cmd) + +def install(): + # Imporant: gpu must be installed after the kernel has been installed + run("wget -P /root %s/build_centos_gpu_drivers.sh" % http_gpu_dir) + cmd = "bash -x /root/build_centos_gpu_drivers.sh %s %s" % (driver_version, http_gpu_dir) + run(cmd) + +#@parallel +def check(): + cmd="ret=`nvidia-smi | grep \"Driver Version\" | grep %s` ; if [[ -z $ret ]]; then exit 1; fi " % driver_version + result = run(cmd) + if result.failed: + abort(env.host_string + ": check failed") + +with open("hosts.yaml", 'r') as stream: + try: + y = yaml.load(stream) + env.hosts = y["hosts"] + env.user = y["user"] + env.password = y["password"] + + boot_strapper = y["boot_strapper"] + driver_version = y["gpu"]["driver_version"] + + http_gpu_dir="http://%s/static/CentOS7/gpu_drivers" % boot_strapper + except yaml.YAMLError as exc: + print(exc) + abort("load yaml error") + + diff --git a/fabric/hosts.template.yaml b/fabric/hosts.template.yaml new file mode 100644 index 0000000..27a532c --- /dev/null +++ b/fabric/hosts.template.yaml @@ -0,0 +1,26 @@ +user: "root" +password: "passwd" +hosts: + - 192.168.16.23 + - 192.168.16.24 + - 192.168.16.25 + - 192.168.16.26 + - 192.168.16.27 + - 192.168.16.28 + - 192.168.16.29 +kernel: + old_version: "3.10.0-327.el7.x86_64" + new_version: "4.4.79-1.el7.elrepo.x86_64" +gpu: + driver_verion: 375.26 +boot_strapper: "192.168.16.23" + +# host, mac, all +set_type: host + +# change hostname to mac? +set_mac_hostname: n + +# set "" if not need to change default path +docker_data_path: "/home/var/lib/docker" +etcd_data_path: "/home/var/lib/etcd" diff --git a/fabric/k8s.py b/fabric/k8s.py new file mode 100644 index 0000000..5a4424a --- /dev/null +++ b/fabric/k8s.py @@ -0,0 +1,63 @@ +from __future__ import with_statement +from fabric.api import * +from fabric.contrib.console import confirm +import fabric.operations as op +import yaml +import sys +import re + + +boot_strapper="" +set_mac_hostname="" +docker_data_path="" +etcd_data_path="" + +def prepare(): + run("systemctl stop firewalld && systemctl disable firewalld") + run("wget -O /etc/yum.repos.d/Cloud-init.repo http://%s/static/CentOS7/repo/cloud-init.repo" % boot_strapper) + run("wget -O /root/post-process.sh http://%s/centos/post-script/00-00-00-00-00-00" % boot_strapper) + run("wget -O /root http://%s/static/CentOS7/post_cloudinit_provision.sh" % boot_strapper) + +def install(): + run("yum --enablerepo=Cloud-init install -y cloud-init docker-engine etcd flannel") + run("""cd /root + && export set_mac_hostname=%s + && export docker_data_path=%s + && bash post-process.sh""" % (set_mac_hostname, docker_data_path)) + + if len(etcd_data_path) > 0 : + run("id -u etcd &>/dev/null || useradd etcd") + run("mkdir -p %s && chown etcd -R %s" % (etcd_data_path, etcd_data_path)) + + run(""" cd /root + && export bootstrapper_ip=%s + && export etcd_data_path=%s + && bash post_cloudinit_provision.sh""" % (boot_strapper, etcd_data_path)) + +def rm_clouinit_cache(): + run("rm -rf /var/lib/cloud/instances/iid-local01") + +def start_etcd(): + run("""systemctl daemon-reload + && systemctl stop etcd + && systemctl enable etcd + && systemctl start etcd""") + +with open("hosts.yaml", 'r') as stream: + try: + y = yaml.load(stream) + env.user = y["user"] + env.password = y["password"] + env.hosts = y["hosts"] + boot_strapper = y["boot_strapper"] + + set_mac_hostname = y["set_mac_hostname"] + docker_data_path = y["docker_data_path"] + etcd_data_path = y["etcd_data_path"] + except yaml.YAMLError as exc: + print(exc) + abort("load yaml error") + + + + diff --git a/fabric/set_hosts.py b/fabric/set_hosts.py new file mode 100644 index 0000000..87d7509 --- /dev/null +++ b/fabric/set_hosts.py @@ -0,0 +1,79 @@ +from __future__ import with_statement +from fabric.api import * +from fabric.contrib.console import confirm +import fabric.operations as op +import yaml +import sys +import re + +mac_ip={} +host_ip={} +set_type="" + +def modify_mac_hosts(path, ips): + import copy + local = copy.deepcopy(ips) + + #hostname->ip + hosts = [] + with open(path, "r") as fp: + for line in fp.read().split('\n'): + if len(re.sub('\s*', '', line)) and not line.startswith('#'): + parts = re.split('\s+', line) + ip = parts[0] + host_name = " ".join(parts[1:]) + hosts.append([host_name, ip]) + fp.close() + + for n in hosts: + if n[0] in local: + n[1] = local[n[0]] + local[n[0]]= "" + + with open(path, "w") as fw: + for n in hosts: + fw.write("%s %s\n" % (n[1], n[0]) ) + for n in local: + if len(local[n]) > 0: + fw.write("%s %s\n" % (local[n], n) ) + fw.close() + +def set_mac_hosts(): + src_path = "/etc/hosts" + dst_path = env.host_string + "/hosts" + get(src_path) + if set_type == "mac" or set_type == "all": + modify_mac_hosts(dst_path, mac_ip) + if set_type == "host" or set_type == "all": + modify_mac_hosts(dst_path, host_ip) + put(dst_path, src_path) + +def display(): + print host_ip + print mac_ip + +with open("hosts.yaml", 'r') as stream: + try: + y = yaml.load(stream) + env.hosts = y["hosts"] + env.user = y["user"] + env.password = y["password"] + + set_type = y["set_type"] + except yaml.YAMLError as exc: + print(exc) + abort("load yaml error") + +for h in env.hosts: + dst_path = h + "/mac_ip_host" + with open(dst_path, "r") as fp: + for line in fp.read().split('\n'): + if len(re.sub('\s*', '', line)) and not line.startswith('#'): + parts = re.split('\s+', line) + mac = parts[0].replace(":", "-") + ip = parts[1] + host_name = parts[2] + + mac_ip[mac] = ip + host_ip[host_name] = ip + diff --git a/fabric/upgrade_kernel.py b/fabric/upgrade_kernel.py new file mode 100644 index 0000000..88d43a4 --- /dev/null +++ b/fabric/upgrade_kernel.py @@ -0,0 +1,65 @@ +from __future__ import with_statement +from fabric.api import * +from fabric.contrib.console import confirm +import fabric.operations as op + +new_kernel_version="" +old_kernel_version="" +boot_strapper="" + +def prepare(): + run("sed -i '/exclude=*/ s/^/#/' /etc/yum.conf") + +def post(): + run("sed -i -e '/exclude=*kernel*/ s/^#//' /etc/yum.conf") + +@parallel +def upgrade(): + put("./upgrade_kernel.sh", "/tmp/upgrade_kernel.sh") + result = run("bash /tmp/upgrade_kernel.sh %s" % boot_strapper) + if result.failed: + abort("failed") + run("grub2-set-default \"CentOS Linux (%s) 7 (Core)\"" % new_kernel_version) + +def reset(): + cmd = "grub2-set-default \"CentOS Linux (%s) 7 (Core)\"" % old_kernel_version + run(cmd) + +def check(): + cmd = "if [[ ! -d /usr/src/kernels/%s ]]; then exit 1; fi" % new_kernel_version + result = run(cmd) + if result.failed: + abort(env.host_string + " check failed") + + start = "saved_entry=CentOS Linux (%s) 7 (Core)" % new_kernel_version + cmd = "if [[ \"$(grub2-editenv list)\" != \"%s\" ]]; then exit 1; fi" % start + result = run(cmd) + if result.failed: + abort(env.host_string + " check failed") + +@parallel +def reboot(): + run("systemctl set-default multi-user.target && reboot") + +def display(): + run("uname -a") + +import yaml + +with open("hosts.yaml", 'r') as stream: + try: + y = yaml.load(stream) + env.hosts = y["hosts"] + env.user = y["user"] + env.password = y["password"] + + new_kernel_version=y["kernel"]["new_version"] + old_kernel_version=y["kernel"]["old_version"] + boot_strapper = y["boot_strapper"] + + #print new_kernel_version, old_kernel_version + print "grub2-set-default \"CentOS Linux (%s) 7 (Core)\"" % new_kernel_version + except yaml.YAMLError as exc: + print(exc) + abort("load yaml error") + diff --git a/fabric/upgrade_kernel.sh b/fabric/upgrade_kernel.sh new file mode 100755 index 0000000..b610e95 --- /dev/null +++ b/fabric/upgrade_kernel.sh @@ -0,0 +1,16 @@ +#!/bin/bash +#export https_proxy=192.168.16.30:3128 +#export http_proxy=192.168.16.30:3128 + +if [[ $# > 0 ]]; then + wget -O /etc/yum.repos.d/Cloud-init.repo http://$1/static/CentOS7/repo/cloud-init.repo + yum --enablerepo=Cloud-init -y -d1 install kernel-lt kernel-lt-devel +else + rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org + rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm + yum --enablerepo=elrepo-kernel -y -d1 install kernel-lt kernel-lt-devel +fi + +if [[ ! -f /boot/grub2/grub.cfg ]]; then + grub2-mkconfig --output=/boot/grub2/grub.cfg +fi diff --git a/golang/cloud-config-server/server_test.go b/golang/cloud-config-server/server_test.go index 1668f14..7965269 100644 --- a/golang/cloud-config-server/server_test.go +++ b/golang/cloud-config-server/server_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "io" "io/ioutil" "log" @@ -19,7 +20,6 @@ import ( ) const ( - tmplFile = "src/github.com/k8sp/sextant/golang/template/cloud-config.template" templateDir = "../template/templatefiles" clusterDescExampleFile = "../template/cluster-desc.sample.yaml" loadTimeout = 15 * time.Second @@ -62,6 +62,8 @@ func TestCloudConfigHandler(t *testing.T) { status, http.StatusOK) } + fmt.Println(rr.Body.String()) + if rr.Body.String() != "" { // Compare only a small fraction -- the etcd2 initial cluster -- for testing. yml := make(map[interface{}]interface{}) diff --git a/golang/clusterdesc/config.go b/golang/clusterdesc/config.go index 00ca792..83731e8 100644 --- a/golang/clusterdesc/config.go +++ b/golang/clusterdesc/config.go @@ -4,9 +4,10 @@ package clusterdesc import ( - "github.com/topicai/candy" "net" "strings" + + "github.com/topicai/candy" ) // Cluster configures a cluster, which includes: (1) a @@ -56,6 +57,7 @@ type Cluster struct { DNSMASQSetNTP bool `yaml:"set_ntp"` DNSMASQLease string `yaml:"lease"` CentOSYumRepo string `yaml:"set_yum_repo"` + StartPXE bool `yaml:"start_pxe"` } // CoreOS defines the system related operations, such as: system updates. @@ -80,6 +82,7 @@ type Ceph struct { // Cluster.IPLow and Cluster.IPHigh. type Node struct { MAC string + CurHostName string `yaml:"cur_host_name"` IngressLabel bool CephMonitor bool `yaml:"ceph_monitor"` KubeMaster bool `yaml:"kube_master"` @@ -107,6 +110,9 @@ func (c Cluster) GetIngressReplicas() int { // Hostname is defined as a method of Node, so can be call in // template. For more details, refer to const tmplDHCPConf. func (n Node) Hostname() string { + if n.CurHostName != "" { + return n.CurHostName + } return strings.ToLower(strings.Replace(n.Mac(), ":", "-", -1)) } diff --git a/golang/template/cluster-desc.sample.yaml b/golang/template/cluster-desc.sample.yaml index 8f7fadb..cb4fe58 100644 --- a/golang/template/cluster-desc.sample.yaml +++ b/golang/template/cluster-desc.sample.yaml @@ -1,3 +1,9 @@ +# NOTICE: +# 1. If you change version of them such as kube,centos, it's better to +# delete the bsroot directory because it does't update a exist file. +# or do what you know. +# 2. Please check that $mirror_site/$centos_version/(isos|os) exist + bootstrapper: 10.10.14.253 subnet: 10.10.14.0 netmask: 255.255.255.0 @@ -12,6 +18,9 @@ dockerdomain: "bootstrapper" k8s_service_cluster_ip_range: 10.100.0.0/24 k8s_cluster_dns: 10.100.0.10 +#start pxe? +start_pxe: n + # Flannel backend only support "host-gw", "vxlan" and "udp" for now. flannel_backend: "host-gw" @@ -22,12 +31,16 @@ coreos_channel: "stable" coreos_version: "1122.2.0" # centos version -download_kernel: n -centos_version: "7.3.1611" +download_kernel: y +# configure mirror_site, sunch as: +# http://mirrors.163.com/centos +mirror_site: http://archive.kernel.org/centos-vault +# Configure the centos version for isos and rpms +centos_version: "7.2.1511" # gpu drivers version -set_gpu: n -gpu_drivers_version: "375.20" +set_gpu: y +gpu_drivers_version: "375.26" ingress_hostnetwork: true @@ -64,44 +77,52 @@ ceph: osd_journal_size: 5000 images: - hyperkube: "pineking/hyperkube-amd64:2169be" - pause: "typhoon1986/pause-amd64:3.0" - flannel: "typhoon1986/flannel:0.5.5" - ingress: "yancey1989/nginx-ingress-controller:0.8.3" - kubednsmasq: "pineking/kube-dnsmasq-amd64:1.4" - healthz: "pineking/exechealthz-amd64:1.2" - addon_manager: "pineking/kube-addon-manager-amd64:v6.4-alpha.3" - kubedns: "pineking/kubedns-amd64:1.9" + hyperkube: "gcr.io/google_containers/hyperkube-amd64:v1.6.2" + pause: "gcr.io/google_containers/pause-amd64:3.0" + flannel: "quay.io/coreos/flannel:v0.7.1-amd64" + ingress: "gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.3" + default_backend: "gcr.io/google_containers/defaultbackend:1.0" + kubedns: "gcr.io/google_containers/kubedns-amd64:1.9" + kubednsmasq: "gcr.io/google_containers/kube-dnsmasq-amd64:1.4" + healthz: "gcr.io/google_containers/exechealthz-amd64:1.2" + addon_manager: "gcr.io/google_containers/kube-addon-manager-amd64:v6.4-beta.1" ceph: "typhoon1986/ceph-daemon:tag-build-master-jewel-ubuntu-14.04-fix370" - default_backend: "yancey1989/defaultbackend:1.0" ntp: "redaphid/docker-ntp-server" heapster: "kubernetes/heapster:canary" grafana: "lupan/heapster_grafana:v2.6.0-2" influxdb: "lupan/heapster_influxdb:v0.5" - dashboard: "pineking/kubernetes-dashboard-amd64:v1.6.0" + dashboard: "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.0" + +# if you wan't use mac as hostname, please delete cur_host_name or +# set them to "" nodes: - mac: "00:25:90:c0:f7:80" + cur_host_name: "node1" ceph_monitor: n kube_master: y etcd_member: y ingress_label: n - mac: "0c:c4:7a:82:c5:bc" + cur_host_name: "node2" ceph_monitor: n kube_master: n etcd_member: y ingress_label: n - mac: "0c:c4:7a:82:c5:b8" + cur_host_name: "node3" ceph_monitor: n kube_master: n etcd_member: y ingress_label: y - mac: "00:25:90:c0:f6:ee" + cur_host_name: "node4" ceph_monitor: n kube_master: n etcd_member: n ingress_label: n - mac: "00:25:90:c0:f6:d6" + cur_host_name: "node5" ceph_monitor: n kube_master: n etcd_member: n diff --git a/golang/template/template.go b/golang/template/template.go index 21a6f3d..cbd4014 100644 --- a/golang/template/template.go +++ b/golang/template/template.go @@ -43,6 +43,7 @@ type ExecutionConfig struct { CoreOSVersion string GPUDriversVersion string OSName string + StartPXE bool } // Execute load template files from "ccTemplateDir", parse clusterDescFile to @@ -106,6 +107,7 @@ func GetConfigDataByMac(mac string, clusterdesc *clusterdesc.Cluster, caKey, caC CoreOSVersion: clusterdesc.CoreOSVersion, GPUDriversVersion: clusterdesc.GPUDriversVersion, OSName: clusterdesc.OSName, + StartPXE: clusterdesc.StartPXE, } } diff --git a/golang/template/templatefiles/cc-centos-post.template b/golang/template/templatefiles/cc-centos-post.template index e065d94..cba4b1b 100644 --- a/golang/template/templatefiles/cc-centos-post.template +++ b/golang/template/templatefiles/cc-centos-post.template @@ -24,6 +24,11 @@ set_docker() { # set iptables to false for docker version >=1.13, to solve the iptables FORWARD drop sed -i -e '/^ExecStart=/ s/$/ --iptables=false/' /usr/lib/systemd/system/docker.service + # set path + if [[ ! -z "${docker_data_path}" ]]; then + sed -i -e "/^ExecStart=/ s@\$@ -g ${docker_data_path} @" /usr/lib/systemd/system/docker.service + fi + # Explicit Docker option sed -i -e '/^ExecStart=/ s/$/ $DOCKER_OPT_BIP $DOCKER_OPT_IPMASQ $DOCKER_OPT_MTU $DOCKER_NETWORK_OPTIONS/' /usr/lib/systemd/system/docker.service } @@ -63,12 +68,15 @@ else fi -yum clean all -yum makecache +#yum clean all +#yum makecache } -set_hostname +if [[ "${set_mac_hostname}" != " n" ]]; then + set_hostname +fi + set_docker set_ssh_config set_yum_repo diff --git a/golang/template/templatefiles/cc-centos.template b/golang/template/templatefiles/cc-centos.template index e66c92a..afdb02b 100644 --- a/golang/template/templatefiles/cc-centos.template +++ b/golang/template/templatefiles/cc-centos.template @@ -190,6 +190,7 @@ --hostname-override={{ .MasterHostname }} \ --cluster-dns={{ .K8sClusterDNS }} \ --cluster-domain=cluster.local \ + --cgroup-driver=systemd \ --feature-gates=Accelerators=true Restart=always RestartSec=10 diff --git a/golang/template/templatefiles/cc-common.template b/golang/template/templatefiles/cc-common.template index 9614c4a..b898eca 100644 --- a/golang/template/templatefiles/cc-common.template +++ b/golang/template/templatefiles/cc-common.template @@ -11,11 +11,13 @@ permissions: 0600 content: | {{ .CaCrt }} + {{- if .StartPXE }} - path: /etc/hosts owner: root content: | 127.0.0.1 localhost {{ .BootstrapperIP }} {{ .Dockerdomain }} + {{- end}} {{/* ********************************************************* */}} {{- if .KubeMaster }} - path: /etc/kubernetes/ssl/apiserver.pem diff --git a/scripts/centos.sh b/scripts/centos.sh index f2e425d..13f81d5 100644 --- a/scripts/centos.sh +++ b/scripts/centos.sh @@ -4,36 +4,37 @@ GPU_DIR='gpu_drivers' ABSOLUTE_GPU_DIR="$BSROOT/html/static/CentOS7/$GPU_DIR" HTTP_GPU_DIR="http://$BS_IP/static/CentOS7/$GPU_DIR" - download_centos_images() { VERSION=CentOS7 mkdir -p $BSROOT/tftpboot - printf "Downloading syslinux ... " - wget --quiet -c -N -P $BSROOT/tftpboot https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-6.03.tar.gz || { echo "Failed"; exit 1; } + log info "Downloading syslinux ..." + wget --quiet -c -N -P $BSROOT/tftpboot https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-6.03.tar.gz || { log fatal "Failed"; exit 1; } cd $BSROOT/tftpboot - tar xzf syslinux-6.03.tar.gz || { echo "Failed"; exit 1; } - cp syslinux-6.03/bios/core/pxelinux.0 $BSROOT/tftpboot || { echo "Failed"; exit 1; } - cp syslinux-6.03/bios/com32/menu/vesamenu.c32 $BSROOT/tftpboot || { echo "Failed"; exit 1; } - cp syslinux-6.03/bios/com32/elflink/ldlinux/ldlinux.c32 $BSROOT/tftpboot || { echo "Failed"; exit 1; } - rm -rf syslinux-6.03 || { echo "Failed"; exit 1; } # Clean the untarred. - echo "Done" - - printf "Downloading CentOS 7 PXE vmlinuz image ... " + tar xzf syslinux-6.03.tar.gz || { log fatal "Failed"; exit 1; } + cp syslinux-6.03/bios/core/pxelinux.0 $BSROOT/tftpboot || { log fatal "Failed"; exit 1; } + cp syslinux-6.03/bios/com32/menu/vesamenu.c32 $BSROOT/tftpboot || { log fatal "Failed"; exit 1; } + cp syslinux-6.03/bios/com32/elflink/ldlinux/ldlinux.c32 $BSROOT/tftpboot || { log fatal "Failed"; exit 1; } + rm -rf syslinux-6.03 || { log fatal "Failed"; exit 1; } # Clean the untarred. + log info "Done" + + log info "Downloading CentOS 7 PXE vmlinuz image ..." cd $BSROOT/tftpboot mkdir -p $BSROOT/tftpboot/CentOS7 - wget --quiet -c -N -P $BSROOT/tftpboot/CentOS7 http://mirrors.163.com/centos/$cluster_desc_centos_version/os/x86_64/images/pxeboot/initrd.img || { echo "Failed"; exit 1; } - wget --quiet -c -N -P $BSROOT/tftpboot/CentOS7 http://mirrors.163.com/centos/$cluster_desc_centos_version/os/x86_64/images/pxeboot/vmlinuz || { echo "Failed"; exit 1; } - echo "Done" + wget --quiet -c -N -P $BSROOT/tftpboot/CentOS7 $cluster_desc_mirror_site/$cluster_desc_centos_version/os/x86_64/images/pxeboot/initrd.img || { log fatal "Failed"; exit 1; } + wget --quiet -c -N -P $BSROOT/tftpboot/CentOS7 $cluster_desc_mirror_site/$cluster_desc_centos_version/os/x86_64/images/pxeboot/vmlinuz || { log fatal "Failed"; exit 1; } + log info "Done" - printf "Downloading CentOS 7 ISO ... " + log info "Downloading CentOS 7 ISO ..." mkdir -p $BSROOT/html/static/CentOS7 - wget --quiet -c -N -P $BSROOT/html/static/CentOS7 http://mirrors.163.com/centos/$cluster_desc_centos_version/isos/x86_64/CentOS-7-x86_64-Everything-1611.iso || { echo "Failed"; exit 1; } - echo "Done" + centos7_src=$cluster_desc_mirror_site/$cluster_desc_centos_version/isos/x86_64/CentOS-7-x86_64-Everything-${cluster_desc_centos_version##*.}.iso + echo $centos7_src + wget --quiet -c -N -P $BSROOT/html/static/CentOS7 ${centos7_src} || { log fatal "Failed"; exit 1; } + log info "Done" } generate_pxe_centos_config() { - printf "Generating pxelinux.cfg ... " + log info "Generating pxelinux.cfg ..." mkdir -p $BSROOT/tftpboot/pxelinux.cfg cat > $BSROOT/tftpboot/pxelinux.cfg/default < $BSROOT/html/static/CentOS7/ks.cfg <> /root/cloudinit.log %end EOF - echo "Done" + log info "Done" } generate_post_cloudinit_script() { - printf "Generating post cloudinit script ... " + log info "Generating post cloudinit script ..." mkdir -p $BSROOT/html/static/CentOS7 cat > $BSROOT/html/static/CentOS7/post_cloudinit_provision.sh <<'EOF' #!/bin/bash default_iface=$(awk '$2 == 00000000 { print $1 }' /proc/net/route | uniq) -bootstrapper_ip=$(grep nameserver /etc/resolv.conf|cut -d " " -f2) +if [[ -z ${bootstrapper_ip} ]]; then + bootstrapper_ip=$(grep nameserver /etc/resolv.conf|cut -d " " -f2) +fi printf "Default interface: ${default_iface}\n" default_iface=`echo ${default_iface} | awk '{ print $1 }'` @@ -155,6 +158,11 @@ cd /var/lib/cloud/seed/nocloud-net/ wget -O user-data http://$bootstrapper_ip/cloud-config/${mac_addr} +if [[ ! -z ${etcd_data_path} ]]; then + sed -i "s@Environment=ETCD_DATA_DIR=.*@Environment=ETCD_DATA_DIR=${etcd_data_path}@" \ + /var/lib/cloud/seed/nocloud-net/user-data +fi + cat > /var/lib/cloud/seed/nocloud-net/meta-data << eof instance-id: iid-local01 eof @@ -166,13 +174,12 @@ systemctl stop NetworkManager systemctl disable NetworkManager systemctl enable docker EOF - echo "Done" - + log info "Done" } generate_rpmrepo_config() { - printf "Generating rpm repo configuration files ..." + log info "Generating rpm repo configuration files ..." mkdir -p $BSROOT/html/static/CentOS7/repo cat > $BSROOT/html/static/CentOS7/repo/cloud-init.repo </dev/null 2>&1 || { echo "Install $tool before run this script"; err=1; } @@ -67,12 +67,12 @@ function check_prerequisites() { if [[ $err -ne 0 ]]; then exit 1 fi - echo "Done" + log info "Done" } check_cluster_desc_file() { - printf "Cross-compiling validate-yaml ... " + log info "Cross-compiling validate-yaml ... " docker run --rm -it \ --volume $GOPATH:/go \ -e CGO_ENABLED=0 \ @@ -80,17 +80,17 @@ check_cluster_desc_file() { -e GOARCH=amd64 \ golang:wheezy \ go get github.com/k8sp/sextant/golang/validate-yaml \ - || { echo "Build sextant failed..."; exit 1; } - echo "Done" + || { log fatal "Build sextant failed..."; exit 1; } + log info "Done" - printf "Copying cloud-config template and cluster-desc.yml ... " + log info "Copying cloud-config template and cluster-desc.yml ... " mkdir -p $BSROOT/config > /dev/null 2>&1 cp -r $SEXTANT_DIR/golang/template/templatefiles $BSROOT/config cp $CLUSTER_DESC $BSROOT/config - echo "Done" + log info "Done" - printf "Checking cluster description file ..." + log info "Checking cluster description file ..." docker run --rm -it \ --volume $GOPATH:/go \ --volume $BSROOT:/bsroot \ @@ -99,11 +99,11 @@ check_cluster_desc_file() { --cloud-config-dir /bsroot/config/templatefiles \ -cluster-desc /bsroot/config/cluster-desc.yml \ > /dev/null 2>&1 || { echo "Failed"; exit 1; } - echo "Done" + log info "Done" } generate_registry_config() { - printf "Generating Docker registry config file ... " + log info "Generating Docker registry config file ... " mkdir -p $BSROOT/registry_data [ ! -d $BSROOT/config ] && mkdir -p $BSROOT/config cat > $BSROOT/config/registry.yml </$OSD_JOURNAL_SIZE/g" \ > $BSROOT/html/static/ceph/install-mon.sh || { echo "install-mon Failed"; exit 1; } sed "s/ceph\/daemon/$CEPH_DAEMON_IMAGE/g" $INSTALL_CEPH_SCRIPT_DIR/install-osd.sh \ > $BSROOT/html/static/ceph/install-osd.sh || { echo "install-osd Failed"; exit 1; } - echo "Done" + log info "Done" } build_bootstrapper_image() { # cloud-config-server and addon compile moved to check_cluster_desc_file - # Compile registry and build docker image here - printf "Cross-compiling Docker registry ... " - docker run --rm -it --name=registry_build \ - --volume $GOPATH:/go \ - -e CGO_ENABLED=0 \ - -e GOOS=linux \ - -e GOARCH=amd64 \ - golang:wheezy \ - sh -c "go get -u -d github.com/docker/distribution/cmd/registry && cd /go/src/github.com/docker/distribution && make PREFIX=/go clean /go/bin/registry >/dev/null" \ - || { echo "Complie Docker registry failed..."; exit 1; } - - printf "Cross-compiling cloud-config-server, addons ... " + log info "Cross-compiling cloud-config-server, addons ... " docker run --rm -it \ --volume $GOPATH:/go \ -e CGO_ENABLED=0 \ @@ -172,49 +161,49 @@ build_bootstrapper_image() { -e GOARCH=amd64 \ golang:wheezy \ go get github.com/k8sp/sextant/golang/cloud-config-server github.com/k8sp/sextant/golang/addons \ - || { echo "Build sextant failed..."; exit 1; } - echo "Done" + || { log fatal "Build sextant failed..."; exit 1; } + log info "Done" - rm -rf $SEXTANT_DIR/docker/{cloud-config-server,addons,registry} - cp $GOPATH/bin/{cloud-config-server,addons,registry} $SEXTANT_DIR/docker - echo "Done" + rm -rf $SEXTANT_DIR/docker/{cloud-config-server,addons} + cp $GOPATH/bin/{cloud-config-server,addons} $SEXTANT_DIR/docker + log info "Done" - printf "Building bootstrapper image ... " + log info "Building bootstrapper image ... " docker rm -f bootstrapper > /dev/null 2>&1 || echo "No such container: bootstrapper ,Pass..." docker rmi bootstrapper:latest > /dev/null 2>&1 || echo "No such images: bootstrapper ,Pass..." cd $SEXTANT_DIR/docker docker build -t bootstrapper . docker save bootstrapper:latest > $BSROOT/bootstrapper.tar || { echo "Failed"; exit 1; } - echo "Done" + log info "Done" - printf "Copying bash scripts ... " + log info "Copying bash scripts ... " cp $SEXTANT_DIR/start_bootstrapper_container.sh $BSROOT/ chmod +x $BSROOT/start_bootstrapper_container.sh cp $SEXTANT_DIR/scripts/load_yaml.sh $BSROOT/ - echo "Done" + log info "Done" - printf "Make directory ... " + log info "Make directory ..." mkdir -p $BSROOT/dnsmasq - echo "Done" + log info "Done" } download_k8s_images() { # Fetch release binary tarball from github accroding to the versions # defined in "cluster-desc.yml" - # hyperkube_version=`grep "hyperkube:" $CLUSTER_DESC | grep -o '".*hyperkube.*:.*"' | sed 's/".*://; s/"//'` - # printf "Downloading kubelet ${hyperkube_version} ... " - # wget --quiet -c -N -O $BSROOT/html/static/kubelet https://storage.googleapis.com/kubernetes-release/release/$hyperkube_version/bin/linux/amd64/kubelet - printf "Downloading kubelet ... " - wget --quiet -c -N -O $BSROOT/html/static/kubelet https://dl.dropboxusercontent.com/u/27178121/kubelet.v1.6.0/kubelet - echo "Done" + hyperkube_version=`grep "hyperkube:" $CLUSTER_DESC | grep -o '".*hyperkube.*:.*"' | sed 's/".*://; s/"//'` + log info "Downloading kubelet ${hyperkube_version} ... " + src="https://storage.googleapis.com/kubernetes-release/release/${hyperkube_version}/bin/linux/amd64/kubelet" + log debug $src + wget --quiet -c -N -O $BSROOT/html/static/kubelet $src || { log fatal "Failed"; exit 1; } + log info "Done" # setup-network-environment will fetch the default system IP infomation # when using cloud-config file to initiate a kubernetes cluster node - printf "Downloading setup-network-environment file ... " + log info "Downloading setup-network-environment file ... " wget --quiet -c -N -O $BSROOT/html/static/setup-network-environment-1.0.1 https://github.com/kelseyhightower/setup-network-environment/releases/download/1.0.1/setup-network-environment || { echo "Failed"; exit 1; } - echo "Done" + log info "Done" for DOCKER_IMAGE in $(set | grep '^cluster_desc_images_' | grep -o '".*"' | sed 's/"//g'); do @@ -225,15 +214,15 @@ download_k8s_images() { local DOCKER_TAR_FILE=$BSROOT/`echo $DOCKER_IMAGE.tar | sed "s/:/_/g" |awk -F'/' '{print $NF}'` if [[ ! -f $DOCKER_TAR_FILE ]]; then if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep $DOCKER_DOMAIN_IMAGE_URL > /dev/null; then - printf "Pulling image ${DOCKER_IMAGE} ... " + log info "Pulling image ${DOCKER_IMAGE} ... " docker pull $DOCKER_IMAGE > /dev/null 2>&1 - echo "Done" + docker tag $DOCKER_IMAGE $DOCKER_DOMAIN_IMAGE_URL + log info "Done" fi - printf "Exporting $DOCKER_TAR_FILE ... " - docker tag $DOCKER_IMAGE $DOCKER_DOMAIN_IMAGE_URL + log info "Exporting $DOCKER_TAR_FILE ... " docker save $DOCKER_DOMAIN_IMAGE_URL > $DOCKER_TAR_FILE.progress mv $DOCKER_TAR_FILE.progress $DOCKER_TAR_FILE - echo "Done" + log info "Done" else echo "Use existing $DOCKER_TAR_FILE" fi @@ -252,35 +241,36 @@ generate_tls_assets() { else - printf "Generating CA TLS assets ... " + log info "Generating CA TLS assets ... " openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 openssl req -x509 -new -nodes -key ca-key.pem -days 3650 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1 - echo "Done" + log info "Done" - printf "Generating bootstrapper TLS assets ... " + log info "Generating bootstrapper TLS assets ... " openssl genrsa -out bootstrapper.key 2048 > /dev/null 2>&1 openssl req -new -key bootstrapper.key -out bootstrapper.csr -subj "/CN=bootstrapper" > /dev/null 2>&1 openssl x509 -req -in bootstrapper.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out bootstrapper.crt -days 3650 > /dev/null 2>&1 - echo "Done" + log info "Done" fi } prepare_setup_kubectl() { - printf "Downloading kubectl ... " - wget --quiet -c -N -O $BSROOT/html/static/kubectl https://dl.dropboxusercontent.com/u/27178121/kubelet.v1.6.0/kubectl || { echo "Failed"; exit 1; } - echo "Done" + hyperkube_version=`grep "hyperkube:" $CLUSTER_DESC | grep -o '".*hyperkube.*:.*"' | sed 's/".*://; s/"//'` + log info "Downloading kubectl ${hyperkube_version} ... " + wget --quiet -c -N -O $BSROOT/html/static/kubectl https://storage.googleapis.com/kubernetes-release/release/$hyperkube_version/bin/linux/amd64/kubectl || { echo "Failed"; exit 1; } + log info "Done" - printf "Preparing setup kubectl ... " + log info "Preparing setup kubectl ... " sed -i -e "s//$KUBE_MASTER_HOSTNAME/g" $SEXTANT_DIR/setup-kubectl.bash sed -i -e "s//$BS_IP/g" $SEXTANT_DIR/setup-kubectl.bash cp $SEXTANT_DIR/setup-kubectl.bash $BSROOT/setup_kubectl.bash chmod +x $BSROOT/setup_kubectl.bash - echo "Done" + log info "Done" } generate_addons_config() { - printf "Generating configuration files ..." + log info "Generating configuration files ..." mkdir -p $BSROOT/html/static/addons-config/ docker run --rm -it \ @@ -297,6 +287,6 @@ generate_addons_config() { cp $SEXTANT_DIR/golang/addons/template/$file $BSROOT/html/static/addons-config/$file; done - echo "Done" + log info "Done" } diff --git a/scripts/log.sh b/scripts/log.sh new file mode 100644 index 0000000..72d1f15 --- /dev/null +++ b/scripts/log.sh @@ -0,0 +1,25 @@ +#!/bin/bash +loglevel=0 #debug:0; info:1; warn:2; error:3; fatal:4 +function log { + local msg;local logtype + logtype=$1 + msg=$2 + datetime=`date +'%F %H:%M:%S'` + logformat="[${logtype}]${datetime} ${FUNCNAME[@]/log/} [line:`caller 0 | awk '{print$1}'`]\t${msg}" + { + case $logtype in + debug) + [[ $loglevel -le 0 ]] && echo -e "\033[30m${logformat}\033[0m" ;; + info) + [[ $loglevel -le 1 ]] && echo -e "\033[32m${logformat}\033[0m" ;; + warn) + [[ $loglevel -le 2 ]] && echo -e "\033[33m${logformat}\033[0m" ;; + error) + [[ $loglevel -le 3 ]] && echo -e "\033[31m${logformat}\033[0m" ;; + fatal) + [[ $loglevel -le 4 ]] && echo -e "\033[31m${logformat}\033[0m" && exit 1; ;; + + esac + } +} + diff --git a/start_bootstrapper_container.sh b/start_bootstrapper_container.sh index 7b1cf4a..bdae466 100755 --- a/start_bootstrapper_container.sh +++ b/start_bootstrapper_container.sh @@ -21,12 +21,15 @@ if [[ $BSROOT != /* ]]; then exit 1 fi -if [[ -e "$BSROOT/html/static/CentOS7/CentOS-7-x86_64-Everything-1611.iso" ]]; then +source $BSROOT/load_yaml.sh +load_yaml $BSROOT/config/cluster-desc.yml cluster_desc_ + +if [[ -e "$BSROOT/html/static/CentOS7/CentOS-7-x86_64-Everything-${cluster_desc_centos_version##*.}.iso" ]]; then if [[ ! -d "$BSROOT/html/static/CentOS7/dvd_content" ]]; then mkdir -p $BSROOT/html/static/CentOS7/dvd_content fi if [[ ! -f "$BSROOT/html/static/CentOS7/dvd_content/.treeinfo" ]]; then - sudo mount -t iso9660 -o loop $BSROOT/html/static/CentOS7/CentOS-7-x86_64-Everything-1611.iso $BSROOT/html/static/CentOS7/dvd_content || { echo "Mount iso failed"; exit 1; } + sudo mount -t iso9660 -o loop $BSROOT/html/static/CentOS7/CentOS-7-x86_64-Everything-${cluster_desc_centos_version##*.}.iso $BSROOT/html/static/CentOS7/dvd_content || { echo "Mount iso failed"; exit 1; } fi fi @@ -48,13 +51,11 @@ docker run -d \ --privileged \ -v /var/run/docker.sock:/var/run/docker.sock \ -v $BSROOT:/bsroot \ - bootstrapper || { echo "Failed"; exit -1; } + bootstrapper ${cluster_desc_start_pxe} || { log fatal "Failed"; exit -1; } # Sleep 3 seconds, waitting for registry started. sleep 3 -source $BSROOT/load_yaml.sh -load_yaml $BSROOT/config/cluster-desc.yml cluster_desc_ for DOCKER_IMAGE in $(set | grep '^cluster_desc_images_' | grep -o '".*"' | sed 's/"//g'); do DOCKER_TAR_FILE=$BSROOT/$(echo ${DOCKER_IMAGE}.tar | sed "s/:/_/g" |awk -F'/' '{print $NF}')