-
Notifications
You must be signed in to change notification settings - Fork 7
/
bootkube-up
executable file
·259 lines (231 loc) · 11.3 KB
/
bootkube-up
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
#!/bin/bash
# Copyright 2017 The Bootkube-CI Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
### Declare colors to use during the running of this script:
declare -r GREEN="\033[0;32m"
declare -r RED="\033[0;31m"
declare -r YELLOW="\033[0;33m"
function echo_green {
echo -e "${GREEN}$1"; tput sgr0
}
function echo_red {
echo -e "${RED}$1"; tput sgr0
}
function echo_yellow {
echo -e "${YELLOW}$1"; tput sgr0
}
### Add IP Address of Hosts in /etc/hosts:
HOST_IFACE=$(ip route | grep "^default" | head -1 | awk '{ print $5 }')
LOCAL_IP=$(ip addr | awk "/inet/ && /${HOST_IFACE}/{sub(/\/.*$/,\"\",\$2); print \$2}")
cat << EOF | sudo tee -a /etc/hosts
${LOCAL_IP} $(hostname)
EOF
### NEW INSTALLATIONS:
echo_green "\nPhase I: Installing system prerequisites:"
pkg="build-essential curl docker.io ethtool git lldpd socat traceroute vim"
for pkg in $pkg; do
if sudo dpkg --get-selections | grep -q "^$pkg[[:space:]]*install$" >/dev/null; then
echo -e "$pkg is already installed"
else
sudo apt-get update && sudo apt-get -qq install $pkg
echo "Successfully installed $pkg"
fi
done
### PREPARE THE ENVIRONMENT:
source .bootkube_env
echo "export KUBE_MASTER=$KUBE_IP" >> .bootkube_env
echo_green "\nPhase II: Using the following variables during the deployment:"
printf "PROXY_ENABLED: $PROXY_ENABLED
BOOTKUBE_DIR: $BOOTKUBE_DIR
TMPDIR: $TMPDIR
CNI_VERSION: $CNI_VERSION
HELM_VERSION: $HELM_VERSION
BOOTKUBE_VERSION: $BOOTKUBE_VERSION
KUBERNETES_VERSION: $KUBERNETES_VERSION
KUBE_SDN: $KUBE_SDN
KUBE_POD_CIDR: $KUBE_POD_CIDR
KUBE_SVC_CIDR: $KUBE_SVC_CIDR
ETCD_SVC_ADDR: $ETCD_SVC_ADDR
ETCD_SVC_PORT: $ETCD_SVC_PORT
ETCD_SVC_PEER: $ETCD_SVC_PEER
HOST_SUB_CIDR: $HOST_SUB_CIDR
KUBE_HW: $KUBE_HW
KUBE_DNS_API: $KUBE_DNS_API
NSERVER01: $NSERVER01
NSERVER02: $NSERVER02
NSERVER03: $NSERVER03
NSEARCH01: $NSEARCH01
NSEARCH02: $NSEARCH02
KUBE_ARCH: $KUBE_ARCH
KUBE_IMAGE: $KUBE_IMAGE
KUBE_IP: $KUBE_IP
KUBE_MASTER: $KUBE_MASTER \n \n"
### PREPARE: /etc/resolv.conf
echo_green "\nPhase III: Preparing system DNS:"
sudo cp /etc/resolv.conf $BOOTKUBE_DIR/bootkube-ci/backups/
### PREPARE: /etc/resolv.conf
sudo -E bash -c "cat <<EOF > /etc/resolvconf/resolv.conf.d/head
nameserver $NSERVER01
EOF"
sudo -E bash -c "cat <<EOF > /etc/resolvconf/resolv.conf.d/base
search kubernetes $KUBE_DNS_API $NSEARCH01 $NSEARCH02
EOF"
sudo resolvconf -u
### PREPARE: /etc/hosts with idempotency (hostess):
### DOWNLOAD: bootkube
if [ ! -e /usr/local/bin/hostess ]; then
sudo wget -O /usr/local/bin/hostess https://github.com/cbednarski/hostess/releases/download/v0.2.0/hostess_linux_$KUBE_ARCH
sudo chmod +x /usr/local/bin/hostess
fi
sudo hostess add $KUBE_DNS_API $KUBE_MASTER
sudo hostess add kubernetes $KUBE_MASTER
sudo hostess fix
### PREPARE: /etc/systemd/system/kubelet.service
echo_green "\nPhase VI: Deploying the Kubelet systemd unit:"
printf "Kubelet deployed to host:\n"
echo "[Unit]
Description=Kubernetes Kubelet
Documentation=https://kubernetes.io/docs/admin/kubelet/
[Service]
EnvironmentFile=/etc/environment
ExecStartPre=/bin/mkdir -p /etc/kubernetes
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
ExecStartPre=/bin/mkdir -p /var/lib/cni
ExecStart=/usr/local/bin/kubelet \\
--allow-privileged \\
--cloud-provider= \\
--cluster_domain=cluster.local \\
--cluster_dns='$NSERVER01','$NSERVER02','$NSERVER03' \\
--cni-conf-dir=/etc/cni/net.d \\
--exit-on-lock-contention \\
--hostname-override='$KUBE_IP' \\
--kubeconfig=/etc/kubernetes/kubeconfig \\
--lock-file=/var/run/lock/kubelet.lock \\
--minimum-container-ttl-duration=3m0s \\
--node-labels=node-role.kubernetes.io/master \\
--node-ip='$KUBE_IP' \\
--network-plugin=cni \\
--pod-manifest-path=/etc/kubernetes/manifests \\
--register-with-taints=node-role.kubernetes.io/master=:NoSchedule \\
--require-kubeconfig \\
--v=2
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/kubelet.service
### DOWNLOAD: bootkube
echo_green "\nPhase V: Downloading Bootkube required binaries:"
mkdir -p $TMPDIR
### DOWNLOAD: bootkube
if [[ ! -e '$TMPDIR'/'$BOOTKUBE_VERSION'-bootkube.tgz && ! -e /usr/local/bin/bootkube ]]; then
wget -O $TMPDIR/$BOOTKUBE_VERSION-bootkube.tgz https://github.com/kubernetes-incubator/bootkube/releases/download/$BOOTKUBE_VERSION/bootkube.tar.gz
tar zxvf $TMPDIR/$BOOTKUBE_VERSION-bootkube.tgz -C $TMPDIR/
mv $TMPDIR/bin/ $TMPDIR/$BOOTKUBE_VERSION-bootkube-$KUBE_ARCH
chmod +x $TMPDIR/$BOOTKUBE_VERSION-bootkube-$KUBE_ARCH/linux/bootkube
sudo cp $TMPDIR/$BOOTKUBE_VERSION-bootkube-$KUBE_ARCH/linux/bootkube /usr/local/bin/
fi
### DOWNLOAD: kubectl
if [[ ! -e '$TMPDIR'/'$KUBERNETES_VERSION'-kubectl-$KUBE_ARCH && ! -e /usr/local/bin/kubectl ]]; then
wget -O $TMPDIR/$KUBERNETES_VERSION-kubectl-$KUBE_ARCH http://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/$KUBE_ARCH/kubectl
chmod +x $TMPDIR/$KUBERNETES_VERSION-kubectl-$KUBE_ARCH
sudo cp $TMPDIR/$KUBERNETES_VERSION-kubectl-$KUBE_ARCH /usr/local/bin/kubectl
fi
### DOWNLOAD: kubelet
if [[ ! -e '$TMPDIR'/'$KUBERNETES_VERSION'-kubelet-$KUBE_ARCH && ! -e /usr/local/bin/kubelet ]]; then
wget -O $TMPDIR/$KUBERNETES_VERSION-kubelet-$KUBE_ARCH http://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/$KUBE_ARCH/kubelet
chmod +x $TMPDIR/$KUBERNETES_VERSION-kubelet-$KUBE_ARCH
sudo cp $TMPDIR/$KUBERNETES_VERSION-kubelet-$KUBE_ARCH /usr/local/bin/kubelet
fi
### DOWNLOAD: cni
if [[ ! -e '$TMPDIR'/'$CNI_VERSION'-cni-$KUBE_ARCH.tgz && ! -e /opt/cni/bin ]]; then
wget -O $TMPDIR/$CNI_VERSION-cni-$KUBE_ARCH.tgz https://github.com/containernetworking/cni/releases/download/$CNI_VERSION/cni-$KUBE_ARCH-$CNI_VERSION.tgz
sudo mkdir -p /opt/cni/bin
sudo tar -xf $TMPDIR/$CNI_VERSION-cni-$KUBE_ARCH.tgz -C /opt/cni/bin/
fi
### CLEANUP:
cd $BOOTKUBE_DIR
sudo rm -rf $TMPDIR/linux-$KUBE_ARCH
echo_green "\nComplete!"
### RENDER ASSETS:
echo_green "\nPhase VI: Running Bootkube to render the Kubernetes assets:"
bootkube render --asset-dir=$BOOTKUBE_DIR/.bootkube --experimental-self-hosted-etcd --etcd-servers=http://10.3.0.15:12379 --api-servers=https://$KUBE_DNS_API:$KUBE_SVC_PORT,https://$KUBE_IP:$KUBE_SVC_PORT --pod-cidr=$KUBE_POD_CIDR --service-cidr=$KUBE_SVC_CIDR
sudo rm -rf $BOOTKUBE_DIR/.bootkube/manifests/kube-flannel*
if [ $KUBE_SDN = "kube-router" ]; then
echo_yellow "\nSDN variable is set to kube-router. Proceeding to remove kube-proxy manifests for Bootkube!" && sudo rm -rf $BOOTKUBE_DIR/.bootkube/manifests/kube-proxy.yaml
else
echo_green "\nSDN is set to $KUBE_SDN. No additional changes are required for $KUBE_SDN!"
fi
### DEPLOY KUBERNETES SELF-HOSTED CLUSTER:
echo_green "\nPhase VII: Preparing the environment for Kubernetes to run for the first time:"
sudo systemctl daemon-reload
sudo systemctl enable kubelet.service
sudo systemctl restart kubelet.service
sudo cp $BOOTKUBE_DIR/.bootkube/auth/kubeconfig /etc/kubernetes/
sudo cp $BOOTKUBE_DIR/.bootkube/tls/ca.crt /etc/kubernetes/ca.crt
### Ensuring that kubectl has a proper configuration file in the $USER/.kube directory, and backing up the old config file if required:
if [[ ! -e ~/.kube/config ]]; then
echo_green "Copying Kubernetes config to $HOME/.kube/config" && sudo mkdir -p ~/.kube && sudo cp /etc/kubernetes/kubeconfig ~/.kube/config
else
echo_yellow "Moving old $HOME/.kube/config to $HOME/.kube/config.backup" && sudo cp ~/.kube/config ~/.kube/config.backup && sudo cp /etc/kubernetes/kubeconfig ~/.kube/config
fi
sudo chmod 644 ~/.kube/config
### Enabling Kubernetes CronJob Support:
echo_green "\nPhase VIII: Running conditional options to enable Kubernetes CronJobs. Below are enabled options for the API Server:"
awk '/etcd-servers/{print;print " - --runtime-config=batch/v2alpha1=true";next}1' $BOOTKUBE_DIR/.bootkube/manifests/kube-apiserver.yaml > $BOOTKUBE_DIR/.bootkube/manifests/kube-apiserver-custom.yaml
rm -rf $BOOTKUBE_DIR/.bootkube/manifests/kube-apiserver.yaml
cat $BOOTKUBE_DIR/.bootkube/manifests/kube-apiserver-custom.yaml | grep " - --"
echo_green "\nPhase IX: Running Bootkube start to bring up the temporary Kubernetes self-hosted control plane:"
nohup sudo bash -c 'bootkube start --asset-dir='$BOOTKUBE_DIR'/.bootkube' >$BOOTKUBE_DIR/bootkube-ci/log/bootkube-start.log 2>&1 &
### WAIT FOR KUBERNETES ENVIRONMENT TO COME UP:
echo -e -n "Waiting for master components to start..."
while true; do
running_count=$(sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig get pods -n kube-system --no-headers 2>/dev/null | grep "Running" | grep "boot" | wc -l)
### Expect 4 bootstrap components for a truly "Running" state: etcd, apiserver, controller, and scheduler:
if [ "$running_count" -ge 4 ]; then
break
fi
echo -n "."
sleep 1
done
echo_green "SUCCESS"
echo_green "Cluster created!"
echo ""
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig cluster-info
sleep 10
### WAIT FOR KUBERNETES API TO COME UP CLEANLY, THEN APPLY FOLLOWING LABELS AND MANIFESTS:
echo_green "\nPhase X: Cluster created, and now deploying requested SDN along with additional labels:"
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig label node --all node-role.kubernetes.io/$KUBE_SDN-node=true --overwrite
sed "s?KUBE_POD_CIDR?${KUBE_POD_CIDR}?g; s?ETCD_SVC_ADDR?${ETCD_SVC_ADDR}?g; s?ETCD_SVC_PORT?${ETCD_SVC_PORT}?g; s?ETCD_SVC_PEER?${ETCD_SVC_PEER}?g; s?KUBE_SVC_PORT?${KUBE_SVC_PORT}?g" $BOOTKUBE_DIR/bootkube-ci/deploy-sdn/$KUBE_SDN/calico-${KUBE_SDN_VERSION}-v1.7.5.yaml > $BOOTKUBE_DIR/bootkube-ci/deploy-sdn/$KUBE_SDN/calico-${KUBE_SDN_VERSION}-bootkube-v1.7.5.yaml
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig apply -f $BOOTKUBE_DIR/bootkube-ci/deploy-sdn/$KUBE_SDN/calico-${KUBE_SDN_VERSION}-bootkube-v1.7.5.yaml
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig taint nodes --all node-role.kubernetes.io/master- || true
echo_green "\nPhase XI: Writing Kubernetes environment cluster-info logs:"
### WAIT FOR KUBERNETES ENVIRONMENT TO COME UP:
echo -e -n "Waiting for all services to be in a running state..."
while true; do
creating_count=$(sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig get pods -n kube-system --no-headers 2>/dev/null | egrep "i(Pending|ContainerCreating)" | grep "kube" | wc -l)
### Expect all components to be out of a "Pending" state before collecting log data (this includes CrashLoopBackOff states):
if [ "$creating_count" -eq 0 ]; then
break
fi
echo -n "."
sleep 1
done
export CLUSTER_TIMESTAMP="$(date +%s)"
sudo kubectl --kubeconfig=/etc/kubernetes/kubeconfig cluster-info dump --all-namespaces --output-directory "$BOOTKUBE_DIR/bootkube-ci/log/cluster-info-$CLUSTER_TIMESTAMP"
echo_yellow "\nCluster logs can be found in $BOOTKUBE_DIR/bootkube-ci/log/cluster-info-$CLUSTER_TIMESTAMP"
echo_green "\nCOMPLETE!\n"