From 8965a9c7c8dd9cb5b0c881a77578a103ed672d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Sat, 19 Aug 2017 11:43:12 +0200 Subject: [PATCH 1/2] Upgraded Kubernetes to 1.7.3 and Calico to 2.4.1 * Upgraded Kubernetes from 1.5.4 to 1.7.3 * Upgraded Calico from 0.23.0 to 2.4.1 * Upgraded kube-dns from 1.9 to 1.14.4 * Upgraded dns-autoscaler from 1.0.0 to 1.1.2 * Upgraded heapster from 1.2.0 to 1.4.1 * Upgraded kube-dashboard from 1.5.0 to 1.6.3 * Changed from using etcd2 to etcd version 3 * Changed Calico to the default network solution * Changed POD_NETWORK from 10.2.0.0/16 to 192.168.0.0/16 as Calico does not respect CALICO_IPV4POOL_CIDR * Changed Vagrant memory limits for the controller, increased to 2Gb * Removed Kubelet usage of deprecated flag --api-servers * Removed Kubelet usage of deprecated flag --register-schedulable * Added a kubeconfig to make up for the removed flags above * Added possibility to modify how many cpu cores Vagrant assigns the hosts --- multi-node/generic/controller-install.sh | 467 +++++++++++++++++----- multi-node/generic/worker-install.sh | 66 +-- multi-node/vagrant/Vagrantfile | 25 +- multi-node/vagrant/etcd-cloud-config.yaml | 19 - 4 files changed, 408 insertions(+), 169 deletions(-) delete mode 100644 multi-node/vagrant/etcd-cloud-config.yaml diff --git a/multi-node/generic/controller-install.sh b/multi-node/generic/controller-install.sh index 4849eec52b..7fd0330e69 100644 --- a/multi-node/generic/controller-install.sh +++ b/multi-node/generic/controller-install.sh @@ -5,7 +5,7 @@ set -e export ETCD_ENDPOINTS= # Specify the version (vX.Y.Z) of Kubernetes assets to deploy -export K8S_VER=v1.5.4_coreos.0 +export K8S_VER=v1.7.3_coreos.0 # Hyperkube image repository to use. export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube @@ -13,7 +13,7 @@ export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube # The CIDR network to use for pod IPs. # Each pod launched in the cluster will be assigned an IP out of this range. # Each node will be configured such that these IPs will be routable using the flannel overlay network. -export POD_NETWORK=10.2.0.0/16 +export POD_NETWORK=192.168.0.0/16 # The CIDR network to use for service cluster IPs. # Each service will be assigned a cluster IP out of this range. @@ -31,7 +31,7 @@ export K8S_SERVICE_IP=10.3.0.1 export DNS_SERVICE_IP=10.3.0.10 # Whether to use Calico for Kubernetes network policy. -export USE_CALICO=false +export USE_CALICO=true # Determines the container runtime for kubernetes to use. Accepts 'docker' or 'rkt'. export CONTAINER_RUNTIME=docker @@ -96,6 +96,7 @@ function init_templates { local uuid_file="/var/run/kubelet-pod.uuid" if [ ! -f $TEMPLATE ]; then echo "TEMPLATE: $TEMPLATE" + mkdir -p /etc/kubernetes/cni/net.d mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE [Service] @@ -110,6 +111,8 @@ Environment="RKT_RUN_ARGS=--uuid-file-save=${uuid_file} \ --mount volume=var-lib-rkt,target=/var/lib/rkt \ --volume stage,kind=host,source=/tmp \ --mount volume=stage,target=/tmp \ + --volume etc-cni-net,kind=host,source=/etc/kubernetes/cni/net.d \ + --mount volume=etc-cni-net,target=/etc/cni/net.d \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ ${CALICO_OPTS}" @@ -118,8 +121,8 @@ ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/mkdir -p /var/log/containers ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file} ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --api-servers=http://127.0.0.1:8080 \ - --register-schedulable=false \ + --kubeconfig /etc/kubernetes/master-kubeconfig.yaml \ + --require-kubeconfig \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --network-plugin=cni \ --container-runtime=${CONTAINER_RUNTIME} \ @@ -157,6 +160,33 @@ exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@" EOF fi + local TEMPLATE=/etc/kubernetes/master-kubeconfig.yaml + if [ ! -f $TEMPLATE ]; then + echo "TEMPLATE: $TEMPLATE" + mkdir -p $(dirname $TEMPLATE) + cat << EOF > $TEMPLATE +apiVersion: v1 +kind: Config +preferences: {} +current-context: vagrant-multi +clusters: +- cluster: + certificate-authority: /etc/kubernetes/ssl/ca.pem + server: http://127.0.0.1:8080 + name: vagrant-multi-cluster +contexts: +- context: + cluster: vagrant-multi-cluster + namespace: default + user: kubelet + name: vagrant-multi +users: +- name: kubelet + user: + client-certificate: /etc/kubernetes/ssl/apiserver.pem + client-key: /etc/kubernetes/ssl/apiserver-key.pem +EOF + fi local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then @@ -388,6 +418,27 @@ EOF echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + +--- + apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -396,7 +447,12 @@ metadata: labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. strategy: rollingUpdate: maxSurge: 10% @@ -410,12 +466,23 @@ spec: k8s-app: kube-dns annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' spec: + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + volumes: + - name: kube-dns-config + configMap: + name: kube-dns + optional: true containers: - name: kubedns - image: gcr.io/google_containers/kubedns-amd64:1.9 + image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.4 resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. limits: memory: 170Mi requests: @@ -423,8 +490,8 @@ spec: memory: 70Mi livenessProbe: httpGet: - path: /healthz-kubedns - port: 8080 + path: /healthcheck/kubedns + port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 @@ -435,14 +502,14 @@ spec: path: /readiness port: 8081 scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. initialDelaySeconds: 3 timeoutSeconds: 5 args: - --domain=cluster.local. - --dns-port=10053 - - --config-map=kube-dns - # This should be set to v=2 only after the new image (cut from 1.5) has - # been released, otherwise we will flood the logs. + - --config-dir=/kube-dns-config - --v=2 env: - name: PROMETHEUS_PORT @@ -457,22 +524,32 @@ spec: - containerPort: 10055 name: metrics protocol: TCP + volumeMounts: + - name: kube-dns-config + mountPath: /kube-dns-config - name: dnsmasq - image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4 + image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4 livenessProbe: httpGet: - path: /healthz-dnsmasq - port: 8080 + path: /healthcheck/dnsmasq + port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k - --cache-size=1000 - - --no-resolv - - --server=127.0.0.1#10053 - --log-facility=- + - --server=/cluster.local/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 ports: - containerPort: 53 name: dns @@ -484,9 +561,12 @@ spec: resources: requests: cpu: 150m - memory: 10Mi - - name: dnsmasq-metrics - image: gcr.io/google_containers/dnsmasq-metrics-amd64:1.0 + memory: 20Mi + volumeMounts: + - name: kube-dns-config + mountPath: /etc/k8s/dns/dnsmasq-nanny + - name: sidecar + image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.4 livenessProbe: httpGet: path: /metrics @@ -499,33 +579,18 @@ spec: args: - --v=2 - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A ports: - containerPort: 10054 name: metrics protocol: TCP resources: requests: - memory: 10Mi - - name: healthz - image: gcr.io/google_containers/exechealthz-amd64:1.2 - resources: - limits: - memory: 50Mi - requests: + memory: 20Mi cpu: 10m - memory: 50Mi - args: - - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null - - --url=/healthz-dnsmasq - - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 >/dev/null - - --url=/healthz-kubedns - - --port=8080 - - --quiet - ports: - - containerPort: 8080 - protocol: TCP - dnsPolicy: Default - + dnsPolicy: Default # Don't use cluster DNS. + serviceAccountName: kube-dns EOF fi @@ -534,6 +599,16 @@ EOF echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE +kind: ServiceAccount +apiVersion: v1 +metadata: + name: kube-dns-autoscaler + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + +--- + apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -542,6 +617,7 @@ metadata: labels: k8s-app: kube-dns-autoscaler kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile spec: template: metadata: @@ -549,11 +625,10 @@ spec: k8s-app: kube-dns-autoscaler annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' spec: containers: - name: autoscaler - image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.0.0 + image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.2-r2 resources: requests: cpu: "20m" @@ -562,11 +637,17 @@ spec: - /cluster-proportional-autoscaler - --namespace=kube-system - --configmap=kube-dns-autoscaler - - --mode=linear + # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base - --target=Deployment/kube-dns - - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"min":1}} + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} - --logtostderr=true - --v=2 + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + serviceAccountName: kube-dns-autoscaler EOF fi @@ -583,6 +664,7 @@ metadata: labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "KubeDNS" spec: selector: @@ -603,32 +685,43 @@ EOF echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE +apiVersion: v1 +kind: ServiceAccount +metadata: + name: heapster + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + +--- + apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: heapster-v1.2.0 + name: heapster-v1.4.1 namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" - version: v1.2.0 + addonmanager.kubernetes.io/mode: Reconcile + version: v1.4.1 spec: replicas: 1 selector: matchLabels: k8s-app: heapster - version: v1.2.0 + version: v1.4.1 template: metadata: labels: k8s-app: heapster - version: v1.2.0 + version: v1.4.1 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' spec: containers: - - image: gcr.io/google_containers/heapster:v1.2.0 + - image: gcr.io/google_containers/heapster-amd64:v1.4.1 name: heapster livenessProbe: httpGet: @@ -640,7 +733,7 @@ spec: command: - /heapster - --source=kubernetes.summary_api:'' - - image: gcr.io/google_containers/addon-resizer:1.6 + - image: gcr.io/google_containers/addon-resizer:1.7 name: heapster-nanny resources: limits: @@ -664,11 +757,14 @@ spec: - --extra-cpu=4m - --memory=200Mi - --extra-memory=4Mi - - --threshold=5 - - --deployment=heapster-v1.2.0 + - --deployment=heapster-v1.4.1 - --container=heapster - --poll-period=300000 - --estimator=exponential + serviceAccountName: heapster + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" EOF fi @@ -684,6 +780,7 @@ metadata: namespace: kube-system labels: kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "Heapster" spec: ports: @@ -707,6 +804,7 @@ metadata: labels: k8s-app: kubernetes-dashboard kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: @@ -717,19 +815,27 @@ spec: k8s-app: kubernetes-dashboard annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' spec: containers: - name: kubernetes-dashboard - image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.0 + image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.3 + args: + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + # https://github.com/kubernetes/dashboard/issues/2180 + # Should be resolved with dashboard >= 1.6.3 and kubernetes >= 1.7.4 + # and the line below can be removed + - --heapster-host=http://heapster.kube-system:80 resources: # keep request = limit to keep this container in guaranteed class limits: cpu: 100m - memory: 50Mi + memory: 300Mi requests: cpu: 100m - memory: 50Mi + memory: 100Mi ports: - containerPort: 9090 livenessProbe: @@ -738,6 +844,9 @@ spec: port: 9090 initialDelaySeconds: 30 timeoutSeconds: 30 + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" EOF fi @@ -754,6 +863,7 @@ metadata: labels: k8s-app: kubernetes-dashboard kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile spec: selector: k8s-app: kubernetes-dashboard @@ -826,41 +936,95 @@ EOF echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE +# Calico Version v2.4.1 +# https://docs.projectcalico.org/v2.4/releases#v2.4.1 +# This manifest includes the following component versions: +# calico/node:v2.4.1 +# calico/cni:v1.10.0 +# calico/kube-policy-controller:v0.7.0 +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-policy-controller + namespace: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + +--- + # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap apiVersion: v1 metadata: - name: calico-config + name: calico-config namespace: kube-system data: # Configure this with the location of your etcd cluster. etcd_endpoints: "${ETCD_ENDPOINTS}" - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. + # Configure the Calico backend to use. + calico_backend: "bird" + + # The CNI network configuration to install on each node. cni_network_config: |- { - "name": "calico", - "type": "flannel", - "delegate": { - "type": "calico", - "etcd_endpoints": "__ETCD_ENDPOINTS__", - "log_level": "info", - "policy": { - "type": "k8s", - "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", - "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" - }, - "kubernetes": { - "kubeconfig": "/etc/kubernetes/cni/net.d/__KUBECONFIG_FILENAME__" - } + "name": "k8s-pod-network", + "cniVersion": "0.1.0", + "type": "calico", + "etcd_endpoints": "__ETCD_ENDPOINTS__", + "etcd_key_file": "__ETCD_KEY_FILE__", + "etcd_cert_file": "__ETCD_CERT_FILE__", + "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", + "log_level": "info", + "mtu": 1500, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s", + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" } } + # If you're using TLS enabled etcd uncomment the following. + # You must also populate the Secret below with these files. + etcd_ca: "" # "/calico-secrets/etcd-ca" + etcd_cert: "" # "/calico-secrets/etcd-cert" + etcd_key: "" # "/calico-secrets/etcd-key" + +--- + +# The following contains k8s Secrets for use with a TLS enabled etcd cluster. +# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: calico-etcd-secrets + namespace: kube-system +data: + # Populate the following files with etcd TLS configuration if desired, but leave blank if + # not using TLS for etcd. + # This self-hosted install expects three files with the following names. The values + # should be base64 encoded strings of the entire contents of each file. + # etcd-key: null + # etcd-cert: null + # etcd-ca: null + --- # This manifest installs the calico/node container, as well -# as the Calico CNI plugins and network config on +# as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: extensions/v1beta1 @@ -884,12 +1048,13 @@ spec: {"key":"CriticalAddonsOnly", "operator":"Exists"}] spec: hostNetwork: true + serviceAccountName: calico-node containers: - # Runs calico/node container on each Kubernetes node. This + # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v0.23.0 + image: quay.io/calico/node:v2.4.1 env: # The location of the Calico etcd cluster. - name: ETCD_ENDPOINTS @@ -897,36 +1062,92 @@ spec: configMapKeyRef: name: calico-config key: etcd_endpoints - # Choose the backend to use. + # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND - value: "none" - # Disable file logging so 'kubectl logs' works. + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" - - name: NO_DEFAULT_POOLS + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Configure the IP Pool from which Pod IPs will be chosen. + - name: CALICO_IPV4POOL_CIDR + value: "${POD_NETWORK}" + - name: CALICO_IPV4POOL_IPIP + value: "always" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + value: "1440" + - name: FELIX_HEALTHENABLED value: "true" + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_cert + # Auto-detect the BGP IP address. + - name: IP + value: "" securityContext: privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readiness + port: 9099 + periodSeconds: 10 volumeMounts: - mountPath: /lib/modules name: lib-modules - readOnly: false + readOnly: true - mountPath: /var/run/calico name: var-run-calico readOnly: false - - mountPath: /etc/resolv.conf - name: dns - readOnly: true + - mountPath: /calico-secrets + name: etcd-certs + - mountPath: /etc/cni/net.d + name: cni-net-dir # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v1.5.2 - imagePullPolicy: Always + image: quay.io/calico/cni:v1.10.0 command: ["/install-cni.sh"] env: - # CNI configuration filename - - name: CNI_CONF_NAME - value: "10-calico.conf" # The location of the Calico etcd cluster. - name: ETCD_ENDPOINTS valueFrom: @@ -944,6 +1165,8 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir + - mountPath: /calico-secrets + name: etcd-certs volumes: # Used by calico/node. - name: lib-modules @@ -959,42 +1182,46 @@ spec: - name: cni-net-dir hostPath: path: /etc/kubernetes/cni/net.d - - name: dns - hostPath: - path: /etc/resolv.conf + # Mount in the etcd TLS secrets. + - name: etcd-certs + secret: + secretName: calico-etcd-secrets --- # This manifest deploys the Calico policy controller on Kubernetes. # See https://github.com/projectcalico/k8s-policy apiVersion: extensions/v1beta1 -kind: ReplicaSet +kind: Deployment metadata: name: calico-policy-controller namespace: kube-system labels: k8s-app: calico-policy + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: | + [{"key": "dedicated", "value": "master", "effect": "NoSchedule" }, + {"key":"CriticalAddonsOnly", "operator":"Exists"}] spec: # The policy controller can only have a single active instance. replicas: 1 + strategy: + type: Recreate template: metadata: name: calico-policy-controller namespace: kube-system labels: k8s-app: calico-policy - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: | - [{"key": "dedicated", "value": "master", "effect": "NoSchedule" }, - {"key":"CriticalAddonsOnly", "operator":"Exists"}] spec: # The policy controller must run in the host network namespace so that # it isn't governed by policy that would prevent it from working. hostNetwork: true + serviceAccountName: calico-policy-controller containers: - name: calico-policy-controller - image: calico/kube-policy-controller:v0.4.0 + image: quay.io/calico/kube-policy-controller:v0.7.0 env: # The location of the Calico etcd cluster. - name: ETCD_ENDPOINTS @@ -1002,15 +1229,42 @@ spec: configMapKeyRef: name: calico-config key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_cert # The location of the Kubernetes API. Use the default Kubernetes # service for API access. - name: K8S_API value: "https://kubernetes.default:443" - # Since we're running in the host namespace and might not have KubeDNS + # Since we're running in the host namespace and might not have KubeDNS # access, configure the container's /etc/hosts to resolve # kubernetes.default to the correct service clusterIP. - name: CONFIGURE_ETC_HOSTS value: "true" + volumeMounts: + # Mount in the etcd TLS secrets. + - mountPath: /calico-secrets + name: etcd-certs + volumes: + # Mount in the etcd TLS secrets. + - name: etcd-certs + secret: + secretName: calico-etcd-secrets EOF fi } @@ -1024,15 +1278,12 @@ function start_addons { echo echo "K8S: DNS addon" - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dns-de.yaml)" "http://127.0.0.1:8080/apis/extensions/v1beta1/namespaces/kube-system/deployments" > /dev/null - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dns-svc.yaml)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services" > /dev/null - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dns-autoscaler-de.yaml)" "http://127.0.0.1:8080/apis/extensions/v1beta1/namespaces/kube-system/deployments" > /dev/null + docker run --rm --net=host -v /srv/kubernetes/manifests:/host/manifests $HYPERKUBE_IMAGE_REPO:$K8S_VER /hyperkube kubectl apply -f /host/manifests/kube-dns-de.yaml -f /host/manifests/kube-dns-svc.yaml\ + -f /host/manifests/kube-dns-autoscaler-de.yaml echo "K8S: Heapster addon" - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/heapster-de.yaml)" "http://127.0.0.1:8080/apis/extensions/v1beta1/namespaces/kube-system/deployments" > /dev/null - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/heapster-svc.yaml)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services" > /dev/null + docker run --rm --net=host -v /srv/kubernetes/manifests:/host/manifests $HYPERKUBE_IMAGE_REPO:$K8S_VER /hyperkube kubectl apply -f /host/manifests/heapster-de.yaml -f /host/manifests/heapster-svc.yaml echo "K8S: Dashboard addon" - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dashboard-de.yaml)" "http://127.0.0.1:8080/apis/extensions/v1beta1/namespaces/kube-system/deployments" > /dev/null - curl --silent -H "Content-Type: application/yaml" -XPOST -d"$(cat /srv/kubernetes/manifests/kube-dashboard-svc.yaml)" "http://127.0.0.1:8080/api/v1/namespaces/kube-system/services" > /dev/null + docker run --rm --net=host -v /srv/kubernetes/manifests:/host/manifests $HYPERKUBE_IMAGE_REPO:$K8S_VER /hyperkube kubectl apply -f /host/manifests/kube-dashboard-de.yaml -f /host/manifests/kube-dashboard-svc.yaml } function start_calico { diff --git a/multi-node/generic/worker-install.sh b/multi-node/generic/worker-install.sh index 060b72190b..308f9c3ec5 100644 --- a/multi-node/generic/worker-install.sh +++ b/multi-node/generic/worker-install.sh @@ -10,7 +10,7 @@ export ETCD_ENDPOINTS= export CONTROLLER_ENDPOINT= # Specify the version (vX.Y.Z) of Kubernetes assets to deploy -export K8S_VER=v1.5.4_coreos.0 +export K8S_VER=v1.7.3_coreos.0 # Hyperkube image repository to use. export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube @@ -18,14 +18,14 @@ export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube # The CIDR network to use for pod IPs. # Each pod launched in the cluster will be assigned an IP out of this range. # Each node will be configured such that these IPs will be routable using the flannel overlay network. -export POD_NETWORK=10.2.0.0/16 +export POD_NETWORK=192.168.0.0/16 # The IP address of the cluster DNS service. # This must be the same DNS_SERVICE_IP used when configuring the controller nodes. export DNS_SERVICE_IP=10.3.0.10 # Whether to use Calico for Kubernetes network policy. -export USE_CALICO=false +export USE_CALICO=true # Determines the container runtime for kubernetes to use. Accepts 'docker' or 'rkt'. export CONTAINER_RUNTIME=docker @@ -67,6 +67,7 @@ function init_templates { local uuid_file="/var/run/kubelet-pod.uuid" if [ ! -f $TEMPLATE ]; then echo "TEMPLATE: $TEMPLATE" + mkdir -p /etc/kubernetes/cni/net.d mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE [Service] @@ -81,6 +82,8 @@ Environment="RKT_RUN_ARGS=--uuid-file-save=${uuid_file} \ --mount volume=var-lib-rkt,target=/var/lib/rkt \ --volume stage,kind=host,source=/tmp \ --mount volume=stage,target=/tmp \ + --volume etc-cni-net,kind=host,source=/etc/kubernetes/cni/net.d \ + --mount volume=etc-cni-net,target=/etc/cni/net.d \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ ${CALICO_OPTS}" @@ -89,7 +92,8 @@ ExecStartPre=/usr/bin/mkdir -p /var/log/containers ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file} ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --api-servers=${CONTROLLER_ENDPOINT} \ + --kubeconfig /etc/kubernetes/worker-kubeconfig.yaml \ + --require-kubeconfig \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --network-plugin=cni \ --container-runtime=${CONTAINER_RUNTIME} \ @@ -101,7 +105,6 @@ ExecStart=/usr/lib/coreos/kubelet-wrapper \ --hostname-override=${ADVERTISE_IP} \ --cluster_dns=${DNS_SERVICE_IP} \ --cluster_domain=cluster.local \ - --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \ --tls-cert-file=/etc/kubernetes/ssl/worker.pem \ --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem ExecStop=-/usr/bin/rkt stop --uuid-file=${uuid_file} @@ -131,6 +134,34 @@ exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "\$@" EOF fi + local TEMPLATE=/etc/kubernetes/worker-kubeconfig.yaml + if [ ! -f $TEMPLATE ]; then + echo "TEMPLATE: $TEMPLATE" + mkdir -p $(dirname $TEMPLATE) + cat << EOF > $TEMPLATE +apiVersion: v1 +kind: Config +preferences: {} +current-context: vagrant-multi +clusters: +- cluster: + certificate-authority: /etc/kubernetes/ssl/ca.pem + server: ${CONTROLLER_ENDPOINT} + name: vagrant-multi-cluster +contexts: +- context: + cluster: vagrant-multi-cluster + namespace: default + user: kubelet + name: vagrant-multi +users: +- name: kubelet + user: + client-certificate: /etc/kubernetes/ssl/worker.pem + client-key: /etc/kubernetes/ssl/worker-key.pem +EOF + fi + local TEMPLATE=/etc/systemd/system/load-rkt-stage1.service if [ ${CONTAINER_RUNTIME} = "rkt" ] && [ ! -f $TEMPLATE ]; then echo "TEMPLATE: $TEMPLATE" @@ -171,31 +202,6 @@ RequiredBy=kubelet.service EOF fi - local TEMPLATE=/etc/kubernetes/worker-kubeconfig.yaml - if [ ! -f $TEMPLATE ]; then - echo "TEMPLATE: $TEMPLATE" - mkdir -p $(dirname $TEMPLATE) - cat << EOF > $TEMPLATE -apiVersion: v1 -kind: Config -clusters: -- name: local - cluster: - certificate-authority: /etc/kubernetes/ssl/ca.pem -users: -- name: kubelet - user: - client-certificate: /etc/kubernetes/ssl/worker.pem - client-key: /etc/kubernetes/ssl/worker-key.pem -contexts: -- context: - cluster: local - user: kubelet - name: kubelet-context -current-context: kubelet-context -EOF - fi - local TEMPLATE=/etc/kubernetes/manifests/kube-proxy.yaml if [ ! -f $TEMPLATE ]; then echo "TEMPLATE: $TEMPLATE" diff --git a/multi-node/vagrant/Vagrantfile b/multi-node/vagrant/Vagrantfile index feabf95c83..2dabf661a5 100644 --- a/multi-node/vagrant/Vagrantfile +++ b/multi-node/vagrant/Vagrantfile @@ -10,11 +10,12 @@ Vagrant.require_version ">= 1.6.0" $update_channel = "alpha" $controller_count = 1 -$controller_vm_memory = 1024 +$controller_vm_memory = 2048 $worker_count = 1 $worker_vm_memory = 1024 $etcd_count = 1 $etcd_vm_memory = 512 +$num_cpus = 1 CONFIG = File.expand_path("config.rb") if File.exist?(CONFIG) @@ -27,8 +28,6 @@ end CONTROLLER_CLUSTER_IP="10.3.0.1" -ETCD_CLOUD_CONFIG_PATH = File.expand_path("etcd-cloud-config.yaml") - CONTROLLER_CLOUD_CONFIG_PATH = File.expand_path("../generic/controller-install.sh") WORKER_CLOUD_CONFIG_PATH = File.expand_path("../generic/worker-install.sh") @@ -97,19 +96,18 @@ Vagrant.configure("2") do |config| end config.vm.provider :virtualbox do |vb| - vb.cpus = 1 + vb.cpus = $num_cpus vb.gui = false end (1..$etcd_count).each do |i| config.vm.define vm_name = "e%d" % i do |etcd| - data = YAML.load(IO.readlines(ETCD_CLOUD_CONFIG_PATH)[1..-1].join) - data['coreos']['etcd2']['initial-cluster'] = initial_etcd_cluster - data['coreos']['etcd2']['name'] = vm_name - etcd_config_file = Tempfile.new('etcd_config', :binmode => true) - etcd_config_file.write("#cloud-config\n#{data.to_yaml}") - etcd_config_file.close + drop_in = Tempfile.new('drop_in', :binmode => true) + drop_in.write("[Service]\n") + drop_in.write("Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379\n") + drop_in.write("Environment=ETCD_ADVERTISE_CLIENT_URLS=http://#{etcdIP(i)}:2379\n") + drop_in.close etcd.vm.hostname = vm_name @@ -125,8 +123,11 @@ Vagrant.configure("2") do |config| etcd.vm.network :private_network, ip: etcdIP(i) - etcd.vm.provision :file, :source => etcd_config_file.path, :destination => "/tmp/vagrantfile-user-data" - etcd.vm.provision :shell, :inline => "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", :privileged => true + etcd.vm.provision :file, :source => drop_in, :destination => "/tmp/etcd-member.conf" + etcd.vm.provision :shell, :inline => "mkdir -p /etc/systemd/system/etcd-member.service.d &&\ + mv /tmp/etcd-member.conf /etc/systemd/system/etcd-member.service.d/40-listen-address.conf &&\ + systemctl enable etcd-member &&\ + systemctl start etcd-member", :privileged => true end end diff --git a/multi-node/vagrant/etcd-cloud-config.yaml b/multi-node/vagrant/etcd-cloud-config.yaml deleted file mode 100644 index bc4597280c..0000000000 --- a/multi-node/vagrant/etcd-cloud-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -#cloud-config - -coreos: - update: - reboot-strategy: "off" - - etcd2: - name: {{ETCD_NODE_NAME}} - initial-cluster: {{ETCD_INITIAL_CLUSTER}} - advertise-client-urls: http://$private_ipv4:2379 - listen-client-urls: http://0.0.0.0:2379 - initial-advertise-peer-urls: http://$private_ipv4:2380 - listen-peer-urls: http://$private_ipv4:2380 - - units: - - - name: etcd2.service - command: start - From 3e0986915c1e6123c409aab7418656cb83e89ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Thu, 24 Aug 2017 10:48:39 +0200 Subject: [PATCH 2/2] Upgraded Kubernetes & Calico, reverted configuration change * Upgraded Kubernetes from 1.7.3 to 1.7.4 * Upgraded Calico from 2.4.1 to 2.5.0 * Reverted configuration change for POD_NETWORK as my understanding of how Calico treats CALICO_IPV4POOL_CIDR was incorrect. CALICO_IPV4POOL_CIDR is used to create an initial pool and cannot later be changed and used as a way to assign a new pool without first removing the existing pool(s). Thanks @tmjd for clarifying this! --- multi-node/generic/controller-install.sh | 12 ++++++------ multi-node/generic/worker-install.sh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/multi-node/generic/controller-install.sh b/multi-node/generic/controller-install.sh index 7fd0330e69..cf8c77845c 100644 --- a/multi-node/generic/controller-install.sh +++ b/multi-node/generic/controller-install.sh @@ -5,7 +5,7 @@ set -e export ETCD_ENDPOINTS= # Specify the version (vX.Y.Z) of Kubernetes assets to deploy -export K8S_VER=v1.7.3_coreos.0 +export K8S_VER=v1.7.4_coreos.0 # Hyperkube image repository to use. export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube @@ -13,7 +13,7 @@ export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube # The CIDR network to use for pod IPs. # Each pod launched in the cluster will be assigned an IP out of this range. # Each node will be configured such that these IPs will be routable using the flannel overlay network. -export POD_NETWORK=192.168.0.0/16 +export POD_NETWORK=10.2.0.0/16 # The CIDR network to use for service cluster IPs. # Each service will be assigned a cluster IP out of this range. @@ -936,10 +936,10 @@ EOF echo "TEMPLATE: $TEMPLATE" mkdir -p $(dirname $TEMPLATE) cat << EOF > $TEMPLATE -# Calico Version v2.4.1 -# https://docs.projectcalico.org/v2.4/releases#v2.4.1 +# Calico Version v2.5.0 +# https://docs.projectcalico.org/v2.4/releases#v2.5.0 # This manifest includes the following component versions: -# calico/node:v2.4.1 +# calico/node:v2.5.0 # calico/cni:v1.10.0 # calico/kube-policy-controller:v0.7.0 apiVersion: v1 @@ -1054,7 +1054,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v2.4.1 + image: quay.io/calico/node:v2.5.0 env: # The location of the Calico etcd cluster. - name: ETCD_ENDPOINTS diff --git a/multi-node/generic/worker-install.sh b/multi-node/generic/worker-install.sh index 308f9c3ec5..210feeefa5 100644 --- a/multi-node/generic/worker-install.sh +++ b/multi-node/generic/worker-install.sh @@ -10,7 +10,7 @@ export ETCD_ENDPOINTS= export CONTROLLER_ENDPOINT= # Specify the version (vX.Y.Z) of Kubernetes assets to deploy -export K8S_VER=v1.7.3_coreos.0 +export K8S_VER=v1.7.4_coreos.0 # Hyperkube image repository to use. export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube @@ -18,7 +18,7 @@ export HYPERKUBE_IMAGE_REPO=quay.io/coreos/hyperkube # The CIDR network to use for pod IPs. # Each pod launched in the cluster will be assigned an IP out of this range. # Each node will be configured such that these IPs will be routable using the flannel overlay network. -export POD_NETWORK=192.168.0.0/16 +export POD_NETWORK=10.2.0.0/16 # The IP address of the cluster DNS service. # This must be the same DNS_SERVICE_IP used when configuring the controller nodes.