deploy: Deploy glusterd2-cluster as individual StatefulSets

Using single-replica StatefulSets for each glusterd2 pod instead of a
single DaemonSet, allows setting up of and use of pre-known hostnames as
the listen address for glusterd2. The StatefulSets are pinned to
individual nodes.

Also, the glusterd2 pods are now deployed with 'emptyDir' volumes for
/var/lib/glusterd2 which allows persistence of peerid.

With the above 2 changes, glusterd2 pods survive pod restarts.

Fixes gluster#10, gluster#11

Signed-off-by: Kaushal M <kshlmster@gmail.com>

deploy/Vagrantfile

@@ -12,18 +12,22 @@ Vagrant.configure("2") do |config|
   config.vm.synced_folder ".", "/home/vagrant/sync", disabled: true
 
   config.vm.box = "centos/atomic-host"
+  host_vars = {}
   (1..3).each do |i|
-    config.vm.define "kube#{i}" do |vm|
-      vm.vm.hostname = "kube#{i}"
+    config.vm.define vm_name = "kube#{i}" do |vm|
+      vm.vm.hostname = vm_name
 
       vm.vm.provider :libvirt do |lv|
         lv.default_prefix = "gcs"
         lv.cpus = 2
         lv.memory = 2048
 
+        disks = []
         (1..3).each do |d|
           lv.storage :file, :device => "vd#{driveletters[d]}", :size => '1024G'
+          disks.push "/dev/vd#{driveletters[d]}"
         end
+        host_vars[vm_name] = {"gcs_disks" => disks.to_s}
       end
       # TODO: Maybe support other providers... like VirtualBox
 
@@ -36,8 +40,10 @@ Vagrant.configure("2") do |config|
             "etcd" => ["kube[1:3]"],
             "kube-master" => ["kube[1:2]"],
             "kube-node" => ["kube[1:3]"],
-            "k8s-cluster:children" => ["kube-master", "kube-node"]
+            "k8s-cluster:children" => ["kube-master", "kube-node"],
+            "gcs-node" => ["kube[1:3]"]
           }
+          ansible.host_vars = host_vars
         end
       end
     end

deploy/add-devices.yml

@@ -3,38 +3,27 @@
   hosts: kube-master[0]
   gather_facts: false
   vars:
-    kubectl: /usr/local/bin/kubectl
-    devices_list:
-      - /dev/vdb
-      - /dev/vdc
-      - /dev/vdd
+    kubectl: "/usr/local/bin/kubectl"
 
   tasks:
-    - name: Get glusterd2-client service
+    - name: Add devices | Get glusterd2-client service
       command: "{{ kubectl }} -n{{ gcs_namespace }} -ojsonpath='{.spec.clusterIP}' get service glusterd2-client "
       register: gd2_client_service
       until: gd2_client_service.rc == 0
       delay: 5
       retries: 5
 
-    - name: Set gd2_client_endpoint
+    - name: Add devices | Set gd2_client_endpoint
       set_fact:
         gd2_client_endpoint: "http://{{ gd2_client_service.stdout }}:24007"
         cacheable: true
 
-    - name: Get peers in cluster
+    - name: Add devices | Get peers in cluster
       uri:
         url: "{{ gd2_client_endpoint }}/v1/peers"
       register: peers_resp
 
-    - name: Set gd2_peers fact
-      set_fact:
-        gd2_peers: "{{ peers_resp.json | json_query('[].id') }}"
-
-    - name: Register devices
-      uri:
-        method: POST
-        url: "{{ gd2_client_endpoint }}/v1/devices/{{ item.0 }}"
-        body: "{ \"device\": \"{{ item.1 }}\"}"
-        body_format: json
-      loop: "{{ query('nested', gd2_peers, devices_list)}}"
+    - include_tasks: ./tasks/add-devices-to-peer.yml
+      loop: "{{ peers_resp.json }}"
+      loop_control:
+        loop_var: peer

deploy/deploy-gcs.yml

@@ -29,18 +29,22 @@
           set_fact:
             manifests_dir: "{{ tempdir.path }}"
 
-        - name: GCS Pre | Manifests
-          block:
-            - name: GCS Pre | Manifests | Sync GCS manifests
-              template:
-                src: "gcs-manifests/{{ item }}.j2"
-                dest: "{{ manifests_dir }}/{{ item }}"
-              loop:
-                - gcs-namespace.yml
-                - gcs-etcd-operator.yml
-                - gcs-etcd-cluster.yml
-                - gcs-gd2-cluster.yml
-                - gcs-csi.yml
+        - name: GCS Pre | Manifests | Sync GCS manifests
+          template:
+            src: "gcs-manifests/{{ item }}.j2"
+            dest: "{{ manifests_dir }}/{{ item }}"
+          loop:
+            - gcs-namespace.yml
+            - gcs-etcd-operator.yml
+            - gcs-etcd-cluster.yml
+            - gcs-gd2-services.yml
+            - gcs-csi.yml
+
+        - name: GCS Pre | Manifests | Create GD2 manifests
+          include_tasks: tasks/create-gd2-manifests.yml
+          loop: "{{ groups['gcs-node'] }}"
+          loop_control:
+            loop_var: gcs_node
 
   post_tasks:
     - name: GCS Post | Manifests | Delete
@@ -101,10 +105,16 @@
 
     - name: GCS | GD2 Cluster
       block:
-        - name: GCS | GD2 Cluster | Deploy glusterd2-cluster
+        - name: GCS | GD2 Cluster | Deploy GD2 services
           kube:
             kubectl: "{{ kubectl }}"
-            file: "{{ manifests_dir }}/gcs-gd2-cluster.yml"
+            file: "{{ manifests_dir }}/gcs-gd2-services.yml"
+
+        - name: GCS | GD2 Cluster | Deploy GD2
+          include_tasks: ./tasks/deploy-gd2.yml
+          loop: "{{ groups['gcs-node'] }}"
+          loop_control:
+            loop_var: gcs_node
 
         - name: GCS | GD2 Cluster | Get glusterd2-client service
           command: "{{ kubectl }} -n{{ gcs_namespace }} -ojsonpath='{.spec.clusterIP}' get service glusterd2-client "

deploy/tasks/add-devices-to-peer.yml

@@ -0,0 +1,16 @@
+## This file is included as a loop from the add-devices playbook
+---
+- name: Add device | Set facts
+  set_fact:
+    peer_id: "{{ peer.id }}"
+    kube_hostname: "{{ peer.name.split('-')[0]}}"
+
+- name: Add device | Register devices for {{ kube_hostname }}
+  uri:
+    url: "{{ gd2_client_endpoint }}/v1/devices/{{ peer_id }}"
+    method: POST
+    body: "{ \"device\": \"{{ disk }}\"}"
+    body_format: json
+  loop: "{{ hostvars[kube_hostname].gcs_disks }}"
+  loop_control:
+    loop_var: disk

deploy/tasks/create-gd2-manifests.yml

@@ -0,0 +1,10 @@
+## This file is included as a loop from the deploy-gcs playbook
+---
+- name: GCS Pre | Manifests | Create GD2 manifests for {{ gcs_node }} | Set fact kube_hostname
+  set_fact:
+    kube_hostname: "{{ gcs_node }}"
+
+- name: GCS Pre | Manifests | Create GD2 manifests for {{ gcs_node }} | Create gcs-gd2-{{ gcs_node }}.yml
+  template:
+    src: "gcs-manifests/gcs-gd2.yml.j2"
+    dest: "{{ manifests_dir }}/gcs-gd2-{{ gcs_node }}.yml"

deploy/tasks/deploy-gd2.yml

@@ -0,0 +1,11 @@
+## This file is included as a loop from the deploy-gcs playbook
+---
+- name: GCS | GD2 Cluster | Set fact kube_hostname
+  set_fact:
+    kube_hostname: "{{ gcs_node }}"
+
+- name: GCS | GD2 Cluster | Deploy glusterd2 on {{ gcs_node }}
+  kube:
+    kubectl: "{{ kubectl }}"
+    file: "{{ manifests_dir }}/gcs-gd2-{{ gcs_node }}.yml"
+

deploy/templates/gcs-manifests/gcs-csi.yml.j2

@@ -41,16 +41,21 @@ apiVersion: apps/v1
 metadata:
   name: csi-attacher-glusterfsplugin
   namespace: {{ gcs_namespace }}
+  labels:
+    gcs: csi
+    gcs/csi: attacher
 spec:
   serviceName: csi-attacher
   replicas: 1
   selector:
     matchLabels:
-      app: csi-attacher-glusterfsplugin
+      gcs: csi
+      gcs/csi: attacher
   template:
     metadata:
       labels:
-        app: csi-attacher-glusterfsplugin
+        gcs: csi
+        gcs/csi: attacher
     spec:
       serviceAccountName: csi-attacher
       containers:
@@ -131,14 +136,19 @@ apiVersion: apps/v1
 metadata:
   name: csi-nodeplugin-glusterfsplugin
   namespace: {{ gcs_namespace }}
+  labels:
+    gcs: csi
+    gcs/csi: nodeplugin
 spec:
   selector:
     matchLabels:
-      app: csi-nodeplugin-glusterfsplugin
+      gcs: csi
+      gcs/csi: nodeplugin
   template:
     metadata:
       labels:
-        app: csi-nodeplugin-glusterfsplugin
+        gcs: csi
+        gcs/csi: nodeplugin
       namespace: {{ gcs_namespace }}
     spec:
       serviceAccount: csi-nodeplugin
@@ -240,18 +250,23 @@ apiVersion: apps/v1
 metadata:
   name: csi-provisioner-glusterfsplugin
   namespace: {{ gcs_namespace }}
+  labels:
+    gcs: csi
+    gcs/csi: provisioner
 spec:
   serviceName: csi-provisioner-glusterfsplugin
   replicas: 1
   selector:
     matchLabels:
-      app: csi-provisioner-glusterfsplugin
+      gcs: csi
+      gcs/csi: provisioner
   template:
     metadata:
       name: csi-provisioner
       namespace: {{ gcs_namespace }}
       labels:
-        app: csi-provisioner-glusterfsplugin
+        gcs: csi
+        gcs/csi: provisioner
     spec:
       serviceAccountName: csi-provisioner
       containers:

deploy/templates/gcs-manifests/gcs-gd2-services.yml.j2

@@ -0,0 +1,64 @@
+---
+# Headless service for GD2 Stateful sets
+kind: Service
+apiVersion: v1
+metadata:
+  name: glusterd2
+  namespace: {{ gcs_namespace }}
+  labels:
+    gcs: glusterd2-service
+spec:
+  clusterIP: None
+  selector:
+    gcs: glusterd2
+    gcs/glusterd2: app
+  ports:
+    - name: clientport
+      protocol: TCP
+      port: 24007
+      targetPort: 24007
+    - name: peerport
+      protocol: TCP
+      port: 24008
+      targetPort: 24008
+
+---
+# Client service for the CSI driver
+kind: Service
+apiVersion: v1
+metadata:
+  name: glusterd2-client
+  namespace: {{ gcs_namespace }}
+  labels:
+    gcs: glusterd2-service
+spec:
+  selector:
+    gcs: glusterd2
+    gcs/glusterd2: app
+  ports:
+    - name: clientport
+      protocol: TCP
+      port: 24007
+      targetPort: 24007
+
+---
+# Nodeport service for access to GD2 from outside the cluster
+# TODO: Do not enable by default
+kind: Service
+apiVersion: v1
+metadata:
+  name: glusterd2-client-nodeport
+  namespace: {{ gcs_namespace }}
+  labels:
+    gcs: glusterd2-service
+spec:
+  selector:
+    gcs: glusterd2
+    gcs/glusterd2: app
+  ports:
+    - name: clientport
+      protocol: TCP
+      port: 24007
+      targetPort: 24007
+      nodePort: 31007
+  type: NodePort