Autocreate data folder in Kubernetes deployments (#7437)

We need to persist beats data folder, so Pod restarts can still access
to the previous data.

This is particularly important in the case of Filebeat, as it needs the
registry file to avoid sending all logs again.

.travis.yml

@@ -14,6 +14,7 @@ env:
     - GOX_FLAGS="-arch amd64"
     - DOCKER_COMPOSE_VERSION=1.11.1
     - GO_VERSION="$(cat .go-version)"
+    # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704
     - TRAVIS_MINIKUBE_VERSION=v0.25.2
 
 jobs:
@@ -109,18 +110,6 @@ jobs:
       stage: test
 
     # Kubernetes
-    - os: linux
-      install: deploy/kubernetes/.travis/setup.sh
-      env:
-        - TARGETS="-C deploy/kubernetes test"
-        - TRAVIS_K8S_VERSION=v1.6.4
-      stage: test
-    - os: linux
-      install: deploy/kubernetes/.travis/setup.sh
-      env:
-        - TARGETS="-C deploy/kubernetes test"
-        - TRAVIS_K8S_VERSION=v1.7.5
-      stage: test
     - os: linux
       install: deploy/kubernetes/.travis/setup.sh
       env:
@@ -139,6 +128,13 @@ jobs:
         - TARGETS="-C deploy/kubernetes test"
         - TRAVIS_K8S_VERSION=v1.10.0
       stage: test
+    # TODO include 1.11 once minikube supports it
+    #- os: linux
+    #  install: deploy/kubernetes/.travis/setup.sh
+    #  env:
+    #    - TARGETS="-C deploy/kubernetes test"
+    #    - TRAVIS_K8S_VERSION=v1.11.0
+    #  stage: test
 
 addons:
   apt:

deploy/kubernetes/.travis/setup.sh

@@ -2,14 +2,15 @@
 #!/bin/bash
 
 set -x
+set -e
 
 export CHANGE_MINIKUBE_NONE_USER=true
 
 curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$TRAVIS_K8S_VERSION/bin/linux/amd64/kubectl && \
       chmod +x kubectl && sudo mv kubectl /usr/local/bin/
 curl -Lo minikube https://storage.googleapis.com/minikube/releases/$TRAVIS_MINIKUBE_VERSION/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
 sudo minikube start --vm-driver=none --kubernetes-version=$TRAVIS_K8S_VERSION --logtostderr
-minikube update-context
+sudo minikube update-context
 JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; \
         until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done
 

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -114,11 +114,11 @@ spec:
         configMap:
           defaultMode: 0600
           name: filebeat-inputs
-      # We set an `emptyDir` here to ensure the manifest will deploy correctly.
-      # It's recommended to change this to a `hostPath` folder, to ensure internal data
-      # files survive pod changes (ie: version upgrade)
+      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
       - name: data
-        emptyDir: {}
+        hostPath:
+          path: /var/lib/filebeat-data
+          type: DirectoryOrCreate
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding

deploy/kubernetes/filebeat/filebeat-daemonset.yaml

@@ -66,8 +66,8 @@ spec:
         configMap:
           defaultMode: 0600
           name: filebeat-inputs
-      # We set an `emptyDir` here to ensure the manifest will deploy correctly.
-      # It's recommended to change this to a `hostPath` folder, to ensure internal data
-      # files survive pod changes (ie: version upgrade)
+      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
       - name: data
-        emptyDir: {}
+        hostPath:
+          path: /var/lib/filebeat-data
+          type: DirectoryOrCreate

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -151,11 +151,10 @@ spec:
         configMap:
           defaultMode: 0600
           name: metricbeat-daemonset-modules
-      # We set an `emptyDir` here to ensure the manifest will deploy correctly.
-      # It's recommended to change this to a `hostPath` folder, to ensure internal data
-      # files survive pod changes (ie: version upgrade)
       - name: data
-        emptyDir: {}
+        hostPath:
+          path: /var/lib/metricbeat-data
+          type: DirectoryOrCreate
 ---
 apiVersion: v1
 kind: ConfigMap

deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml

@@ -79,8 +79,7 @@ spec:
         configMap:
           defaultMode: 0600
           name: metricbeat-daemonset-modules
-      # We set an `emptyDir` here to ensure the manifest will deploy correctly.
-      # It's recommended to change this to a `hostPath` folder, to ensure internal data
-      # files survive pod changes (ie: version upgrade)
       - name: data
-        emptyDir: {}
+        hostPath:
+          path: /var/lib/metricbeat-data
+          type: DirectoryOrCreate

filebeat/docs/running-on-kubernetes.asciidoc

@@ -32,6 +32,13 @@ To get the manifests just run:
 curl -L -O https://raw.githubusercontent.com/elastic/beats/{doc-branch}/deploy/kubernetes/filebeat-kubernetes.yaml
 ------------------------------------------------
 
+[WARNING]
+=======================================
+If you are using Kubernetes 1.7 or earlier: {beatname_uc} uses a hostPath volume to persist internal data, it's located
+under /var/lib/{beatname_lc}-data. The manifest uses folder autocreation (`DirectoryOrCreate`), which was introduced in
+Kubernetes 1.8. You will need to remove `type: DirectoryOrCreate` from the manifest and create the host folder yourself.
+=======================================
+
 [float]
 ==== Settings
 

metricbeat/docs/running-on-kubernetes.asciidoc

@@ -36,6 +36,13 @@ To get the manifests just run:
 curl -L -O https://raw.githubusercontent.com/elastic/beats/{doc-branch}/deploy/kubernetes/metricbeat-kubernetes.yaml
 ------------------------------------------------
 
+[WARNING]
+=======================================
+If you are using Kubernetes 1.7 or earlier: {beatname_uc} uses a hostPath volume to persist internal data, it's located
+under /var/lib/{beatname_lc}-data. The manifest uses folder autocreation (`DirectoryOrCreate`), which was introduced in
+Kubernetes 1.8. You will need to remove `type: DirectoryOrCreate` from the manifest and create the host folder yourself.
+=======================================
+
 [float]
 ==== Settings