Adds large tests

Adds large tests and example that runs in docker-compose/kubernetes.

Adds README for examples/tasks and scripts.

examples/tasks/.setup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# check for dependencies
+EXIT_ON_ERROR=0
+command -v docker >/dev/null 2>&1 || { echo >&2 "Error: docker needs to be installed."; EXIT_ON_ERROR=1; }
+command -v docker-compose >/dev/null 2>&1 || { echo >&2 "Error: docker-compose needs to be installed."; EXIT_ON_ERROR=1; }
+docker version >/dev/null 2>&1 || { echo >&2 "Error: docker needs to be configured."; EXIT_ON_ERROR=1; }
+if [[ $EXIT_ON_ERROR > 0 ]]; then
+    exit 1
+fi
+
+# start the container(s)
+(cd $__dir && docker-compose up -d) 
+
+# clean up containers on exit
+function finish {
+  (cd $__dir && docker-compose down)
+}
+trap finish EXIT INT TERM

examples/tasks/README.md

@@ -0,0 +1,28 @@
+# Example tasks
+
+[This](task-mock-disk.yml) example task will collect metrics from the disk collector.
+
+## Running the example
+
+### Requirements 
+ * `docker` and `docker-compose` are **installed** and **configured** 
+
+Running the sample is as *easy* as running the script `./run-mock-disk.sh`. 
+
+Note: If you want to run the example without going through Docker you could 
+run `mock-disk.sh`.  
+
+## Files
+
+- [run-mock-disk.sh](run-mock-disk.sh) 
+    - The example is launched with this script
+- [task-mock-disk.yml](task-mock-disk.yml)
+    - Snap task definition
+- [docker-compose.yml](docker-compose.yml)
+    - A docker compose file which defines a snapd container
+        - "runner" is the container where snapd is run from.  You will be dumped 
+        into a shell in this container after running 
+        [run-mock-disk.sh](run-mock-disk.sh).  Exiting the shell will
+        trigger cleaning up the containers used in the example.
+- [mock-disk.sh](mock-disk.sh)
+    - Downloads `snapd`, `snapctl`, `snap-plugin-collector-disk`, and starts the task [task-mock-disk.yml](task-mock-disk.yml).

examples/tasks/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '2'
+services:
+  main:
+    container_name: runner
+    image: intelsdi/snap:alpine
+    environment:
+     - SNAP_VERSION=latest
+    volumes:
+      - ${PLUGIN_SRC}:/snap-plugin-collector-disk

examples/tasks/mock-disk.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+# get the directory the script exists in
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# source the common bash script 
+. "${__dir}/../../scripts/common.sh"
+
+# ensure PLUGIN_PATH is set
+TMPDIR=${TMPDIR:-"/tmp"}
+PLUGIN_PATH=${PLUGIN_PATH:-"${TMPDIR}/snap/plugins"}
+mkdir -p $PLUGIN_PATH
+
+_info "Get latest plugins"
+(cd $PLUGIN_PATH && curl -sSO http://snap.ci.snap-telemetry.io/plugins/snap-plugin-collector-disk/master/latest/linux/x86_64/snap-plugin-collector-disk && chmod 755 snap-plugin-collector-disk)
+
+SNAP_FLAG=0
+
+# this block will wait check if snapctl and snapd are loaded before the plugins are loaded and the task is started
+ for i in `seq 1 5`; do
+             if [[ -f /usr/local/bin/snapctl && -f /usr/local/bin/snapd ]];
+                then
+
+                    _info "loading plugins"
+                    snapctl plugin load "${PLUGIN_PATH}/snap-plugin-collector-disk"
+
+                    _info "creating and starting a task"
+                    snapctl task create -t "${__dir}/task-mock-disk.yml" 
+
+                    SNAP_FLAG=1
+
+                    break
+             fi 
+
+        _info "snapctl and/or snapd are unavailable, sleeping for 3 seconds" 
+        sleep 3
+done 
+
+
+# check if snapctl/snapd have loaded
+if [ $SNAP_FLAG -eq 0 ]
+    then
+     echo "Could not load snapctl or snapd"
+     exit 1
+fi

examples/tasks/run-mock-disk.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+# get the directory the script exists in
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(cd $__dir && cd ../../ && pwd)"
+__proj_name="$(basename $__proj_dir)"
+
+export PLUGIN_SRC="${__proj_dir}"
+
+# source the common bash script 
+. "${__proj_dir}/scripts/common.sh"
+
+# verifies dependencies
+. "${__proj_dir}/examples/tasks/.setup.sh"
+
+# downloads plugins, starts snap, load plugins and start a task
+cd "${__proj_dir}/examples/tasks" && docker-compose exec main bash -c "PLUGIN_PATH=/etc/snap/plugins /${__proj_name}/examples/tasks/mock-disk.sh && printf \"\n\nhint: type 'snapctl task list'\ntype 'exit' when your done\n\n\" && bash"

examples/tasks/task-mock-disk.yml

@@ -0,0 +1,13 @@
+---
+  version: 1
+  schedule:
+    type: "simple"
+    interval: "1s"
+  max-failures: 10
+  workflow:
+    collect:
+      metrics:
+        /intel/procfs/disk/*/* : {}
+      config:
+        /intel/procfs/proc:
+            proc_path: "/proc"

scripts/README.md

@@ -0,0 +1,25 @@
+# Scripts
+
+The scripts contained in this directory facilitate **building** and **testing** 
+the plugin.  The main entry point for most of these scripts is the 
+[Makefile](../Makefile). 
+
+## Running tests
+
+### small tests
+
+From the root of the project run `make test-small`
+
+### medium tests
+
+From the root of the project run `make test-medium`
+
+### Large
+
+From the root of the projet run `make test-large`
+
+Large tests require that `docker` is available and configured.  By default the 
+tests will also require and use `docker-compose` to orchestrate the containers 
+used. As an alternative to `docker-compose` kubernetes can used to orchestrate 
+the containers by running the test with the env `TEST_K8S=1 make test-large`.
+

scripts/config/disk-deployment.yml

@@ -0,0 +1,20 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+    name: disk-deployment
+spec:
+    replicas: 1
+    template:
+        metadata:
+            labels:
+                app: disk-collector
+        spec:
+            containers:
+            - name: main
+              image: intelsdi/snap:alpine
+              env:
+                - name: SNAP_VERSION
+                  value: "latest"
+              imagePullPolicy: "IfNotPresent"
+

scripts/config/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2'
+services:
+  golang:
+    container_name: golang_${GOLANGVER}
+    image: intelsdi/gvm:latest
+    volumes:
+      - ${PLUGIN_SRC}:/snap-plugin-publisher-disk

scripts/large_compose.sh

@@ -0,0 +1,40 @@
+#!/bin/bash 
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+__proj_name="$(basename $__proj_dir)"
+
+. "${__dir}/common.sh"
+
+# NOTE: these variables control the docker-compose image.
+export PLUGIN_SRC="${__proj_dir}"
+export LOG_LEVEL="${LOG_LEVEL:-"7"}"
+export PROJECT_NAME="${__proj_name}"
+
+TEST_TYPE="${TEST_TYPE:-"large"}"
+
+docker_folder="${__proj_dir}/examples/tasks"
+
+_docker_project () {
+  (cd "${docker_folder}" && "$@")
+}
+
+_info "docker folder : $docker_folder"
+
+_debug "running docker compose images"
+_docker_project docker-compose up -d 
+_debug "running test: ${TEST_TYPE}"
+
+set +e
+_docker_project docker-compose exec main bash -c "export LOG_LEVEL=$LOG_LEVEL; /${__proj_name}/scripts/large_tests.sh" 
+test_res=$?
+set -e
+echo "exit code from large_compose $test_res"
+_debug "stopping and removing containers"
+_docker_project docker-compose down
+
+exit $test_res

scripts/large_k8s.sh

@@ -0,0 +1,56 @@
+#!/bin/bash -x
+
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2015 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+__proj_name="$(basename $__proj_dir)"
+__deployment_file="$__dir/config/disk-deployment.yml"
+__deployment_name="disk-deployment"
+
+. "${__dir}/common.sh"
+
+_debug "__dir ${__dir}"
+_debug "__proj_dir ${__proj_dir}"
+_debug "__proj_name ${__proj_name}"
+
+_debug "start k8 deployement $__deployment_file"
+kubectl create -f $__deployment_file
+while ! [ "$(kubectl get po --no-headers | grep $__deployment_name | grep Running | awk '{print $2}')" = "1/1" ]; do
+    kubectl get po --no-headers | grep $__deployment_name | grep CrashLoopBackOff && echo 'container failed' && exit 1
+    echo 'waiting for pod to come up'
+    sleep 5
+done
+_debug "copying the src into the runner"
+kubectl exec $(kubectl get po --no-headers | grep $__deployment_name | grep Running | awk '{print $1}') -c main -i  -- mkdir /src
+tar c  . | kubectl exec $(kubectl get po --no-headers | grep $__deployment_name | grep Running | awk '{print $1}') -c main -i  --  tar -x -C /src
+
+set +e
+_debug "running tests through the runner"
+kubectl exec $(kubectl get po --no-headers | grep $__deployment_name | grep Running | awk '{print $1}') -c main -i -- /src/scripts/large_tests.sh
+test_res=$?
+set -e
+_debug "exit code $test_res"
+_debug "removing k8 deployment"
+kubectl delete -f $__deployment_file
+exit $test_res
+

scripts/large_tests.sh

@@ -0,0 +1,47 @@
+#!/bin/bash  
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+
+. "${__dir}/common.sh"
+
+_info "running the example ${__proj_dir}/examples/tasks/mock-disk.sh"
+export PLUGIN_PATH="/etc/snap/path"
+source "${__proj_dir}/examples/tasks/mock-disk.sh"
+
+_debug "sleeping for 10 seconds so the task can do some work"
+sleep 10
+
+# begin assertions
+return_code=0
+echo -n "[task is running] "
+task_list=$(snapctl task list | tail -1)
+if echo $task_list | grep -q Running; then
+    echo "ok"
+else 
+    echo "not ok"
+    return_code=-1
+fi
+
+echo -n "[task is hitting] "
+if [ $(echo $task_list | awk '{print $4}') -gt 0 ]; then
+    echo "ok"
+else 
+    _debug $task_list
+    echo "not ok"
+    return_code=-1
+fi
+
+echo -n "[task has no errors] "
+if [ $(echo $task_list | awk '{print $6}') -eq 0 ]; then
+    echo "ok"
+else 
+    echo "not ok"
+    return_code=-1
+fi
+
+exit $return_code