Fixes to swarm check_swarm, #43

Adding python 3.8 to tests

Pipfile

@@ -14,4 +14,4 @@ pytest-cov = '<2.6'
 poetry = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"

check_docker/__init__.py

@@ -1,2 +1,2 @@
 """Nagios/NRPE compatible plugins for checking docker based services"""
-__version__ = "2.2.0"
+__version__ = "2.2.1"

check_docker/check_docker.py

@@ -1,32 +1,31 @@
 #!/usr/bin/env python3
 # logging.basicConfig(level=logging.DEBUG)
-import math
-from collections import deque, namedtuple, UserDict, defaultdict
-from sys import argv
-
 import argparse
 import json
 import logging
+import math
 import os
 import re
 import socket
 import stat
 import traceback
+from collections import deque, namedtuple, UserDict, defaultdict
 from concurrent import futures
 from datetime import datetime, timezone
 from functools import lru_cache
 from http.client import HTTPConnection
+from sys import argv
 from urllib import request
 from urllib.error import HTTPError, URLError
 from urllib.request import AbstractHTTPHandler, HTTPHandler, HTTPSHandler, OpenerDirector, HTTPRedirectHandler, \
     Request, HTTPBasicAuthHandler
 
 logger = logging.getLogger()
 __author__ = 'Tim Laurence'
-__copyright__ = "Copyright 2018"
+__copyright__ = "Copyright 2019"
 __credits__ = ['Tim Laurence']
 __license__ = "GPL"
-__version__ = "2.2.0"
+__version__ = "2.2.1"
 
 '''
 nrpe compatible check for docker containers.
@@ -79,7 +78,9 @@ def __init__(self, warn, crit, units=''):
         super().__init__(warn=warn, crit=crit, units=units)
 
     def __getattr__(self, item):
-        return self[item]
+        if item in ('warn', 'crit', 'units'):
+            return self.data[item]
+        return super().__getattr__(item)
 
 
 # How much threading can we do? We are generally not CPU bound so I am using this a worse case cap
@@ -151,7 +152,7 @@ def _get_outh2_token(www_authenticate_header):
 
     def process_oauth2(self, request, response, www_authenticate_header):
 
-        # This keep infinite auth loops from happening
+        # This keeps infinite auth loops from happening
         full_url = request.full_url
         self.auth_failure_tracker[full_url] += 1
         if self.auth_failure_tracker[full_url] > 1:

check_docker/check_swarm.py

@@ -13,10 +13,10 @@
 
 logger = logging.getLogger()
 __author__ = 'Tim Laurence'
-__copyright__ = "Copyright 2018"
+__copyright__ = "Copyright 2019"
 __credits__ = ['Tim Laurence']
 __license__ = "GPL"
-__version__ = "2.2.0"
+__version__ = "2.2.1"
 
 '''
 nrpe compatible check for docker swarm
@@ -103,6 +103,16 @@ def get_service_info(name):
     return get_url(daemon + '/services/{service}'.format(service=name))
 
 
+def get_service_running_tasks(name):
+    tasks, status = get_url(daemon + '/tasks?service={service}'.format(service=name))
+    running_tasks = [task for task in tasks if task['Status']['State'] == 'running']
+    return running_tasks
+
+
+def get_nodes():
+    return get_url(daemon + '/nodes')
+
+
 def get_services(names):
     services_list, status = get_url(daemon + '/services')
     if status == 406:
@@ -163,10 +173,41 @@ def check_swarm():
                        critical_msg='Node is not in a swarm', unknown_msg='Error accessing swarm info')
 
 
+def process_global_service(name):
+    # Find how many nodes should have service running. I assume 'active' and 'paused' are "online"
+    node_list, status = get_nodes()
+    online_nodes = len([node for node in node_list if node['Spec']['Availability'] != 'drain'])
+
+    # Get a count of the task found running
+    num_tasks = len(get_service_running_tasks(name))
+    if num_tasks < online_nodes:
+        critical('Global service {service} tasks not found for every active and paused node'.format(service=name))
+    elif num_tasks > online_nodes:
+        critical('Global service {service} has more tasks than the number nodes'.format(service=name))
+    else:
+        ok('Global service {service} OK'.format(service=name))
+
+
+def process_replicated_service(name, replicas_desired):
+    num_tasks = len(get_service_running_tasks(name))
+    if num_tasks != replicas_desired:
+        critical('Replicated service {service} has {num_tasks} tasks, {replicas_desired} desired'.
+                 format(service=name, num_tasks=num_tasks, replicas_desired=replicas_desired))
+    else:
+        ok('Replicated service {service} OK'.format(service=name))
+
+
 def check_service(name):
-    info, status = get_service_info(name)
-    process_url_status(status, ok_msg='Service {service} is up and running'.format(service=name),
-                       critical_msg='Service {service} was not found on the swarm'.format(service=name))
+    # get service mode
+    service_info, status = get_service_info(name)
+    mode_info = service_info['Spec']['Mode']
+
+    # if global ensure one per node
+    if 'Global' in mode_info:
+        process_global_service(name)
+    # if replicated ensure sufficient number of replicas
+    elif 'Replicated' in mode_info:
+        process_replicated_service(name, mode_info['Replicated']['Replicas'])
 
 
 def process_url_status(status, ok_msg=None, critical_msg=None, unknown_msg=None):

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api"
 
 [tool.poetry]
 name = "check_docker"
-version = "2.2.0"
+version = "2.2.1"
 description = "Nagios/NRPE compatible plugins for checking Docker based services"
 license = "GPL-3.0"
 authors = ["Tim Laurence <timdaman@gmail.com>"]

tests/test_check_swarm.py

@@ -50,10 +50,11 @@ def test_get_swarm_status(check_swarm):
 
 
 def test_get_service_info(check_swarm):
-    with patch('check_docker.check_swarm.get_url', return_value=('FOO', 999)):
-        response_data, response_status = check_swarm.get_service_info('FOO')
-        assert response_data == 'FOO'
-        assert response_status == 999
+    sample_response = ([{'Status': {'State': 'running', 'DesiredState': 'running'}},
+                        {'Status': {'State': 'failed', 'DesiredState': 'running'}}], 999)
+    with patch('check_docker.check_swarm.get_url', return_value=sample_response):
+        response_data = check_swarm.get_service_running_tasks('FOO')
+        assert len(response_data) == 1
 
 
 def test_get_services_not_swarm(check_swarm):
@@ -243,13 +244,16 @@ def test_check_service_called(check_swarm, services, fs):
             assert patched.call_count == 1
 
 
-def test_check_service_results_OK(check_swarm, services, fs):
+@pytest.mark.parametrize("service_info, expected_func", (
+        ({'Spec': {'Mode': {'Global': {}}}}, 'process_global_service'),
+        ({'Spec': {'Mode': {'Replicated': {'Replicas': 1}}}}, 'process_replicated_service'),
+))
+def test_check_services_routing(check_swarm, service_info, expected_func, fs):
     fs.create_file(check_swarm.DEFAULT_SOCKET, contents='', st_mode=(stat.S_IFSOCK | 0o666))
-    args = ['--service', 'FOO']
-    with patch('check_docker.check_swarm.get_services', return_value=['FOO', 'BAR']):
-        with patch('check_docker.check_swarm.get_service_info', return_value=(services, 200)):
-            check_swarm.perform_checks(args)
-            assert check_swarm.rc == cs.OK_RC
+    with patch('check_docker.check_swarm.get_service_info', return_value=(service_info, 999)), \
+         patch('check_docker.check_swarm.{}'.format(expected_func)) as patched:
+        check_swarm.check_service('FOO')
+        assert patched.call_count == 1
 
 
 def test_check_service_results_FAIL_missing(check_swarm, services, fs):

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py35,py36,py37
+envlist = py35,py36,py37,py38
 tox_pyenv_fallback = False
 isolated_build = True
 skip_missing_interpreters = True