Merge pull request #4 from network-automate/dev_titom73

v1.0 Publication

Makefile

@@ -2,8 +2,8 @@ build-telegraf-conf:
 	@echo "======================================================================"
 	@echo "Build telegraf configuration files from template"
 	@echo "======================================================================"
-	python ./render-telegraf-configuration.py -o 'configs/telegraf-openconfig.conf' -t 'templates/telegraf-openconfig.j2' -y 'data.yml'
-	python ./render-telegraf-configuration.py -o 'configs/telegraf-snmp.conf' -t 'templates/telegraf-snmp.j2' -y 'data.yml'
+	python ./configuration-rendering.py -o 'configs/telegraf-openconfig.conf' -t 'templates/telegraf-openconfig.j2' -y 'data.yml'
+	python ./configuration-rendering.py -o 'configs/telegraf-snmp.conf' -t 'templates/telegraf-snmp.j2' -y 'data.yml'
 
 grafana-cli:
 	@echo "======================================================================"
@@ -36,7 +36,7 @@ build:
 	@echo "======================================================================"
 	@echo "create docker networks, pull docker images, create and start docker containers"
 	@echo "======================================================================"
-	python ./render-telegraf-configuration.py -o 'docker-compose.yml' -t 'templates/docker-compose-tig.j2' -y 'data.yml'
+	python ./configuration-rendering.py -o 'docker-compose.yml' -t 'templates/docker-compose-tig.j2' -y 'data.yml'
 	docker-compose -f ./docker-compose.yml up -d
 
 destroy:

README.md

@@ -1,19 +1,134 @@
-This repository provides instructions about how to monitor Junos devices using a TIG stack (Telegraf-Influxdb-Grafana).  
-It currently supports data collection on Junos using SNMP and OpenConfig.
+# Network Monitoring with Telegraf / Influxdb / Grafana
 
-Please visit the [**wiki**](https://github.com/ksator/junos_monitoring_with_a_TIG_stack/wiki) for detailled instructions.  
+<!-- MarkdownTOC -->
 
-Here are some Grafana screenshots, with data collected using Openconfig telemetry (GRPC) on Junos devices:
+- Quick path to demo
+- Requiremetns & Installation
+- Commands and usage
+    - Grafana access
+    - Device List
+- Components
+- Contributors
 
-![EBGP_peers_configured.png](resources/EBGP_peers_configured.png)
+<!-- /MarkdownTOC -->
 
-![BGP_sessions_state_established.png](resources/BGP_sessions_state_established.png)
+Complete stack to monitor [Juniper](https://www.juniper.net) datacenter. it uses following components:
 
-![transitions_to_bgp_established.png](resources/transitions_to_bgp_established.png)
+- [telegraf](https://www.influxdata.com/time-series-platform/telegraf/) : Agent for collecting information from devices. It can be used in 2 different flavor: `snmp` and/or [`openconfig`](http://www.openconfig.net/)
+- [influxdb](https://www.influxdata.com/) : Time series database (TSN) to store all information sent by telegraf agents.
+- [grafana](https://grafana.com/) : Time series analytics engine to build reports from Influxdb.
 
-![BGP_prefixes_received.png](resources/BGP_prefixes_received.png)
+All the configuration is managed by templating for following components:
 
-![BGP_prefixes_sent.png](resources/BGP_prefixes_sent.png)
+- [`docker-compose`](templates/docker-compose-tig.j2) file
+- [`telegraf`](templates/telegraf-snmp.j2) configuration
 
+Dashboards available:
 
+- **Fabric Monitoring**: information from generic SNMP based device running in an IP-Fabric environment (Interface counters / BGP status / BGP Update and state messages)
+- **Fabric Reporting**: information from all Junos SNMP based devices running in an IP-Fabric environment (Hw type / Serial number / Junos Version)
 
+!["BGP Status"](resources/snmp-bgp-status.png)
+
+## Quick path to demo
+
+```shell
+# Install python requirements
+pip intall -r requirements.txt
+
+# Edit data.yml to add your devices
+vim data.yml
+
+# Build and start stack
+make build
+
+# Stop and delete stack
+make destroy
+```
+
+Open a browser to http://127.0.0.1:9081/ with `super`/`juniper123`
+
+## Requiremetns & Installation
+
+As it is based on docker, you have to install docker first:
+
+- [Centos Installation](https://docs.docker.com/install/linux/docker-ce/centos/)
+- [Ubuntu Installation](https://docs.docker.com/install/linux/docker-ce/ubuntu/)
+- [MacOS](https://docs.docker.com/docker-for-mac/install/)
+
+Then,, you have to install python requirements
+
+```shell
+pip install -r requirements.txt
+```
+
+## Commands and usage
+
+Define your targets by editing `data.yml` file:
+
+### Grafana access
+
+Access to grafna can be managed in `data.yml` file with the following section:
+
+```yaml
+grafana:
+  web:
+    port: "9081"
+    username: "super"
+    password: "juniper123"
+```
+
+> If not set, access is configured to be like: http://127.0.0.1:9081/ with username/password set to super/juniper123
+
+### Device List
+__SNMP devices__
+
+```yaml
+---
+telegraf:
+  snmp:
+    community: "public"
+    hosts:
+      172.25.90.67: 161
+      172.25.90.68: 161
+```
+
+__Junos Openconfig devices__
+
+```yaml
+---
+telegraf:
+  openconfig:
+    username: 'ansible'
+    password: 'juniper123'
+    hosts:
+      172.25.90.67: 32768
+      172.25.90.68: 32768
+```
+
+> Both `snmp` and `openconfig` definition can be configure in this `data.yml`
+
+Some commands are available to manage repository
+
+- `make build-telegraf-conf` : Build telegraf configuration with template rendering
+- `make build` : Build telegraf config, build docker-compose stack, start stack
+- `make destroy` : Stop docker stack and remove containers
+- `make start` : start an already configured stack. (Must be done if stack were built previously)
+- `make stop` : stop running stack. Containers are not deleted and can be restarted with `make start`
+- `make restart` : stop and start stack
+- `make rebuild` : destroy and build stack
+- `make {telegraf-snmp|telegraf-openconfig|influxdb|grafana}-cli` : connect to containers
+
+## Components
+
+Repository is based on docker containers and they are all managed with `docker-compose`:
+
+- `telegraf:1.9.1` image for openconfig polling
+- `inetsix/telegraf-snmp` image for SNMP polling
+- `influxdb:1.7.2`
+- `grafana:grafana/grafana:5.4.2`
+
+## Contributors
+
+- [Khelil Sator](https://github.com/ksator)
+- [Thomas Grimonet](https://github.com/titom73)

dashboards/snmp-fabric-monitoring.json

@@ -16,7 +16,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "iteration": 1547120178787,
+  "iteration": 1547136972693,
   "links": [],
   "panels": [
     {
@@ -185,6 +185,12 @@
                 "params": [],
                 "type": "last"
               },
+              {
+                "params": [
+                  " / 100"
+                ],
+                "type": "math"
+              },
               {
                 "params": [
                   "Uptime"

data.yml

@@ -1,4 +1,9 @@
 ---
+grafana:
+  web:
+    port: "9081"
+    username: "super"
+    password: "juniper123"
 telegraf:
   snmp:
     community: "public"

docker-compose.yml

@@ -29,8 +29,9 @@ services:
     container_name: grafana
     ports:
       - "9081:3000"
-    env_file:
-      - 'env.grafana'
+    environment:
+      - GF_SECURITY_ADMIN_USER=super
+      - GF_SECURITY_ADMIN_PASSWORD=juniper123
     volumes:
       - $PWD/configs/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml:ro
       - $PWD/configs/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro

requirements.txt

@@ -0,0 +1,24 @@
+backports.ssl-match-hostname==3.5.0.1
+cached-property==1.5.1
+certifi==2018.11.29
+chardet==3.0.4
+docker==3.7.0
+docker-compose==1.23.2
+docker-pycreds==0.4.0
+dockerpty==0.4.1
+docopt==0.6.2
+enum34==1.1.6
+functools32==3.2.3.post2
+idna==2.7
+ipaddress==1.0.22
+Jinja2==2.10
+jsonschema==2.6.0
+MarkupSafe==1.1.0
+pprint==0.1
+pyaml==18.11.0
+PyYAML==3.13
+requests==2.20.1
+six==1.12.0
+texttable==0.9.1
+urllib3==1.24.1
+websocket-client==0.54.0

templates/docker-compose-tig.j2

@@ -42,9 +42,10 @@ services:
     image: grafana/grafana:5.4.2
     container_name: grafana
     ports:
-      - "9081:3000"
-    env_file:
-      - 'env.grafana'
+      - "{{ grafana.web.port|default("9081") }}:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER={{ grafana.web.username | default("super") }}
+      - GF_SECURITY_ADMIN_PASSWORD={{ grafana.web.password | default("juniper123") }}
     volumes:
       - $PWD/configs/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml:ro
       - $PWD/configs/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro