diff --git a/Makefile b/Makefile index 2cbce8f..edb6ace 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ build-telegraf-conf: @echo "======================================================================" @echo "Build telegraf configuration files from template" @echo "======================================================================" - python ./render-telegraf-configuration.py -o 'configs/telegraf-openconfig.conf' -t 'templates/telegraf-openconfig.j2' -y 'data.yml' - python ./render-telegraf-configuration.py -o 'configs/telegraf-snmp.conf' -t 'templates/telegraf-snmp.j2' -y 'data.yml' + python ./configuration-rendering.py -o 'configs/telegraf-openconfig.conf' -t 'templates/telegraf-openconfig.j2' -y 'data.yml' + python ./configuration-rendering.py -o 'configs/telegraf-snmp.conf' -t 'templates/telegraf-snmp.j2' -y 'data.yml' grafana-cli: @echo "======================================================================" @@ -36,7 +36,7 @@ build: @echo "======================================================================" @echo "create docker networks, pull docker images, create and start docker containers" @echo "======================================================================" - python ./render-telegraf-configuration.py -o 'docker-compose.yml' -t 'templates/docker-compose-tig.j2' -y 'data.yml' + python ./configuration-rendering.py -o 'docker-compose.yml' -t 'templates/docker-compose-tig.j2' -y 'data.yml' docker-compose -f ./docker-compose.yml up -d destroy: diff --git a/README.md b/README.md index b5f1e39..10d7db9 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,134 @@ -This repository provides instructions about how to monitor Junos devices using a TIG stack (Telegraf-Influxdb-Grafana). -It currently supports data collection on Junos using SNMP and OpenConfig. +# Network Monitoring with Telegraf / Influxdb / Grafana -Please visit the [**wiki**](https://github.com/ksator/junos_monitoring_with_a_TIG_stack/wiki) for detailled instructions. + -Here are some Grafana screenshots, with data collected using Openconfig telemetry (GRPC) on Junos devices: +- Quick path to demo +- Requiremetns & Installation +- Commands and usage + - Grafana access + - Device List +- Components +- Contributors -![EBGP_peers_configured.png](resources/EBGP_peers_configured.png) + -![BGP_sessions_state_established.png](resources/BGP_sessions_state_established.png) +Complete stack to monitor [Juniper](https://www.juniper.net) datacenter. it uses following components: -![transitions_to_bgp_established.png](resources/transitions_to_bgp_established.png) +- [telegraf](https://www.influxdata.com/time-series-platform/telegraf/) : Agent for collecting information from devices. It can be used in 2 different flavor: `snmp` and/or [`openconfig`](http://www.openconfig.net/) +- [influxdb](https://www.influxdata.com/) : Time series database (TSN) to store all information sent by telegraf agents. +- [grafana](https://grafana.com/) : Time series analytics engine to build reports from Influxdb. -![BGP_prefixes_received.png](resources/BGP_prefixes_received.png) +All the configuration is managed by templating for following components: -![BGP_prefixes_sent.png](resources/BGP_prefixes_sent.png) +- [`docker-compose`](templates/docker-compose-tig.j2) file +- [`telegraf`](templates/telegraf-snmp.j2) configuration +Dashboards available: +- **Fabric Monitoring**: information from generic SNMP based device running in an IP-Fabric environment (Interface counters / BGP status / BGP Update and state messages) +- **Fabric Reporting**: information from all Junos SNMP based devices running in an IP-Fabric environment (Hw type / Serial number / Junos Version) +!["BGP Status"](resources/snmp-bgp-status.png) + +## Quick path to demo + +```shell +# Install python requirements +pip intall -r requirements.txt + +# Edit data.yml to add your devices +vim data.yml + +# Build and start stack +make build + +# Stop and delete stack +make destroy +``` + +Open a browser to http://127.0.0.1:9081/ with `super`/`juniper123` + +## Requiremetns & Installation + +As it is based on docker, you have to install docker first: + +- [Centos Installation](https://docs.docker.com/install/linux/docker-ce/centos/) +- [Ubuntu Installation](https://docs.docker.com/install/linux/docker-ce/ubuntu/) +- [MacOS](https://docs.docker.com/docker-for-mac/install/) + +Then,, you have to install python requirements + +```shell +pip install -r requirements.txt +``` + +## Commands and usage + +Define your targets by editing `data.yml` file: + +### Grafana access + +Access to grafna can be managed in `data.yml` file with the following section: + +```yaml +grafana: + web: + port: "9081" + username: "super" + password: "juniper123" +``` + +> If not set, access is configured to be like: http://127.0.0.1:9081/ with username/password set to super/juniper123 + +### Device List +__SNMP devices__ + +```yaml +--- +telegraf: + snmp: + community: "public" + hosts: + 172.25.90.67: 161 + 172.25.90.68: 161 +``` + +__Junos Openconfig devices__ + +```yaml +--- +telegraf: + openconfig: + username: 'ansible' + password: 'juniper123' + hosts: + 172.25.90.67: 32768 + 172.25.90.68: 32768 +``` + +> Both `snmp` and `openconfig` definition can be configure in this `data.yml` + +Some commands are available to manage repository + +- `make build-telegraf-conf` : Build telegraf configuration with template rendering +- `make build` : Build telegraf config, build docker-compose stack, start stack +- `make destroy` : Stop docker stack and remove containers +- `make start` : start an already configured stack. (Must be done if stack were built previously) +- `make stop` : stop running stack. Containers are not deleted and can be restarted with `make start` +- `make restart` : stop and start stack +- `make rebuild` : destroy and build stack +- `make {telegraf-snmp|telegraf-openconfig|influxdb|grafana}-cli` : connect to containers + +## Components + +Repository is based on docker containers and they are all managed with `docker-compose`: + +- `telegraf:1.9.1` image for openconfig polling +- `inetsix/telegraf-snmp` image for SNMP polling +- `influxdb:1.7.2` +- `grafana:grafana/grafana:5.4.2` + +## Contributors + +- [Khelil Sator](https://github.com/ksator) +- [Thomas Grimonet](https://github.com/titom73) \ No newline at end of file diff --git a/render-telegraf-configuration.py b/configuration-rendering.py similarity index 100% rename from render-telegraf-configuration.py rename to configuration-rendering.py diff --git a/dashboards/snmp-fabric-monitoring.json b/dashboards/snmp-fabric-monitoring.json index 9d121c6..8066e16 100644 --- a/dashboards/snmp-fabric-monitoring.json +++ b/dashboards/snmp-fabric-monitoring.json @@ -16,7 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1547120178787, + "iteration": 1547136972693, "links": [], "panels": [ { @@ -185,6 +185,12 @@ "params": [], "type": "last" }, + { + "params": [ + " / 100" + ], + "type": "math" + }, { "params": [ "Uptime" diff --git a/data.yml b/data.yml index 2ac6fe5..58a83ac 100644 --- a/data.yml +++ b/data.yml @@ -1,4 +1,9 @@ --- +grafana: + web: + port: "9081" + username: "super" + password: "juniper123" telegraf: snmp: community: "public" diff --git a/docker-compose.yml b/docker-compose.yml index 5261b4a..a4743c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,8 +29,9 @@ services: container_name: grafana ports: - "9081:3000" - env_file: - - 'env.grafana' + environment: + - GF_SECURITY_ADMIN_USER=super + - GF_SECURITY_ADMIN_PASSWORD=juniper123 volumes: - $PWD/configs/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml:ro - $PWD/configs/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro diff --git a/env.grafana b/env.grafana deleted file mode 100644 index eeb693d..0000000 --- a/env.grafana +++ /dev/null @@ -1,2 +0,0 @@ -GF_SECURITY_ADMIN_USER=super -GF_SECURITY_ADMIN_PASSWORD=juniper123 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1270b00 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,24 @@ +backports.ssl-match-hostname==3.5.0.1 +cached-property==1.5.1 +certifi==2018.11.29 +chardet==3.0.4 +docker==3.7.0 +docker-compose==1.23.2 +docker-pycreds==0.4.0 +dockerpty==0.4.1 +docopt==0.6.2 +enum34==1.1.6 +functools32==3.2.3.post2 +idna==2.7 +ipaddress==1.0.22 +Jinja2==2.10 +jsonschema==2.6.0 +MarkupSafe==1.1.0 +pprint==0.1 +pyaml==18.11.0 +PyYAML==3.13 +requests==2.20.1 +six==1.12.0 +texttable==0.9.1 +urllib3==1.24.1 +websocket-client==0.54.0 diff --git a/resources/snmp-bgp-status.png b/resources/snmp-bgp-status.png new file mode 100644 index 0000000..4ac9ccb Binary files /dev/null and b/resources/snmp-bgp-status.png differ diff --git a/templates/docker-compose-tig.j2 b/templates/docker-compose-tig.j2 index e83af63..8d3e697 100644 --- a/templates/docker-compose-tig.j2 +++ b/templates/docker-compose-tig.j2 @@ -42,9 +42,10 @@ services: image: grafana/grafana:5.4.2 container_name: grafana ports: - - "9081:3000" - env_file: - - 'env.grafana' + - "{{ grafana.web.port|default("9081") }}:3000" + environment: + - GF_SECURITY_ADMIN_USER={{ grafana.web.username | default("super") }} + - GF_SECURITY_ADMIN_PASSWORD={{ grafana.web.password | default("juniper123") }} volumes: - $PWD/configs/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml:ro - $PWD/configs/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro