Add credhub exporter

README.md

@@ -2,7 +2,7 @@
 
 This is a [BOSH](http://bosh.io/) release for [Prometheus](https://prometheus.io/), [Alertmanager](https://prometheus.io/docs/alerting/alertmanager/), and [Grafana](https://grafana.com/).
 
-It includes the following [prometheus exporters](https://prometheus.io/docs/instrumenting/exporters/): [Blackbox](https://github.com/prometheus/blackbox_exporter), [BOSH](https://github.com/bosh-prometheus/bosh_exporter), [BOSH TSDB](https://github.com/bosh-prometheus/bosh_tsdb_exporter), [cAdvisor](https://github.com/google/cadvisor), [Cloud Foundry](https://github.com/bosh-prometheus/cf_exporter), [Cloud Foundry Firehose](https://github.com/bosh-prometheus/firehose_exporter), [Collectd](https://github.com/prometheus/collectd_exporter), [Consul](https://github.com/prometheus/consul_exporter), [Elasticsearch](https://github.com/justwatchcom/elasticsearch_exporter), [Graphite](https://github.com/prometheus/graphite_exporter), [HAProxy](https://github.com/prometheus/haproxy_exporter), [InfluxDB](https://github.com/prometheus/influxdb_exporter), [Kubernetes](https://github.com/kubernetes/kube-state-metrics), [Memcached](https://github.com/prometheus/memcached_exporter), [MongoDB](https://github.com/dcu/mongodb_exporter), [MySQL](https://github.com/prometheus/mysqld_exporter), [NATS](https://github.com/lovoo/nats_exporter), [PostgreSQL](https://github.com/wrouesnel/postgres_exporter), [PushGateway](https://github.com/prometheus/pushgateway), [RabbitMQ](https://github.com/kbudde/rabbitmq_exporter), [Redis](https://github.com/oliver006/redis_exporter), [Shield](https://github.com/bosh-prometheus/shield_exporter), [Stackdriver](https://github.com/frodenas/stackdriver_exporter), [Statsd](https://github.com/prometheus/statsd_exporter).
+It includes the following [prometheus exporters](https://prometheus.io/docs/instrumenting/exporters/): [Blackbox](https://github.com/prometheus/blackbox_exporter), [BOSH](https://github.com/bosh-prometheus/bosh_exporter), [BOSH TSDB](https://github.com/bosh-prometheus/bosh_tsdb_exporter), [cAdvisor](https://github.com/google/cadvisor), [Cloud Foundry](https://github.com/bosh-prometheus/cf_exporter), [Cloud Foundry Firehose](https://github.com/bosh-prometheus/firehose_exporter), [Collectd](https://github.com/prometheus/collectd_exporter), [Consul](https://github.com/prometheus/consul_exporter), [Credhub](https://github.com/orange-cloudfoundry/credhub_exporter), [Elasticsearch](https://github.com/justwatchcom/elasticsearch_exporter), [Graphite](https://github.com/prometheus/graphite_exporter), [HAProxy](https://github.com/prometheus/haproxy_exporter), [InfluxDB](https://github.com/prometheus/influxdb_exporter), [Kubernetes](https://github.com/kubernetes/kube-state-metrics), [Memcached](https://github.com/prometheus/memcached_exporter), [MongoDB](https://github.com/dcu/mongodb_exporter), [MySQL](https://github.com/prometheus/mysqld_exporter), [NATS](https://github.com/lovoo/nats_exporter), [PostgreSQL](https://github.com/wrouesnel/postgres_exporter), [PushGateway](https://github.com/prometheus/pushgateway), [RabbitMQ](https://github.com/kbudde/rabbitmq_exporter), [Redis](https://github.com/oliver006/redis_exporter), [Shield](https://github.com/bosh-prometheus/shield_exporter), [Stackdriver](https://github.com/frodenas/stackdriver_exporter), [Statsd](https://github.com/prometheus/statsd_exporter).
 
 It includes the following [grafana plugins](https://grafana.com/plugins): [clock](https://github.com/grafana/clock-panel), [diagram](https://github.com/jdbranham/grafana-diagram), [histogram](https://github.com/mtanda/grafana-histogram-panel), [piechart](https://github.com/grafana/piechart-panel), [status](https://github.com/Vonage/Grafana_Status_panel), [worldmap](https://github.com/grafana/worldmap-panel), [worldping](https://github.com/raintank/worldping-app).
 

jobs/credhub_alerts/spec

@@ -0,0 +1,31 @@
+---
+name: credhub_alerts
+
+packages: []
+
+templates:
+  credhub.alerts.yml: credhub.alerts.yml
+  prometheus_credhub_exporter.alerts.yml: prometheus_credhub_exporter.alerts.yml
+
+properties:
+  credhub_alerts.credential_expire.threshold:
+    description: "Credential aging alert threshold (in days)"
+    default: 60
+  credhub_alerts.credential_expire.evaluation_time:
+    description: "Credential aging alert evaluation time"
+    default: 72h
+  credhub_alerts.certificate_expire.threshold:
+    description: "Certificate expiration alert threshold (in days)"
+    default: 33
+  credhub_alerts.certificate_expire.evaluation_time:
+    description: "Certificate expiration alert evaluation time"
+    default: 72h
+  credhub_alerts.scrape_too_old.evaluation_time:
+    description: "Scrape too old evaluation time"
+    default: 10m
+  credhub_alerts.scrape_too_old.threshold:
+    description: "Scrape too old alert threshold (in seconds)"
+    default: 3600
+  credhub_alerts.scrape_error.evaluation_time:
+    description: "Scrape error alert evaluation time"
+    default: 10m

jobs/credhub_alerts/templates/credhub.alerts.yml

@@ -0,0 +1,20 @@
+groups:
+  - name: credhub
+    rules:
+      - alert: CredhubCrendentialAging
+        expr: max(round((time() - max_over_time(credhub_credential_created_at{}[1h])) / 86400)) by (deployment, environment, path) > <%= p('credhub_alerts.credential_expire.threshold') %>
+        for: <%= p('credhub_alerts.credential_expire.evaluation_time') %>
+        labels:
+          severity: warning
+        annotations:
+          summary: "Credhub credential `{{$labels.path}}` is `{{$value}}` days old"
+          description: "Credhub credential `{{$labels.path}}` at environment `{{$labels.environment}}`, deployment `{{$labels.deployment}}` has not been rotated in the last <%= p('credhub_alerts.credential_expire.threshold') %> days"
+
+      - alert: CredhubCertificateWillExpire
+        expr: min(round((max_over_time(credhub_certificate_expires_at{}[1h]) - time()) / 86400)) by (deployment, environment, path) < <%= p('credhub_alerts.certificate_expire.threshold') %>
+        for: <%= p('credhub_alerts.certificate_expire.evaluation_time') %>
+        labels:
+          severity: critical
+        annotations:
+          summary: "Credhub certificate `{{$labels.path}}` will expire in `{{$value}}` days"
+          description: "Credhub certificate `{{$labels.path}}` will soon expire at environment `{{$labels.environment}}`, deployment `{{$labels.deployment}}` will expire in less than <%= p('credhub_alerts.certificate_expire.threshold') %> days"

jobs/credhub_alerts/templates/prometheus_credhub_exporter.alerts.yml

@@ -0,0 +1,22 @@
+groups:
+  - name: prometheus-credhub-exporter
+    rules:
+      - alert: CredhubExporterApplicationsScrapeError
+        expr: max(max_over_time(credhub_last_scrape_error{}[1h])) by(director, environment) != 0
+        for: <%= p('credhub_alerts.scrape_error.evaluation_time') %>
+        labels:
+          service: credhub-exporter
+          severity: critical
+        annotations:
+          summary: "credhub_exporter `{{$labels.environment}}/{{$labels.director}}` scrape error"
+          description: "The `credhub_exporter` at `{{$labels.environment}}/{{$labels.director}}` was unable to scrape metrics during the last <%= p('credhub_alerts.scrape_error.evaluation_time') %>"
+
+      - alert: CredhubExporterScrapeTooOld
+        expr: (time() - max(max_over_time(credhub_last_scrape_timestamp{}[1h])) by(environment, deployment)) > <%= p('credhub_alerts.scrape_too_old.threshold') %>
+        for: <%= p('credhub_alerts.scrape_too_old.evaluation_time') %>
+        labels:
+          service: credhub-exporter
+          severity: warning
+        annotations:
+          summary: "credhub_exporter `{{$labels.environment}}/{{$labels.director}}` last scrape > {{humanizeDuration <%= p('credhub_alerts.scrape_too_old.threshold') %>}} ago"
+          description: "The `credhub_exporter` at `{{$labels.environment}}/{{$labels.director}}` last scrape metrics was more than {{humanizeDuration <%= p('credhub_alerts.scrape_too_old.threshold') %>}} ago"

jobs/credhub_exporter/monit

@@ -0,0 +1,5 @@
+check process credhub_exporter
+  with pidfile /var/vcap/sys/run/credhub_exporter/credhub_exporter.pid
+  start program "/var/vcap/jobs/credhub_exporter/bin/credhub_exporter_ctl start"
+  stop program "/var/vcap/jobs/credhub_exporter/bin/credhub_exporter_ctl stop"
+  group vcap

jobs/credhub_exporter/spec

@@ -0,0 +1,64 @@
+---
+name: credhub_exporter
+
+packages:
+  - credhub_exporter
+
+templates:
+  bin/credhub_exporter_ctl: bin/credhub_exporter_ctl
+  config/web_tls_cert.pem: config/web_tls_cert.pem
+  config/web_tls_key.pem: config/web_tls_key.pem
+  config/credhub_tls_ca_cert.pem: config/credhub_tls_ca_cert.pem
+
+consumes:
+- name: credhub
+  type: credhub
+  optional: true
+
+properties:
+  credhub_exporter.credhub.api_url:
+    description: "Credhub API URL"
+  credhub_exporter.credhub.client_id:
+    description: "Credhub Client ID"
+  credhub_exporter.credhub.client_secret:
+    description: "Credhub Client Secret"
+  credhub_exporter.credhub.ca_certs:
+    description: "Credhub CA certificates (PEM format)"
+  credhub_exporter.metrics.deployment:
+    description: "Deployment name to be reported as a metric label"
+  credhub_exporter.metrics.environment:
+    description: "Environment label to be attached to metrics"
+  credhub_exporter.metrics.namespace:
+    description: "Metrics Namespace"
+  credhub_exporter.filters.generic-certificates:
+    description: "Json list of <regexp> to match generic credentials paths that may contains certificates"
+  credhub_exporter.filters.name-like:
+    description: "Fetch credentials whose name contains the query string (fetch all credentials when empty)"
+  credhub_exporter.filters.path:
+    description: "Fetch credentials that exist under the provided path"
+  credhub_exporter.log_format:
+    description: "Set the log target and format. Example: 'logger:syslog?appname=bob&local=7' or 'logger:stdout?json=true'"
+  credhub_exporter.log_level:
+    description: "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]"
+  credhub_exporter.skip_ssl_verify:
+    description: "Disable SSL Verify"
+    default: false
+  credhub_exporter.web.port:
+    description: "Port on which to expose web interface and telemetry"
+    default: "9358"
+  credhub_exporter.web.telemetry_path:
+    description: "Path under which to expose Prometheus metrics"
+  credhub_exporter.web.auth_username:
+    description: "Username for web interface basic auth"
+  credhub_exporter.web.auth_password:
+    description: "Password for web interface basic auth"
+  credhub_exporter.web.tls_cert:
+    description: "TLS certificate (PEM format). If the certificate is signed by a certificate authority, the file should be the concatenation of the server's certificate, any intermediates, and the CA's certificate"
+  credhub_exporter.web.tls_key:
+    description: "TLS private key (PEM format)"
+  env.http_proxy:
+    description: "HTTP proxy to use"
+  env.https_proxy:
+    description: "HTTPS proxy to use"
+  env.no_proxy:
+    description: "List of comma-separated hosts that should skip connecting to the proxy"

jobs/credhub_exporter/templates/bin/credhub_exporter_ctl

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+set -eu
+
+RUN_DIR=/var/vcap/sys/run/credhub_exporter
+LOG_DIR=/var/vcap/sys/log/credhub_exporter
+TMP_DIR=/var/vcap/sys/tmp/credhub_exporter
+STORE_DIR=/var/vcap/store/credhub_exporter
+mkdir -p ${RUN_DIR} ${LOG_DIR} ${TMP_DIR} ${STORE_DIR}
+
+PIDFILE=${RUN_DIR}/credhub_exporter.pid
+
+source /var/vcap/packages/credhub_exporter/common/utils.sh
+exec 1>> ${LOG_DIR}/$(basename "$0").stdout.log
+exec 2>> ${LOG_DIR}/$(basename "$0").stderr.log
+
+export PATH=/var/vcap/packages/credhub_exporter/bin:${PATH}
+
+case $1 in
+  start)
+  pid_guard ${PIDFILE} "credhub_exporter"
+    echo $$ > ${PIDFILE}
+
+    <% if_p('env.http_proxy') do |http_proxy| %>
+    export HTTP_PROXY="<%= http_proxy %>"
+    export http_proxy="<%= http_proxy %>"
+    <% end %>
+    <% if_p('env.https_proxy') do |https_proxy| %>
+    export HTTPS_PROXY="<%= https_proxy %>"
+    export https_proxy="<%= https_proxy %>"
+    <% end %>
+    <% if_p('env.no_proxy') do |no_proxy| %>
+    export NO_PROXY="<%= no_proxy %>"
+    export no_proxy="<%= no_proxy %>"
+    <% end %>
+
+    <%
+       url = ""
+       ca_certs = ""
+       if_link("credhub") do |link|
+         url      = sprintf("https://%s:%d", link.p('internal_url'), link.p('port'))
+         ca_certs = link.p('ca_certificate')
+       end.else do
+         url      = p("credhub_exporter.credhub.api_url")
+         ca_certs = p("credhub_exporter.credhub.ca_certs", "")
+      end
+    %>
+
+    exec credhub_exporter \
+      --credhub.api-url="<%= url %>" \
+      --credhub.client-id="<%= p('credhub_exporter.credhub.client_id') %>" \
+      --credhub.client-secret="<%= p('credhub_exporter.credhub.client_secret') %>" \
+      <% if not ca_certs.empty? %> \
+      --credhub.ca-certs-path="/var/vcap/jobs/credhub_exporter/config/credhub_tls_ca_cert.pem" \
+      <% end %> \
+      <% if_p('credhub_exporter.filters.generic-certificates') do |list| %> \
+      --filters.generic-certificates='<%= list.to_json %>' \
+      <% end %> \
+      <% if_p('credhub_exporter.filters.name-like') do |name| %> \
+      --filters.name-like='<%= name %>' \
+      <% end %> \
+      <% if_p('credhub_exporter.filters.path') do |path| %> \
+      --filters.path='<%= path %>' \
+      <% end %> \
+      <% if_p('credhub_exporter.log_format') do |log_format| %> \
+      --log.format="<%= log_format %>" \
+      <% end %> \
+      <% if_p('credhub_exporter.log_level') do |log_level| %> \
+      --log.level="<%= log_level %>" \
+      <% end %> \
+      --metrics.deployment-name="<%= p('credhub_exporter.metrics.deployment') %>" \
+      --metrics.environment="<%= p('credhub_exporter.metrics.environment') %>" \
+      <% if_p('credhub_exporter.metrics.namespace') do |namespace| %> \
+      --metrics.namespace="<%= namespace %>" \
+      <% end %> \
+      <% if p('credhub_exporter.skip_ssl_verify') %> \
+      --skip-ssl-verify \
+      <% end %> \
+      --web.listen-address=":<%= p('credhub_exporter.web.port') %>" \
+      <% if_p('credhub_exporter.web.telemetry_path') do |telemetry_path| %> \
+      --web.telemetry-path="<%= telemetry_path %>" \
+      <% end %> \
+      <% if_p('credhub_exporter.web.auth_username') do |auth_username| %> \
+      --web.auth.username="<%= auth_username %>" \
+      <% end %> \
+      <% if_p('credhub_exporter.web.auth_password') do |auth_password| %> \
+      --web.auth.password="<%= auth_password %>" \
+      <% end %> \
+      <% if_p('credhub_exporter.web.tls_cert', 'credhub_exporter.web.tls_key') do %> \
+      --web.tls.cert_file="/var/vcap/jobs/credhub_exporter/config/web_tls_cert.pem" \
+      --web.tls.key_file="/var/vcap/jobs/credhub_exporter/config/web_tls_key.pem" \
+      <% end %> \
+      >>  ${LOG_DIR}/credhub_exporter.stdout.log \
+      2>> ${LOG_DIR}/credhub_exporter.stderr.log
+    ;;
+
+  stop)
+    kill_and_wait ${PIDFILE}
+    ;;
+
+  *)
+    echo "Usage: $0 {start|stop}"
+    exit 1
+    ;;
+
+esac
+exit 0

jobs/credhub_exporter/templates/config/credhub_tls_ca_cert.pem

@@ -0,0 +1,5 @@
+<% if_link("credhub") do |link| %>
+<%= link.p('ca_certificate') %>
+<% end.else do %>
+<%= p("credhub_exporter.credhub.ca_certs", "") %>
+<% end %>

jobs/credhub_exporter/templates/config/web_tls_cert.pem

@@ -0,0 +1 @@
+<%= p('credhub_exporter.web.tls_cert', '') %>

jobs/credhub_exporter/templates/config/web_tls_key.pem

@@ -0,0 +1 @@
+<%= p('credhub_exporter.web.tls_key', '') %>

manifests/operators/bosh/add-credhub-exporter-uaa-clients.yml

@@ -0,0 +1,18 @@
+# Apply to your bosh-deployment
+
+# UAA client for bosh_exporter
+- type: replace
+  path: /instance_groups/name=bosh/jobs/name=uaa/properties/uaa/clients/credhub_exporter?
+  value:
+    access-token-validity: 3600
+    authorities: credhub.read,credhub.write
+    authorized-grant-types: client_credentials,refresh_token
+    override: true
+    scope: ""
+    secret: "((uaa_credhub_exporter_client_secret))"
+
+- type: replace
+  path: /variables/-
+  value:
+    name: uaa_credhub_exporter_client_secret
+    type: password

manifests/operators/monitor-credhub.yml

@@ -0,0 +1,40 @@
+# Apply ./bosh/add-credhub-exporter-uaa-clients.yml to your bosh-deployment
+
+# Exporter jobs
+- type: replace
+  path: /instance_groups/name=prometheus2/jobs/-
+  value:
+    name: credhub_exporter
+    release: prometheus
+    properties:
+      credhub_exporter:
+        credhub:
+          api_url: ((credhub_url))
+          client_id: credhub_exporter
+          client_secret: "((uaa_credhub_exporter_client_secret))"
+          ca_certs: ((credhub_ca.ca))
+        metrics:
+          deployment: ((credhub_deployment_name))
+          environment: "((metrics_environment))"
+        skip_ssl_verify: ((skip_ssl_verify))
+        filters:
+          generic-certificates:
+            - "/static/.*"
+
+- type: replace
+  path: /instance_groups/name=prometheus2/jobs/name=prometheus2/properties/prometheus/scrape_configs/-
+  value:
+    job_name: credhub
+    scrape_interval: 30m
+    scrape_timeout: 4m
+    static_configs:
+    - targets:
+      - localhost:9358
+
+# Prometheus Alerts
+- type: replace
+  path: /instance_groups/name=prometheus2/jobs/name=credhub_alerts?/release
+  value: prometheus
+- type: replace
+  path: /instance_groups/name=prometheus2/jobs/name=prometheus2/properties/prometheus/rule_files/-
+  value: /var/vcap/jobs/credhub_alerts/*.alerts.yml

packages/credhub_exporter/packaging

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Copy common utils
+mkdir -p ${BOSH_INSTALL_TARGET}/common
+cp -a ${BOSH_COMPILE_TARGET}/common/* ${BOSH_INSTALL_TARGET}/common
+
+# Extract credhub_exporter package
+mkdir -p ${BOSH_INSTALL_TARGET}/bin
+tar xzvf ${BOSH_COMPILE_TARGET}/credhub_exporter/credhub_exporter-0.1.4.linux-amd64.tar.gz
+cp -a ${BOSH_COMPILE_TARGET}/credhub_exporter-0.1.4.linux-amd64/* ${BOSH_INSTALL_TARGET}/bin

packages/credhub_exporter/spec

@@ -0,0 +1,7 @@
+---
+name: credhub_exporter
+
+files:
+  - common/utils.sh
+  - credhub_exporter/credhub_exporter-0.1.4.linux-amd64.tar.gz
+