forked from cloudfoundry/prometheus-boshrelease
-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[INTERNAL] Add credhub exporter (#6)
* add credhub exporter job and alerts
- Loading branch information
Showing
15 changed files
with
334 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| --- | ||
| name: credhub_alerts | ||
|
|
||
| packages: [] | ||
|
|
||
| templates: | ||
| credhub.alerts.yml: credhub.alerts.yml | ||
| prometheus_credhub_exporter.alerts.yml: prometheus_credhub_exporter.alerts.yml | ||
|
|
||
| properties: | ||
| credhub_alerts.credential_expire.threshold: | ||
| description: "Credential aging alert threshold (in days)" | ||
| default: 60 | ||
| credhub_alerts.credential_expire.evaluation_time: | ||
| description: "Credential aging alert evaluation time" | ||
| default: 72h | ||
| credhub_alerts.certificate_expire.threshold: | ||
| description: "Certificate expiration alert threshold (in days)" | ||
| default: 33 | ||
| credhub_alerts.certificate_expire.evaluation_time: | ||
| description: "Certificate expiration alert evaluation time" | ||
| default: 72h | ||
| credhub_alerts.scrape_too_old.evaluation_time: | ||
| description: "Scrape too old evaluation time" | ||
| default: 10m | ||
| credhub_alerts.scrape_too_old.threshold: | ||
| description: "Scrape too old alert threshold (in seconds)" | ||
| default: 3600 | ||
| credhub_alerts.scrape_error.evaluation_time: | ||
| description: "Scrape error alert evaluation time" | ||
| default: 10m |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| groups: | ||
| - name: credhub | ||
| rules: | ||
| - alert: CredhubCrendentialAging | ||
| expr: max(round((time() - max_over_time(credhub_credential_created_at{}[1h])) / 86400)) by (deployment, environment, path) > <%= p('credhub_alerts.credential_expire.threshold') %> | ||
| for: <%= p('credhub_alerts.credential_expire.evaluation_time') %> | ||
| labels: | ||
| severity: warning | ||
| annotations: | ||
| summary: "Credhub credential `{{$labels.path}}` is `{{$value}}` days old" | ||
| description: "Credhub credential `{{$labels.path}}` at environment `{{$labels.environment}}`, deployment `{{$labels.deployment}}` has not been rotated in the last <%= p('credhub_alerts.credential_expire.threshold') %> days" | ||
|
|
||
| - alert: CredhubCertificateWillExpire | ||
| expr: min(round((max_over_time(credhub_certificate_expires_at{}[1h]) - time()) / 86400)) by (deployment, environment, path) < <%= p('credhub_alerts.certificate_expire.threshold') %> | ||
| for: <%= p('credhub_alerts.certificate_expire.evaluation_time') %> | ||
| labels: | ||
| severity: critical | ||
| annotations: | ||
| summary: "Credhub certificate `{{$labels.path}}` will expire in `{{$value}}` days" | ||
| description: "Credhub certificate `{{$labels.path}}` will soon expire at environment `{{$labels.environment}}`, deployment `{{$labels.deployment}}` will expire in less than <%= p('credhub_alerts.certificate_expire.threshold') %> days" |
22 changes: 22 additions & 0 deletions
22
jobs/credhub_alerts/templates/prometheus_credhub_exporter.alerts.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| groups: | ||
| - name: prometheus-credhub-exporter | ||
| rules: | ||
| - alert: CredhubExporterApplicationsScrapeError | ||
| expr: max(max_over_time(credhub_last_scrape_error{}[1h])) by(director, environment) != 0 | ||
| for: <%= p('credhub_alerts.scrape_error.evaluation_time') %> | ||
| labels: | ||
| service: credhub-exporter | ||
| severity: critical | ||
| annotations: | ||
| summary: "credhub_exporter `{{$labels.environment}}/{{$labels.director}}` scrape error" | ||
| description: "The `credhub_exporter` at `{{$labels.environment}}/{{$labels.director}}` was unable to scrape metrics during the last <%= p('credhub_alerts.scrape_error.evaluation_time') %>" | ||
|
|
||
| - alert: CredhubExporterScrapeTooOld | ||
| expr: (time() - max(max_over_time(credhub_last_scrape_timestamp{}[1h])) by(environment, deployment)) > <%= p('credhub_alerts.scrape_too_old.threshold') %> | ||
| for: <%= p('credhub_alerts.scrape_too_old.evaluation_time') %> | ||
| labels: | ||
| service: credhub-exporter | ||
| severity: warning | ||
| annotations: | ||
| summary: "credhub_exporter `{{$labels.environment}}/{{$labels.director}}` last scrape > {{humanizeDuration <%= p('credhub_alerts.scrape_too_old.threshold') %>}} ago" | ||
| description: "The `credhub_exporter` at `{{$labels.environment}}/{{$labels.director}}` last scrape metrics was more than {{humanizeDuration <%= p('credhub_alerts.scrape_too_old.threshold') %>}} ago" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| check process credhub_exporter | ||
| with pidfile /var/vcap/sys/run/credhub_exporter/credhub_exporter.pid | ||
| start program "/var/vcap/jobs/credhub_exporter/bin/credhub_exporter_ctl start" | ||
| stop program "/var/vcap/jobs/credhub_exporter/bin/credhub_exporter_ctl stop" | ||
| group vcap |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| --- | ||
| name: credhub_exporter | ||
|
|
||
| packages: | ||
| - credhub_exporter | ||
|
|
||
| templates: | ||
| bin/credhub_exporter_ctl: bin/credhub_exporter_ctl | ||
| config/web_tls_cert.pem: config/web_tls_cert.pem | ||
| config/web_tls_key.pem: config/web_tls_key.pem | ||
| config/credhub_tls_ca_cert.pem: config/credhub_tls_ca_cert.pem | ||
|
|
||
| consumes: | ||
| - name: credhub | ||
| type: credhub | ||
| optional: true | ||
|
|
||
| properties: | ||
| credhub_exporter.credhub.api_url: | ||
| description: "Credhub API URL" | ||
| credhub_exporter.credhub.client_id: | ||
| description: "Credhub Client ID" | ||
| credhub_exporter.credhub.client_secret: | ||
| description: "Credhub Client Secret" | ||
| credhub_exporter.credhub.ca_certs: | ||
| description: "Credhub CA certificates (PEM format)" | ||
| credhub_exporter.metrics.deployment: | ||
| description: "Deployment name to be reported as a metric label" | ||
| credhub_exporter.metrics.environment: | ||
| description: "Environment label to be attached to metrics" | ||
| credhub_exporter.metrics.namespace: | ||
| description: "Metrics Namespace" | ||
| credhub_exporter.filters.generic-certificates: | ||
| description: "Json list of <regexp> to match generic credentials paths that may contains certificates" | ||
| credhub_exporter.filters.name-like: | ||
| description: "Fetch credentials whose name contains the query string (fetch all credentials when empty)" | ||
| credhub_exporter.filters.path: | ||
| description: "Fetch credentials that exist under the provided path" | ||
| credhub_exporter.log_format: | ||
| description: "Set the log target and format. Example: 'logger:syslog?appname=bob&local=7' or 'logger:stdout?json=true'" | ||
| credhub_exporter.log_level: | ||
| description: "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]" | ||
| credhub_exporter.skip_ssl_verify: | ||
| description: "Disable SSL Verify" | ||
| default: false | ||
| credhub_exporter.web.port: | ||
| description: "Port on which to expose web interface and telemetry" | ||
| default: "9358" | ||
| credhub_exporter.web.telemetry_path: | ||
| description: "Path under which to expose Prometheus metrics" | ||
| credhub_exporter.web.auth_username: | ||
| description: "Username for web interface basic auth" | ||
| credhub_exporter.web.auth_password: | ||
| description: "Password for web interface basic auth" | ||
| credhub_exporter.web.tls_cert: | ||
| description: "TLS certificate (PEM format). If the certificate is signed by a certificate authority, the file should be the concatenation of the server's certificate, any intermediates, and the CA's certificate" | ||
| credhub_exporter.web.tls_key: | ||
| description: "TLS private key (PEM format)" | ||
| env.http_proxy: | ||
| description: "HTTP proxy to use" | ||
| env.https_proxy: | ||
| description: "HTTPS proxy to use" | ||
| env.no_proxy: | ||
| description: "List of comma-separated hosts that should skip connecting to the proxy" |
107 changes: 107 additions & 0 deletions
107
jobs/credhub_exporter/templates/bin/credhub_exporter_ctl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -eu | ||
|
|
||
| RUN_DIR=/var/vcap/sys/run/credhub_exporter | ||
| LOG_DIR=/var/vcap/sys/log/credhub_exporter | ||
| TMP_DIR=/var/vcap/sys/tmp/credhub_exporter | ||
| STORE_DIR=/var/vcap/store/credhub_exporter | ||
| mkdir -p ${RUN_DIR} ${LOG_DIR} ${TMP_DIR} ${STORE_DIR} | ||
|
|
||
| PIDFILE=${RUN_DIR}/credhub_exporter.pid | ||
|
|
||
| source /var/vcap/packages/credhub_exporter/common/utils.sh | ||
| exec 1>> ${LOG_DIR}/$(basename "$0").stdout.log | ||
| exec 2>> ${LOG_DIR}/$(basename "$0").stderr.log | ||
|
|
||
| export PATH=/var/vcap/packages/credhub_exporter/bin:${PATH} | ||
|
|
||
| case $1 in | ||
| start) | ||
| pid_guard ${PIDFILE} "credhub_exporter" | ||
| echo $$ > ${PIDFILE} | ||
|
|
||
| <% if_p('env.http_proxy') do |http_proxy| %> | ||
| export HTTP_PROXY="<%= http_proxy %>" | ||
| export http_proxy="<%= http_proxy %>" | ||
| <% end %> | ||
| <% if_p('env.https_proxy') do |https_proxy| %> | ||
| export HTTPS_PROXY="<%= https_proxy %>" | ||
| export https_proxy="<%= https_proxy %>" | ||
| <% end %> | ||
| <% if_p('env.no_proxy') do |no_proxy| %> | ||
| export NO_PROXY="<%= no_proxy %>" | ||
| export no_proxy="<%= no_proxy %>" | ||
| <% end %> | ||
|
|
||
| <% | ||
| url = "" | ||
| ca_certs = "" | ||
| if_link("credhub") do |link| | ||
| url = sprintf("https://%s:%d", link.p('internal_url'), link.p('port')) | ||
| ca_certs = link.p('ca_certificate') | ||
| end.else do | ||
| url = p("credhub_exporter.credhub.api_url") | ||
| ca_certs = p("credhub_exporter.credhub.ca_certs", "") | ||
| end | ||
| %> | ||
|
|
||
| exec credhub_exporter \ | ||
| --credhub.api-url="<%= url %>" \ | ||
| --credhub.client-id="<%= p('credhub_exporter.credhub.client_id') %>" \ | ||
| --credhub.client-secret="<%= p('credhub_exporter.credhub.client_secret') %>" \ | ||
| <% if not ca_certs.empty? %> \ | ||
| --credhub.ca-certs-path="/var/vcap/jobs/credhub_exporter/config/credhub_tls_ca_cert.pem" \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.filters.generic-certificates') do |list| %> \ | ||
| --filters.generic-certificates='<%= list.to_json %>' \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.filters.name-like') do |name| %> \ | ||
| --filters.name-like='<%= name %>' \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.filters.path') do |path| %> \ | ||
| --filters.path='<%= path %>' \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.log_format') do |log_format| %> \ | ||
| --log.format="<%= log_format %>" \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.log_level') do |log_level| %> \ | ||
| --log.level="<%= log_level %>" \ | ||
| <% end %> \ | ||
| --metrics.deployment-name="<%= p('credhub_exporter.metrics.deployment') %>" \ | ||
| --metrics.environment="<%= p('credhub_exporter.metrics.environment') %>" \ | ||
| <% if_p('credhub_exporter.metrics.namespace') do |namespace| %> \ | ||
| --metrics.namespace="<%= namespace %>" \ | ||
| <% end %> \ | ||
| <% if p('credhub_exporter.skip_ssl_verify') %> \ | ||
| --skip-ssl-verify \ | ||
| <% end %> \ | ||
| --web.listen-address=":<%= p('credhub_exporter.web.port') %>" \ | ||
| <% if_p('credhub_exporter.web.telemetry_path') do |telemetry_path| %> \ | ||
| --web.telemetry-path="<%= telemetry_path %>" \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.web.auth_username') do |auth_username| %> \ | ||
| --web.auth.username="<%= auth_username %>" \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.web.auth_password') do |auth_password| %> \ | ||
| --web.auth.password="<%= auth_password %>" \ | ||
| <% end %> \ | ||
| <% if_p('credhub_exporter.web.tls_cert', 'credhub_exporter.web.tls_key') do %> \ | ||
| --web.tls.cert_file="/var/vcap/jobs/credhub_exporter/config/web_tls_cert.pem" \ | ||
| --web.tls.key_file="/var/vcap/jobs/credhub_exporter/config/web_tls_key.pem" \ | ||
| <% end %> \ | ||
| >> ${LOG_DIR}/credhub_exporter.stdout.log \ | ||
| 2>> ${LOG_DIR}/credhub_exporter.stderr.log | ||
| ;; | ||
|
|
||
| stop) | ||
| kill_and_wait ${PIDFILE} | ||
| ;; | ||
|
|
||
| *) | ||
| echo "Usage: $0 {start|stop}" | ||
| exit 1 | ||
| ;; | ||
|
|
||
| esac | ||
| exit 0 |
5 changes: 5 additions & 0 deletions
5
jobs/credhub_exporter/templates/config/credhub_tls_ca_cert.pem
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| <% if_link("credhub") do |link| %> | ||
| <%= link.p('ca_certificate') %> | ||
| <% end.else do %> | ||
| <%= p("credhub_exporter.credhub.ca_certs", "") %> | ||
| <% end %> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| <%= p('credhub_exporter.web.tls_cert', '') %> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| <%= p('credhub_exporter.web.tls_key', '') %> |
18 changes: 18 additions & 0 deletions
18
manifests/operators/bosh/add-credhub-exporter-uaa-clients.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| # Apply to your bosh-deployment | ||
|
|
||
| # UAA client for bosh_exporter | ||
| - type: replace | ||
| path: /instance_groups/name=bosh/jobs/name=uaa/properties/uaa/clients/credhub_exporter? | ||
| value: | ||
| access-token-validity: 3600 | ||
| authorities: credhub.read,credhub.write | ||
| authorized-grant-types: client_credentials,refresh_token | ||
| override: true | ||
| scope: "" | ||
| secret: "((uaa_credhub_exporter_client_secret))" | ||
|
|
||
| - type: replace | ||
| path: /variables/- | ||
| value: | ||
| name: uaa_credhub_exporter_client_secret | ||
| type: password |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| # Apply ./bosh/add-credhub-exporter-uaa-clients.yml to your bosh-deployment | ||
|
|
||
| # Exporter jobs | ||
| - type: replace | ||
| path: /instance_groups/name=prometheus2/jobs/- | ||
| value: | ||
| name: credhub_exporter | ||
| release: prometheus | ||
| properties: | ||
| credhub_exporter: | ||
| credhub: | ||
| api_url: ((credhub_url)) | ||
| client_id: credhub_exporter | ||
| client_secret: "((uaa_credhub_exporter_client_secret))" | ||
| ca_certs: ((credhub_ca.ca)) | ||
| metrics: | ||
| deployment: ((credhub_deployment_name)) | ||
| environment: "((metrics_environment))" | ||
| skip_ssl_verify: ((skip_ssl_verify)) | ||
| filters: | ||
| generic-certificates: | ||
| - "/static/.*" | ||
|
|
||
| - type: replace | ||
| path: /instance_groups/name=prometheus2/jobs/name=prometheus2/properties/prometheus/scrape_configs/- | ||
| value: | ||
| job_name: credhub | ||
| scrape_interval: 30m | ||
| scrape_timeout: 4m | ||
| static_configs: | ||
| - targets: | ||
| - localhost:9358 | ||
|
|
||
| # Prometheus Alerts | ||
| - type: replace | ||
| path: /instance_groups/name=prometheus2/jobs/name=credhub_alerts?/release | ||
| value: prometheus | ||
| - type: replace | ||
| path: /instance_groups/name=prometheus2/jobs/name=prometheus2/properties/prometheus/rule_files/- | ||
| value: /var/vcap/jobs/credhub_alerts/*.alerts.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -eux | ||
|
|
||
| # Copy common utils | ||
| mkdir -p ${BOSH_INSTALL_TARGET}/common | ||
| cp -a ${BOSH_COMPILE_TARGET}/common/* ${BOSH_INSTALL_TARGET}/common | ||
|
|
||
| # Extract credhub_exporter package | ||
| mkdir -p ${BOSH_INSTALL_TARGET}/bin | ||
| tar xzvf ${BOSH_COMPILE_TARGET}/credhub_exporter/credhub_exporter-0.1.4.linux-amd64.tar.gz | ||
| cp -a ${BOSH_COMPILE_TARGET}/credhub_exporter-0.1.4.linux-amd64/* ${BOSH_INSTALL_TARGET}/bin |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| --- | ||
| name: credhub_exporter | ||
|
|
||
| files: | ||
| - common/utils.sh | ||
| - credhub_exporter/credhub_exporter-0.1.4.linux-amd64.tar.gz | ||
|
|