Add CrateDB output plugin (#3210)

influxdata · Nov 9, 2017 · 62ec3e5 · 62ec3e5
1 parent 07297e8
commit 62ec3e5
Show file tree

Hide file tree

Showing 10 changed files with 522 additions and 15 deletions.
diff --git a/Godeps b/Godeps
@@ -32,7 +32,7 @@ github.com/hashicorp/consul 63d2fc68239b996096a1c55a0d4b400ea4c2583f
 github.com/influxdata/tail a395bf99fe07c233f41fba0735fa2b13b58588ea
 github.com/influxdata/toml 5d1d907f22ead1cd47adde17ceec5bda9cacaf8f
 github.com/influxdata/wlog 7c63b0a71ef8300adc255344d275e10e5c3a71ec
-github.com/jackc/pgx b84338d7d62598f75859b2b146d830b22f1b9ec8
+github.com/jackc/pgx 63f58fd32edb5684b9e9f4cfaac847c6b42b3917
 github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
 github.com/kardianos/osext c2c54e542fb797ad986b31721e1baedf214ca413
 github.com/kardianos/service 6d3a0ee7d3425d9d835debc51a0ca1ffa28f4893

diff --git a/Makefile b/Makefile
@@ -86,6 +86,12 @@ docker-run:
 		-e SLAPD_CONFIG_ROOTPW="secret" \
 		-p "389:389" -p "636:636" \
 		-d cobaugh/openldap-alpine
+	docker run --name cratedb \
+		-p "6543:5432" \
+		-d crate crate \
+		-Cnetwork.host=0.0.0.0 \
+		-Ctransport.host=localhost \
+		-Clicense.enterprise=false
 
 # Run docker containers necessary for integration tests; skipping services provided
 # by CircleCI
@@ -110,12 +116,18 @@ docker-run-circle:
 		-e SLAPD_CONFIG_ROOTPW="secret" \
 		-p "389:389" -p "636:636" \
 		-d cobaugh/openldap-alpine
+	docker run --name cratedb \
+		-p "6543:5432" \
+		-d crate crate \
+		-Cnetwork.host=0.0.0.0 \
+		-Ctransport.host=localhost \
+		-Clicense.enterprise=false
 
 docker-kill:
 	-docker kill aerospike elasticsearch kafka memcached mqtt mysql nats nsq \
-		openldap postgres rabbitmq redis riemann zookeeper
+		openldap postgres rabbitmq redis riemann zookeeper cratedb
 	-docker rm aerospike elasticsearch kafka memcached mqtt mysql nats nsq \
-		openldap postgres rabbitmq redis riemann zookeeper
+		openldap postgres rabbitmq redis riemann zookeeper cratedb
 
 .PHONY: deps telegraf telegraf.exe install test test-windows lint test-all \
 	package clean docker-run docker-run-circle docker-kill docker-image
diff --git a/README.md b/README.md
@@ -267,6 +267,7 @@ formats may be used with input plugins supporting the `data_format` option:
 * [amqp](./plugins/outputs/amqp) (rabbitmq)
 * [aws kinesis](./plugins/outputs/kinesis)
 * [aws cloudwatch](./plugins/outputs/cloudwatch)
+* [cratedb](./plugins/outputs/cratedb)
 * [datadog](./plugins/outputs/datadog)
 * [discard](./plugins/outputs/discard)
 * [elasticsearch](./plugins/outputs/elasticsearch)

diff --git a/etc/telegraf.conf b/etc/telegraf.conf
@@ -199,6 +199,19 @@
 #   namespace = "InfluxData/Telegraf"
 
 
+# # Configuration for CrateDB to send metrics to.
+# [[outputs.cratedb]]
+#   # A github.com/jackc/pgx connection string.
+#   # See https://godoc.org/github.com/jackc/pgx#ParseDSN
+#   url = "postgres://user:password@localhost/schema?sslmode=disable"
+#   # Timeout for all CrateDB queries.
+#   timeout = "5s"
+#   # Name of the table to store metrics in.
+#   table = "metrics"
+#   # If true, and the metrics table does not exist, create it automatically.
+#   table_create = true
+
+
 # # Configuration for DataDog API to send metrics to.
 # [[outputs.datadog]]
 #   ## Datadog API key

diff --git a/plugins/inputs/postgresql/postgresql_test.go b/plugins/inputs/postgresql/postgresql_test.go
@@ -51,12 +51,12 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 		"checkpoints_req",
 		"checkpoints_timed",
 		"maxwritten_clean",
-	}
-
-	int32Metrics := []string{
+		"datid",
 		"numbackends",
 	}
 
+	int32Metrics := []string{}
+
 	floatMetrics := []string{
 		"blk_read_time",
 		"blk_write_time",
@@ -66,7 +66,6 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 
 	stringMetrics := []string{
 		"datname",
-		"datid",
 	}
 
 	metricsCounted := 0

diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go
@@ -53,20 +53,19 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 		"temp_files",
 		"temp_bytes",
 		"deadlocks",
-	}
-
-	int32Metrics := []string{
 		"numbackends",
+		"datid",
 	}
 
+	int32Metrics := []string{}
+
 	floatMetrics := []string{
 		"blk_read_time",
 		"blk_write_time",
 	}
 
 	stringMetrics := []string{
 		"datname",
-		"datid",
 	}
 
 	metricsCounted := 0
@@ -175,20 +174,19 @@ func TestPostgresqlFieldOutput(t *testing.T) {
 		"temp_files",
 		"temp_bytes",
 		"deadlocks",
-	}
-
-	int32Metrics := []string{
 		"numbackends",
+		"datid",
 	}
 
+	int32Metrics := []string{}
+
 	floatMetrics := []string{
 		"blk_read_time",
 		"blk_write_time",
 	}
 
 	stringMetrics := []string{
 		"datname",
-		"datid",
 	}
 
 	for _, field := range intMetrics {

diff --git a/plugins/outputs/all/all.go b/plugins/outputs/all/all.go
@@ -4,6 +4,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/outputs/amon"
 	_ "github.com/influxdata/telegraf/plugins/outputs/amqp"
 	_ "github.com/influxdata/telegraf/plugins/outputs/cloudwatch"
+	_ "github.com/influxdata/telegraf/plugins/outputs/cratedb"
 	_ "github.com/influxdata/telegraf/plugins/outputs/datadog"
 	_ "github.com/influxdata/telegraf/plugins/outputs/discard"
 	_ "github.com/influxdata/telegraf/plugins/outputs/elasticsearch"

diff --git a/plugins/outputs/cratedb/README.md b/plugins/outputs/cratedb/README.md
@@ -0,0 +1,38 @@
+# CrateDB Output Plugin for Telegraf
+
+This plugin writes to [CrateDB](https://crate.io/) via its [PostgreSQL protocol](https://crate.io/docs/crate/reference/protocols/postgres.html).
+
+## Table Schema
+
+The plugin requires a table with the following schema.
+
+
+```sql
+CREATE TABLE my_metrics (
+  "hash_id" LONG INDEX OFF,
+  "timestamp" TIMESTAMP,
+  "name" STRING,
+  "tags" OBJECT(DYNAMIC),
+  "fields" OBJECT(DYNAMIC),
+  PRIMARY KEY ("timestamp", "hash_id","day")
+) PARTITIONED BY("day");
+```
+
+The plugin can create this table for you automatically via the `table_create`
+config option, see below.
+
+## Configuration
+
+```toml
+# Configuration for CrateDB to send metrics to.
+[[outputs.cratedb]]
+  # A github.com/jackc/pgx connection string.
+  # See https://godoc.org/github.com/jackc/pgx#ParseDSN
+  url = "postgres://user:password@localhost/schema?sslmode=disable"
+  # Timeout for all CrateDB queries.
+  timeout = "5s"
+  # Name of the table to store metrics in.
+  table = "metrics"
+  # If true, and the metrics table does not exist, create it automatically.
+  table_create = true
+```