Add docker log plugin (influxdata#4773)

plugins/inputs/all/all.go

@@ -32,6 +32,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/dmcache"
 	_ "github.com/influxdata/telegraf/plugins/inputs/dns_query"
 	_ "github.com/influxdata/telegraf/plugins/inputs/docker"
+	_ "github.com/influxdata/telegraf/plugins/inputs/docker_log"
 	_ "github.com/influxdata/telegraf/plugins/inputs/dovecot"
 	_ "github.com/influxdata/telegraf/plugins/inputs/ecs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"

plugins/inputs/docker_log/README.md

@@ -0,0 +1,60 @@
+# Docker Log Input Plugin
+
+The docker log plugin uses the Docker Engine API to get logs on running
+docker containers.
+
+The docker plugin uses the [Official Docker Client](https://github.com/moby/moby/tree/master/client)
+to gather logs from the [Engine API](https://docs.docker.com/engine/api/v1.24/).
+Note: This plugin works only for containers with the `local` or  `json-file` or `journald` logging driver.
+### Configuration:
+
+```toml
+# Read metrics about docker containers
+[[inputs.docker_log]]
+  ## Docker Endpoint
+  ##   To use TCP, set endpoint = "tcp://[ip]:[port]"
+  ##   To use environment variables (ie, docker-machine), set endpoint = "ENV"
+  endpoint = "unix:///var/run/docker.sock"
+
+  ## Containers to include and exclude. Collect all if empty. Globs accepted.
+  container_name_include = []
+  container_name_exclude = []
+
+  ## Container states to include and exclude. Globs accepted.
+  ## When empty only containers in the "running" state will be captured.
+  # container_state_include = []
+  # container_state_exclude = []
+
+  ## docker labels to include and exclude as tags.  Globs accepted.
+  ## Note that an empty array for both will include all labels as tags
+  docker_label_include = []
+  docker_label_exclude = []
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+```
+
+#### Environment Configuration
+
+When using the `"ENV"` endpoint, the connection is configured using the
+[cli Docker environment variables](https://godoc.org/github.com/moby/moby/client#NewEnvClient).
+
+### Metrics:
+
+- docker_log
+  - tags:
+    - container_id
+    - container_name
+    - stream
+  - fields:
+    - message
+### Example Output:
+
+```
+docker_log,com.docker.compose.config-hash=e19e13df8fd01ba2d7c1628158fca45cc91afbbe9661b2d30550547eb53a861e,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=distribution,com.docker.compose.service=influxdb,com.docker.compose.version=1.21.2,containerId=fce475bbfa4c8380ff858d5d767f78622ca6de955b525477624c2b7896a5b8e4,containerName=aicon-influxdb,host=prash-laptop,logType=stderr log=" [httpd] 172.23.0.2 - aicon_admin [13/Apr/2019:08:35:53 +0000] \"POST /query?db=&q=SHOW+SUBSCRIPTIONS HTTP/1.1\" 200 232 \"-\" \"KapacitorInfluxDBClient\" 2661bc9c-5dc7-11e9-82f8-0242ac170007 1360\n" 1555144553541000000
+docker_log,com.docker.compose.config-hash=fd91b3b096c7ab346971c681b88fe1357c609dcc6850e4ea5b1287ad28a57e5d,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=distribution,com.docker.compose.service=kapacitor,com.docker.compose.version=1.21.2,containerId=6514d1cf6d19e7ecfedf894941f0a2ea21b8aac5e6f48e64f19dbc9bb2805a25,containerName=aicon-kapacitor,host=prash-laptop,logType=stderr log=" ts=2019-04-13T08:36:00.019Z lvl=info msg=\"http request\" service=http host=172.23.0.7 username=- start=2019-04-13T08:36:00.013835165Z method=POST uri=/write?consistency=&db=_internal&precision=ns&rp=monitor protocol=HTTP/1.1 status=204 referer=- user-agent=InfluxDBClient request-id=2a3eb481-5dc7-11e9-825b-000000000000 duration=5.814404ms\n" 1555144560024000000
+```

plugins/inputs/docker_log/client.go

@@ -0,0 +1,63 @@
+package docker_log
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net/http"
+
+	"github.com/docker/docker/api/types"
+	docker "github.com/docker/docker/client"
+)
+
+/*This file is inherited from telegraf docker input plugin*/
+var (
+	version        = "1.24"
+	defaultHeaders = map[string]string{"User-Agent": "engine-api-cli-1.0"}
+)
+
+type Client interface {
+	ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error)
+	ContainerLogs(ctx context.Context, containerID string, options types.ContainerLogsOptions) (io.ReadCloser, error)
+	ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error)
+}
+
+func NewEnvClient() (Client, error) {
+	client, err := docker.NewClientWithOpts(docker.FromEnv)
+	if err != nil {
+		return nil, err
+	}
+	return &SocketClient{client}, nil
+}
+
+func NewClient(host string, tlsConfig *tls.Config) (Client, error) {
+	transport := &http.Transport{
+		TLSClientConfig: tlsConfig,
+	}
+	httpClient := &http.Client{Transport: transport}
+	client, err := docker.NewClientWithOpts(
+		docker.WithHTTPHeaders(defaultHeaders),
+		docker.WithHTTPClient(httpClient),
+		docker.WithVersion(version),
+		docker.WithHost(host))
+
+	if err != nil {
+		return nil, err
+	}
+	return &SocketClient{client}, nil
+}
+
+type SocketClient struct {
+	client *docker.Client
+}
+
+func (c *SocketClient) ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) {
+	return c.client.ContainerList(ctx, options)
+}
+
+func (c *SocketClient) ContainerLogs(ctx context.Context, containerID string, options types.ContainerLogsOptions) (io.ReadCloser, error) {
+	return c.client.ContainerLogs(ctx, containerID, options)
+}
+func (c *SocketClient) ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) {
+	return c.client.ContainerInspect(ctx, containerID)
+}