From 5b0f68fdc15bedc5f5a1f966b274effc55811fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 6 Jul 2018 13:09:50 +0200 Subject: [PATCH 1/7] Add `bearer_token_file` paramter to HTTP helper This change allows to load barer tokens from files in modules using the HTTP helper. For instance: ``` - module: kubernetes metricsets: - pod bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt ``` --- .../appender/kubernetes/token/token.go | 2 +- metricbeat/helper/http.go | 36 +++++++++++-- metricbeat/helper/http_test.go | 54 +++++++++++++++++++ 3 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 metricbeat/helper/http_test.go diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index 2b6deb3bd93..5471f874ca9 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -41,7 +41,7 @@ type tokenAppender struct { // NewTokenAppender creates a token appender that can append a bearer token required to authenticate with // protected endpoints func NewTokenAppender(cfg *common.Config) (autodiscover.Appender, error) { - cfgwarn.Beta("The token appender is beta") + cfgwarn.Deprecate("7.0.0", "token appender is deprecated in favor of bearer_token_file config parameter") conf := defaultConfig() err := cfg.Unpack(&conf) diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index 495df91b1f5..b01fa781885 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -27,6 +27,8 @@ import ( "net/http" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common/transport/tlscommon" "github.com/elastic/beats/libbeat/outputs/transport" "github.com/elastic/beats/metricbeat/mb" @@ -44,9 +46,10 @@ type HTTP struct { // NewHTTP creates new http helper func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) { config := struct { - TLS *tlscommon.Config `config:"ssl"` - Timeout time.Duration `config:"timeout"` - Headers map[string]string `config:"headers"` + TLS *tlscommon.Config `config:"ssl"` + Timeout time.Duration `config:"timeout"` + Headers map[string]string `config:"headers"` + BearerTokenFile string `config:"bearer_token_file"` }{} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -56,6 +59,14 @@ func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) { config.Headers = map[string]string{} } + if config.BearerTokenFile != "" { + header, err := getAuthHeaderFromToken(config.BearerTokenFile) + if err != nil { + return nil, err + } + config.Headers["Authorization"] = header + } + tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) if err != nil { return nil, err @@ -179,3 +190,22 @@ func (h *HTTP) FetchJSON() (map[string]interface{}, error) { return data, nil } + +// getAuthHeaderFromToken reads a bearer authorizaiton token from the given file +func getAuthHeaderFromToken(path string) (string, error) { + var token string + + b, err := ioutil.ReadFile(path) + if err != nil { + return "", errors.Wrap(err, "reading bearer token file") + } + + if len(b) != 0 { + if b[len(b)-1] == '\n' { + b = b[0 : len(b)-1] + } + token = fmt.Sprintf("Bearer %s", string(b)) + } + + return token, nil +} diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go new file mode 100644 index 00000000000..685e990cbc4 --- /dev/null +++ b/metricbeat/helper/http_test.go @@ -0,0 +1,54 @@ +package helper + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetAuthHeaderFromToken(t *testing.T) { + tests := []struct { + Name, Content, Expected string + }{ + { + "Test a token is read", + "testtoken", + "Bearer testtoken", + }, + { + "Test a token is trimmed", + "testtoken\n", + "Bearer testtoken", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + content := []byte(test.Content) + tmpfile, err := ioutil.TempFile("", "token") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + if _, err := tmpfile.Write(content); err != nil { + t.Fatal(err) + } + if err := tmpfile.Close(); err != nil { + t.Fatal(err) + } + + header, err := getAuthHeaderFromToken(tmpfile.Name()) + assert.NoError(t, err) + assert.Equal(t, test.Expected, header) + }) + } +} + +func TestGetAuthHeaderFromTokenNoFile(t *testing.T) { + header, err := getAuthHeaderFromToken("nonexistingfile") + assert.Equal(t, "", header) + assert.Error(t, err) +} From 4134f37f7b43968ff3d3508afb5eca34d9e3b8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 6 Jul 2018 13:31:10 +0200 Subject: [PATCH 2/7] Add documentation --- metricbeat/module/kubernetes/_meta/config.yml | 5 ++++- metricbeat/module/prometheus/_meta/config.yml | 5 +++++ metricbeat/modules.d/kubernetes.yml.disabled | 5 ++++- metricbeat/modules.d/prometheus.yml.disabled | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 5492b0a3b99..f5e917eadc1 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -7,7 +7,10 @@ # - container # - volume period: 10s - hosts: ["localhost:10255"] + hosts: ["localhost:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 176fd562f01..1ff113c2d0f 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -7,3 +7,8 @@ #namespace: example #username: "user" #password: "secret" + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 5c569025d57..c9b1eaf8d94 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -10,7 +10,10 @@ # - container # - volume period: 10s - hosts: ["localhost:10255"] + hosts: ["localhost:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index 21684d3adf1..c13619278c2 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -10,3 +10,8 @@ #namespace: example #username: "user" #password: "secret" + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt From 404c4fcb4decd01bf0935e833efa17c54bea99c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 9 Jul 2018 11:43:54 +0200 Subject: [PATCH 3/7] Add docs --- metricbeat/docs/metricbeat-options.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metricbeat/docs/metricbeat-options.asciidoc b/metricbeat/docs/metricbeat-options.asciidoc index 426dccafd4f..c188df1405f 100644 --- a/metricbeat/docs/metricbeat-options.asciidoc +++ b/metricbeat/docs/metricbeat-options.asciidoc @@ -216,3 +216,9 @@ The username to use for basic authentication. ==== `password` The password to use for basic authentication. + +[float] +==== `bearer_token_file` + +If defined, Metricbeat will read the contents of the file once at initialization +and then use the value in an HTTP Authorization header. \ No newline at end of file From 10193120a137837b038f219cc559c483b10c89f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 9 Jul 2018 11:56:20 +0200 Subject: [PATCH 4/7] Update CHANGELOG.asciidoc --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 00f8b75cd2d..b5ecd972584 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -300,6 +300,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Add TLS support to MongoDB module. {pull}7401[7401] - Added Traefik module with health metricset. {pull}7413[7413] - Add Elasticsearch ml_job metricsets. {pull}7196[7196] +- Add support for bearer token files to HTTP helper. {pull}7527[7527] *Packetbeat* From 3965a1dd12692f75ad5b6967be451cffd69e2c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 9 Jul 2018 13:10:48 +0200 Subject: [PATCH 5/7] Add missing license header --- metricbeat/helper/http_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go index 685e990cbc4..a3e5d615e4d 100644 --- a/metricbeat/helper/http_test.go +++ b/metricbeat/helper/http_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package helper import ( From 5bd726e4a021e53f1544ad83e4ea92a4379547e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 10 Jul 2018 13:42:02 +0200 Subject: [PATCH 6/7] Update reference configs --- metricbeat/docs/modules/kubernetes.asciidoc | 4 +++- metricbeat/docs/modules/prometheus.asciidoc | 6 +++++- metricbeat/metricbeat.reference.yml | 8 +++++++- metricbeat/module/kubernetes/_meta/config.reference.yml | 4 +++- metricbeat/module/kubernetes/_meta/config.yml | 3 --- metricbeat/module/prometheus/_meta/config.reference.yml | 5 +++++ metricbeat/modules.d/kubernetes.yml.disabled | 3 --- 7 files changed, 23 insertions(+), 10 deletions(-) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index a3b8cc2fcc7..11cce59d264 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -35,7 +35,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:10255"] enabled: true - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index d25d1618726..5c83c265bc2 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -37,7 +37,11 @@ metricbeat.modules: hosts: ["localhost:9090"] #metrics_path: /metrics #namespace: example ----- + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt---- This module supports TLS connection when using `ssl` config field, as described in <>. It also supports the options described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index ed2f25bb766..4279279410b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -361,7 +361,9 @@ metricbeat.modules: period: 10s hosts: ["localhost:10255"] enabled: true - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" @@ -542,6 +544,10 @@ metricbeat.modules: #metrics_path: /metrics #namespace: example + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #------------------------------ RabbitMQ Module ------------------------------ - module: rabbitmq metricsets: ["node", "queue", "connection"] diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index b5ef0867653..5b87632596b 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -9,7 +9,9 @@ period: 10s hosts: ["localhost:10255"] enabled: true - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #ssl.certificate: "/etc/pki/client/cert.pem" #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index f5e917eadc1..896092f7b37 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -11,9 +11,6 @@ bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - #ssl.certificate: "/etc/pki/client/cert.pem" - #ssl.key: "/etc/pki/client/cert.key" #username: "user" #password: "secret" diff --git a/metricbeat/module/prometheus/_meta/config.reference.yml b/metricbeat/module/prometheus/_meta/config.reference.yml index dda648a4243..89f17c6ea64 100644 --- a/metricbeat/module/prometheus/_meta/config.reference.yml +++ b/metricbeat/module/prometheus/_meta/config.reference.yml @@ -13,3 +13,8 @@ hosts: ["localhost:9090"] #metrics_path: /metrics #namespace: example + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt \ No newline at end of file diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index c9b1eaf8d94..c78dfa15d47 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -14,9 +14,6 @@ bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - #ssl.certificate: "/etc/pki/client/cert.pem" - #ssl.key: "/etc/pki/client/cert.key" #username: "user" #password: "secret" From 833d0a570419fc54d0244cae65dce3d86c35b4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 11 Jul 2018 16:22:34 +0200 Subject: [PATCH 7/7] fix docs --- metricbeat/docs/modules/prometheus.asciidoc | 3 ++- metricbeat/metricbeat.reference.yml | 1 + metricbeat/module/prometheus/_meta/config.reference.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 5c83c265bc2..cada7468f0d 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -41,7 +41,8 @@ metricbeat.modules: # This can be used for service account based authorization: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt---- + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt +---- This module supports TLS connection when using `ssl` config field, as described in <>. It also supports the options described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4279279410b..750ebeb6e15 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -548,6 +548,7 @@ metricbeat.modules: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + #------------------------------ RabbitMQ Module ------------------------------ - module: rabbitmq metricsets: ["node", "queue", "connection"] diff --git a/metricbeat/module/prometheus/_meta/config.reference.yml b/metricbeat/module/prometheus/_meta/config.reference.yml index 89f17c6ea64..59680f2a901 100644 --- a/metricbeat/module/prometheus/_meta/config.reference.yml +++ b/metricbeat/module/prometheus/_meta/config.reference.yml @@ -17,4 +17,4 @@ # This can be used for service account based authorization: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt \ No newline at end of file + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt