From 559eb3ef20cd59ac7e260b4e00b9e0eb2379aedf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 3 Jun 2020 09:07:37 -0700 Subject: [PATCH 1/9] Encode API key as base64 in common code --- libbeat/esleg/eslegclient/connection.go | 4 +++- libbeat/outputs/elasticsearch/client.go | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 46d4840cda8..775f7702588 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -18,6 +18,7 @@ package eslegclient import ( + "encoding/base64" "encoding/json" "fmt" "io" @@ -436,7 +437,8 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) } if conn.APIKey != "" { - req.Header.Add("Authorization", "ApiKey "+conn.APIKey) + encoded := base64.StdEncoding.EncodeToString([]byte(conn.APIKey)) + req.Header.Add("Authorization", "ApiKey "+encoded) } for name, value := range conn.Headers { diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 4a3c71df3bf..c9df4c1bab4 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -19,7 +19,6 @@ package elasticsearch import ( "context" - "encoding/base64" "errors" "fmt" "net/http" @@ -84,7 +83,7 @@ func NewClient( URL: s.URL, Username: s.Username, Password: s.Password, - APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), + APIKey: s.APIKey, Headers: s.Headers, TLS: s.TLS, Kerberos: s.Kerberos, From afa3e911c93753a04d2482777781b1919765d41f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 3 Jun 2020 09:18:38 -0700 Subject: [PATCH 2/9] Adding comment on API key field --- libbeat/esleg/eslegclient/connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 775f7702588..46598dd33be 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -61,7 +61,7 @@ type ConnectionSettings struct { Username string Password string - APIKey string + APIKey string // Raw API key, NOT base64-encoded Headers map[string]string TLS *tlscommon.TLSConfig From 218145fa9bf009151886df0687fb5946dd6d7612 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 3 Jun 2020 09:22:08 -0700 Subject: [PATCH 3/9] Adding CHANGELOG entries --- CHANGELOG-developer.next.asciidoc | 1 + CHANGELOG.next.asciidoc | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index b89bd78fbbb..c4bcc67e780 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -45,6 +45,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. fully based on Go text/template and no longer uses file concatenation to generate the config. Your magefile.go will require a change to adapt the devtool API. See the pull request for more details. {pull}18148[18148] +- The Elasticsearch client settings expect the API key to be raw (not base64-encoded). {issue}18939[18939] {pull}18945[18945] ==== Bugfixes diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 532faa78410..6f209b14807 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix regression in `add_kubernetes_metadata`, so configured `indexers` and `matchers` are used if defaults are not disabled. {issue}18481[18481] {pull}18818[18818] - Fix potential race condition in fingerprint processor. {pull}18738[18738] - Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916] +- The `monitoring.elasticsearch.api_key` value is correctly base64-encoded before being sent to the monitoring Elasticsearch cluster. {issue}18939[18939] {pull}18945[18945] *Auditbeat* From 4afdf1139b4648016a7e4755fb579eafe04cfcd9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 3 Jun 2020 09:53:36 -0700 Subject: [PATCH 4/9] Adding test --- libbeat/esleg/eslegclient/connection_test.go | 66 ++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 libbeat/esleg/eslegclient/connection_test.go diff --git a/libbeat/esleg/eslegclient/connection_test.go b/libbeat/esleg/eslegclient/connection_test.go new file mode 100644 index 00000000000..72b78f27e1d --- /dev/null +++ b/libbeat/esleg/eslegclient/connection_test.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package eslegclient + +import ( + "bufio" + "bytes" + "encoding/base64" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAPIKeyEncoding(t *testing.T) { + apiKey := "foobar" + encoded := base64.StdEncoding.EncodeToString([]byte(apiKey)) + + conn, err := NewConnection(ConnectionSettings{ + APIKey: apiKey, + }) + require.NoError(t, err) + + httpClient := newMockClient() + conn.HTTP = httpClient + + req, err := http.NewRequest("GET", "http://fakehost/some/path", nil) + require.NoError(t, err) + + _, _, err = conn.execHTTPRequest(req) + require.NoError(t, err) + + require.Equal(t, "ApiKey "+encoded, httpClient.Req.Header.Get("Authorization")) +} + +type mockClient struct { + Req *http.Request +} + +func (c *mockClient) Do(req *http.Request) (*http.Response, error) { + c.Req = req + + r := bytes.NewReader([]byte("HTTP/1.1 200 OK\n\nHello, world")) + return http.ReadResponse(bufio.NewReader(r), req) +} + +func (c *mockClient) CloseIdleConnections() {} + +func newMockClient() *mockClient { + return &mockClient{} +} From 36a5b8106159c51c2056ea44be3381b56b653172 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 4 Jun 2020 05:02:01 -0700 Subject: [PATCH 5/9] Base64-encode API key in constructor --- libbeat/esleg/eslegclient/connection.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 46598dd33be..2cb90b69c77 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -76,6 +76,8 @@ type ConnectionSettings struct { Timeout time.Duration IdleConnTimeout time.Duration + + encodedAPIKey string // Base64-encoded API key } // NewConnection returns a new Elasticsearch client @@ -158,6 +160,10 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { logp.Info("kerberos client created") } + if s.APIKey != "" { + s.encodedAPIKey = base64.StdEncoding.EncodeToString([]byte(s.APIKey)) + } + return &Connection{ ConnectionSettings: s, HTTP: httpClient, @@ -436,9 +442,8 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) req.SetBasicAuth(conn.Username, conn.Password) } - if conn.APIKey != "" { - encoded := base64.StdEncoding.EncodeToString([]byte(conn.APIKey)) - req.Header.Add("Authorization", "ApiKey "+encoded) + if conn.encodedAPIKey != "" { + req.Header.Add("Authorization", "ApiKey "+conn.encodedAPIKey) } for name, value := range conn.Headers { From 97806e077a0965bd6f65e328c21c60118166262a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 4 Jun 2020 07:00:59 -0700 Subject: [PATCH 6/9] Move encodedAPIKey field to Connection --- libbeat/esleg/eslegclient/connection.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 2cb90b69c77..e0d085d095c 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -49,8 +49,9 @@ type Connection struct { Encoder BodyEncoder HTTP esHTTPClient - version common.Version - log *logp.Logger + encodedAPIKey string // Base64-encoded API key + version common.Version + log *logp.Logger } // ConnectionSettings are the settings needed for a Connection @@ -76,8 +77,6 @@ type ConnectionSettings struct { Timeout time.Duration IdleConnTimeout time.Duration - - encodedAPIKey string // Base64-encoded API key } // NewConnection returns a new Elasticsearch client @@ -160,16 +159,18 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { logp.Info("kerberos client created") } - if s.APIKey != "" { - s.encodedAPIKey = base64.StdEncoding.EncodeToString([]byte(s.APIKey)) - } - - return &Connection{ + conn := Connection{ ConnectionSettings: s, HTTP: httpClient, Encoder: encoder, log: logp.NewLogger("esclientleg"), - }, nil + } + + if s.APIKey != "" { + conn.encodedAPIKey = base64.StdEncoding.EncodeToString([]byte(s.APIKey)) + } + + return &conn, nil } func settingsWithDefaults(s ConnectionSettings) ConnectionSettings { From dce0e2a8bfbfdf298e632ece0afe5776ca12d4ce Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 4 Jun 2020 16:45:01 -0700 Subject: [PATCH 7/9] Update doc on `api_key` setting value --- libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 5c18a2b9a78..267f8a7b349 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -37,13 +37,14 @@ output.elasticsearch: password: "{pwd}" ------------------------------------------------------------------------------ -To use an API key to connect to {es}, use `api_key`. +To use an API key to connect to {es}, use `api_key`. The value must be the ID of +the API key and the API key joined by a colon. ["source","yaml",subs="attributes,callouts"] ------------------------------------------------------------------------------ output.elasticsearch: hosts: ["https://localhost:9200"] - api_key: "KnR6yE41RrSowb0kQ0HWoA" + api_key: "VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw" ------------------------------------------------------------------------------ If the Elasticsearch nodes are defined by `IP:PORT`, then add `protocol: https` to the yaml file. From 2eb7a5f697d4b72d6d2504327565ef56ef5c1d52 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 5 Jun 2020 07:51:34 -0700 Subject: [PATCH 8/9] Adding API key format to setting section --- libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 267f8a7b349..f0b5cbf04ae 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -136,8 +136,8 @@ The default value is 1. ===== `api_key` -Instead of using usernames and passwords, -you can use API keys to secure communication with {es}. +Instead of using usernames and passwords, you can use API keys to secure communication +with {es}. The value must be the ID of the API key and the API key joined by a colon. For more information, see <>. ===== `username` From 9982b8d6ae7350a647c58a9946f20528d699ab87 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 5 Jun 2020 07:53:34 -0700 Subject: [PATCH 9/9] Compute entire API key auth header value up front --- libbeat/esleg/eslegclient/connection.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index e0d085d095c..3802ae1cb0e 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -49,9 +49,9 @@ type Connection struct { Encoder BodyEncoder HTTP esHTTPClient - encodedAPIKey string // Base64-encoded API key - version common.Version - log *logp.Logger + apiKeyAuthHeader string // Authorization HTTP request header with base64-encoded API key + version common.Version + log *logp.Logger } // ConnectionSettings are the settings needed for a Connection @@ -167,7 +167,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } if s.APIKey != "" { - conn.encodedAPIKey = base64.StdEncoding.EncodeToString([]byte(s.APIKey)) + conn.apiKeyAuthHeader = "ApiKey " + base64.StdEncoding.EncodeToString([]byte(s.APIKey)) } return &conn, nil @@ -443,8 +443,8 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) req.SetBasicAuth(conn.Username, conn.Password) } - if conn.encodedAPIKey != "" { - req.Header.Add("Authorization", "ApiKey "+conn.encodedAPIKey) + if conn.apiKeyAuthHeader != "" { + req.Header.Add("Authorization", conn.apiKeyAuthHeader) } for name, value := range conn.Headers {