Fix loading dashboards in ES 5.4 + tests connection timeout increase (#…

…4525) I started with investigating a flaky test and increasing the connection timeout, but went down the rabbit hole a bit: * The dashboard loader didn't work against 5.4, i think since #4471. This fixes it by checking the version and only setting single_type for > 6.0. * GetTestingElasticsearch now asserts on errors and increases the connection timeout * Several tests were calling Connect twice
elastic · Jun 19, 2017 · 76a9b87 · 76a9b87
1 parent fdfcaa3
commit 76a9b87
Show file tree

Hide file tree

Showing 10 changed files with 37 additions and 39 deletions.
diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go
@@ -13,7 +13,7 @@ import (
 )
 
 func TestLoadPipeline(t *testing.T) {
-	client := elasticsearch.GetTestingElasticsearch()
+	client := elasticsearch.GetTestingElasticsearch(t)
 	if !hasIngest(client) {
 		t.Skip("Skip tests because ingest is missing in this elasticsearch version: %s", client.GetVersion())
 	}
@@ -56,7 +56,7 @@ func TestLoadPipeline(t *testing.T) {
 }
 
 func TestSetupNginx(t *testing.T) {
-	client := elasticsearch.GetTestingElasticsearch()
+	client := elasticsearch.GetTestingElasticsearch(t)
 	if !hasIngest(client) {
 		t.Skip("Skip tests because ingest is missing in this elasticsearch version: %s", client.GetVersion())
 	}
@@ -88,7 +88,7 @@ func TestSetupNginx(t *testing.T) {
 }
 
 func TestAvailableProcessors(t *testing.T) {
-	client := elasticsearch.GetTestingElasticsearch()
+	client := elasticsearch.GetTestingElasticsearch(t)
 	if !hasIngest(client) {
 		t.Skip("Skip tests because ingest is missing in this elasticsearch version: %s", client.GetVersion())
 	}

diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go
@@ -13,6 +13,7 @@ type DashboardLoader interface {
 	LoadJSON(path string, json map[string]interface{}) ([]byte, error)
 	CreateIndex(index string, body interface{}) (int, *elasticsearch.QueryResult, error)
 	IndexExists(index string) (int, error)
+	GetVersion() string
 }
 
 func ImportDashboards(beatName, beatVersion string, esClient DashboardLoader, cfg *common.Config) error {

diff --git a/libbeat/dashboards/importer.go b/libbeat/dashboards/importer.go
@@ -79,12 +79,19 @@ func (imp Importer) CreateKibanaIndex() error {
 		if status != 404 {
 			return err
 		} else {
-			_, _, err = imp.client.CreateIndex(imp.cfg.KibanaIndex,
-				common.MapStr{
+			var settings common.MapStr
+			// XXX: this can be removed when the dashboard loaded will no longer need to support 6.0,
+			// because the Kibana API is used instead
+			if strings.HasPrefix(imp.client.GetVersion(), "6.") {
+				settings = common.MapStr{
 					"settings": common.MapStr{
 						"index.mapping.single_type": false,
 					},
-				})
+				}
+			} else {
+				settings = nil
+			}
+			_, _, err = imp.client.CreateIndex(imp.cfg.KibanaIndex, settings)
 			if err != nil {
 				return fmt.Errorf("Failed to create index: %v", err)
 			}

diff --git a/libbeat/dashboards/importer_integration_test.go b/libbeat/dashboards/importer_integration_test.go
@@ -15,7 +15,7 @@ func TestImporter(t *testing.T) {
 		logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"})
 	}
 
-	client := elasticsearch.GetTestingElasticsearch()
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	imp, err := NewImporter(&DashboardsConfig{
 		KibanaIndex: ".kibana-test",
@@ -37,7 +37,7 @@ func TestImporterEmptyBeat(t *testing.T) {
 		logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"})
 	}
 
-	client := elasticsearch.GetTestingElasticsearch()
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	imp, err := NewImporter(&DashboardsConfig{
 		KibanaIndex: ".kibana-test-nobeat",

diff --git a/libbeat/outputs/elasticsearch/api.go b/libbeat/outputs/elasticsearch/api.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 
 	"github.com/elastic/beats/libbeat/logp"
+	"github.com/pkg/errors"
 )
 
 // QueryResult contains the result of a query.
@@ -52,7 +53,7 @@ func (r QueryResult) String() string {
 
 func withQueryResult(status int, resp []byte, err error) (int, *QueryResult, error) {
 	if err != nil {
-		return status, nil, err
+		return status, nil, errors.Wrapf(err, "Elasticsearch response: %s", resp)
 	}
 	result, err := readQueryResult(resp)
 	return status, result, err

diff --git a/libbeat/outputs/elasticsearch/api_integration_test.go b/libbeat/outputs/elasticsearch/api_integration_test.go
@@ -21,7 +21,7 @@ func TestIndex(t *testing.T) {
 
 	index := fmt.Sprintf("beats-test-index-%d", os.Getpid())
 
-	client := GetTestingElasticsearch()
+	client := GetTestingElasticsearch(t)
 
 	body := map[string]interface{}{
 		"user":      "test",
@@ -77,7 +77,7 @@ func TestIngest(t *testing.T) {
 		},
 	}
 
-	client := GetTestingElasticsearch()
+	client := GetTestingElasticsearch(t)
 	if strings.HasPrefix(client.Connection.version, "2.") {
 		t.Skip("Skipping tests as pipeline not available in 2.x releases")
 	}

diff --git a/libbeat/outputs/elasticsearch/bulkapi_integration_test.go b/libbeat/outputs/elasticsearch/bulkapi_integration_test.go
@@ -15,7 +15,7 @@ func TestBulk(t *testing.T) {
 		logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"elasticsearch"})
 	}
 
-	client := GetTestingElasticsearch()
+	client := GetTestingElasticsearch(t)
 	index := fmt.Sprintf("packetbeat-unittest-%d", os.Getpid())
 
 	ops := []map[string]interface{}{
@@ -66,7 +66,7 @@ func TestEmptyBulk(t *testing.T) {
 		logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"elasticsearch"})
 	}
 
-	client := GetTestingElasticsearch()
+	client := GetTestingElasticsearch(t)
 	index := fmt.Sprintf("packetbeat-unittest-%d", os.Getpid())
 
 	body := make([]interface{}, 0, 10)
@@ -88,7 +88,7 @@ func TestBulkMoreOperations(t *testing.T) {
 		logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"elasticsearch"})
 	}
 
-	client := GetTestingElasticsearch()
+	client := GetTestingElasticsearch(t)
 	index := fmt.Sprintf("packetbeat-unittest-%d", os.Getpid())
 
 	ops := []map[string]interface{}{

diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go
@@ -17,7 +17,7 @@ import (
 
 func TestClientConnect(t *testing.T) {
 
-	client := GetTestingElasticsearch()
+	client := GetTestingElasticsearch(t)
 	err := client.Connect(5 * time.Second)
 	assert.NoError(t, err)
 }

diff --git a/libbeat/outputs/elasticsearch/testing.go b/libbeat/outputs/elasticsearch/testing.go
@@ -3,6 +3,7 @@ package elasticsearch
 import (
 	"math/rand"
 	"os"
+	"testing"
 	"time"
 
 	"github.com/elastic/beats/libbeat/outputs/outil"
@@ -38,14 +39,17 @@ func GetEsPort() string {
 }
 
 // GetTestingElasticsearch creates a test client.
-func GetTestingElasticsearch() *Client {
+func GetTestingElasticsearch(t *testing.T) *Client {
 	var address = "http://" + GetEsHost() + ":" + GetEsPort()
 	username := os.Getenv("ES_USER")
 	pass := os.Getenv("ES_PASS")
 	client := newTestClientAuth(address, username, pass)
 
 	// Load version number
-	client.Connect(3 * time.Second)
+	err := client.Connect(60 * time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
 	return client
 }
 

diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go
@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/outputs/elasticsearch"
@@ -17,10 +16,7 @@ import (
 
 func TestCheckTemplate(t *testing.T) {
 
-	client := elasticsearch.GetTestingElasticsearch()
-
-	err := client.Connect(5 * time.Second)
-	assert.Nil(t, err)
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	loader := &Loader{
 		client: client,
@@ -33,9 +29,7 @@ func TestCheckTemplate(t *testing.T) {
 func TestLoadTemplate(t *testing.T) {
 
 	// Setup ES
-	client := elasticsearch.GetTestingElasticsearch()
-	err := client.Connect(5 * time.Second)
-	assert.Nil(t, err)
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	// Load template
 	absPath, err := filepath.Abs("../")
@@ -76,9 +70,7 @@ func TestLoadInvalidTemplate(t *testing.T) {
 	}
 
 	// Setup ES
-	client := elasticsearch.GetTestingElasticsearch()
-	err := client.Connect(5 * time.Second)
-	assert.Nil(t, err)
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	templateName := "invalidtemplate"
 
@@ -87,7 +79,7 @@ func TestLoadInvalidTemplate(t *testing.T) {
 	}
 
 	// Try to load invalid template
-	err = loader.LoadTemplate(templateName, template)
+	err := loader.LoadTemplate(templateName, template)
 	assert.Error(t, err)
 
 	// Make sure template was not loaded
@@ -127,10 +119,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		assert.Nil(t, err)
 
 		// Setup ES
-		client := elasticsearch.GetTestingElasticsearch()
-
-		err = client.Connect(5 * time.Second)
-		assert.Nil(t, err)
+		client := elasticsearch.GetTestingElasticsearch(t)
 
 		fieldsPath := absPath + "/fields.yml"
 		index := beat
@@ -162,9 +151,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 func TestTemplateSettings(t *testing.T) {
 
 	// Setup ES
-	client := elasticsearch.GetTestingElasticsearch()
-	err := client.Connect(5 * time.Second)
-	assert.Nil(t, err)
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	// Load template
 	absPath, err := filepath.Abs("../")
@@ -214,9 +201,7 @@ func TestTemplateSettings(t *testing.T) {
 func TestOverwrite(t *testing.T) {
 
 	// Setup ES
-	client := elasticsearch.GetTestingElasticsearch()
-	err := client.Connect(5 * time.Second)
-	assert.Nil(t, err)
+	client := elasticsearch.GetTestingElasticsearch(t)
 
 	beatInfo := common.BeatInfo{
 		Beat:    "testbeat",