Bring back support for really old Docker versions (elastic#7553)

Introducing API version negotiation (elastic#7165) brough support for new
Docker versions, but broke support for versions in EOL.

This change puts old API version (1.229 back on the default fallback,
plus introduces version hardcoding through `DOCKER_API_VERSION`
environment variable.

I added some integration tests to check things are working.

Fixes elastic#7542

(cherry picked from commit 2215d24)

CHANGELOG.asciidoc

@@ -170,6 +170,14 @@ https://github.com/elastic/beats/compare/v6.2.3...v6.3.0[View commits]
 - Fix map overwrite panics by cloning shared structs before doing the update. {pull}6947[6947]
 - Fix delays on autodiscovery events handling caused by blocking runner stops. {pull}7170[7170]
 - Do not emit Kubernetes autodiscover events for Pods without IP address. {pull}7235[7235]
+- Allow to override the `ignore_above` option when defining new field with the type keyword. {pull}7238[7238]
+- Allow index-pattern only setup when setup.dashboards.only_index=true. {pull}7285[7285]
+- Fix duplicating dynamic_fields in template when overwriting the template. {pull}7352[7352]
+- Fix a panic on the Dissect processor when we have data remaining after the last delimiter. {pull}7449[7449]
+- When we fail to build a Kubernetes' indexer or matcher we produce a warning but we don't add them to the execution. {pull}7466[7466]
+- Fix default value for logging.files.keepfiles. It was being set to 0 and now
+  it's set to the documented value of 7. {issue}7494[7494]
+- Retain compatibility with older Docker server versions. {issue}7542[7542]
 
 *Auditbeat*
 

libbeat/common/docker/client.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"net/http"
+	"os"
 
 	"github.com/elastic/beats/libbeat/logp"
 
@@ -11,31 +12,35 @@ import (
 	"golang.org/x/net/context"
 )
 
-// Select Docker API version
-const dockerAPIVersion = api.DefaultVersion
-
 // NewClient builds and returns a new Docker client
-// It uses version 1.26 by default, and negotiates it with the server so it is downgraded if 1.26 is too high
+// It uses version 1.30 by default, and negotiates it with the server so it is downgraded if 1.30 is too high
 func NewClient(host string, httpClient *http.Client, httpHeaders map[string]string) (*client.Client, error) {
-	c, err := client.NewClient(host, dockerAPIVersion, httpClient, nil)
-	if err != nil {
-		return c, err
+	version := os.Getenv("DOCKER_API_VERSION")
+	if version == "" {
+		version = api.DefaultVersion
 	}
 
-	logp.Debug("docker", "Negotiating client version")
-	ping, err := c.Ping(context.Background())
+	c, err := client.NewClient(host, version, httpClient, nil)
 	if err != nil {
-		logp.Debug("docker", "Failed to perform ping: %s", err)
-	}
-
-	// try the latest version before versioning headers existed
-	if ping.APIVersion == "" {
-		ping.APIVersion = "1.24"
+		return c, err
 	}
 
-	// if server version is lower than the client version, downgrade
-	if versions.LessThan(ping.APIVersion, dockerAPIVersion) {
-		c.UpdateClientVersion(ping.APIVersion)
+	if os.Getenv("DOCKER_API_VERSION") == "" {
+		logp.Debug("docker", "Negotiating client version")
+		ping, err := c.Ping(context.Background())
+		if err != nil {
+			logp.Debug("docker", "Failed to perform ping: %s", err)
+		}
+
+		// try a really old version, before versioning headers existed
+		if ping.APIVersion == "" {
+			ping.APIVersion = "1.22"
+		}
+
+		// if server version is lower than the client version, downgrade
+		if versions.LessThan(ping.APIVersion, version) {
+			c.UpdateClientVersion(ping.APIVersion)
+		}
 	}
 
 	logp.Debug("docker", "Client version set to %s", c.ClientVersion())

libbeat/common/docker/client_test.go

@@ -0,0 +1,62 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package docker
+
+import (
+	"os"
+	"testing"
+
+	"github.com/docker/docker/api"
+	"github.com/docker/docker/api/types"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+)
+
+func TestNewClient(t *testing.T) {
+	host := "unix:///var/run/docker.sock"
+
+	client, err := NewClient(host, nil, nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, client)
+
+	_, err = client.ContainerList(context.Background(), types.ContainerListOptions{})
+	assert.NoError(t, err)
+
+	// This test only works on newer Docker versions (any supported one really)
+	switch client.ClientVersion() {
+	case "1.22":
+		t.Skip("Docker version is too old for this test")
+	case api.DefaultVersion:
+		t.Logf("Using default API version: %s", api.DefaultVersion)
+	default:
+		t.Logf("Negotiated version: %s", client.ClientVersion())
+	}
+
+	// Test we can hardcode version
+	os.Setenv("DOCKER_API_VERSION", "1.22")
+
+	client, err = NewClient(host, nil, nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, client)
+	assert.Equal(t, "1.22", client.ClientVersion())
+
+	_, err = client.ContainerList(context.Background(), types.ContainerListOptions{})
+	assert.NoError(t, err)
+}