Update test versions of NGINX, Apache and MongoDB (#33789)

The old tests were based on Debian Jessie released in April 2015 (end-of-life on June 30, 2020). We started having problems with expired GPG keys when installing dependencies in our Docker files. Also: * Add the missing MPM module and the `ServerName` directive for Apache. * Add retries to Logstash tests because they are flaky * Do not install mage v1.14.0 * Update elastic-agent-client version to avoid broken agent dependency (cherry picked from commit 4f639d7) # Conflicts: # NOTICE.txt # go.mod # go.sum
elastic · Nov 28, 2022 · ad00eb5 · ad00eb5
1 parent 1b1416d
commit ad00eb5
Show file tree

Hide file tree

Showing 17 changed files with 54 additions and 24 deletions.
diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml
@@ -30,5 +30,4 @@ jobs:
       run: sudo apt-get install -y librpm-dev
     - name: Run check
       run: |
-        go install github.com/magefile/mage
         make check
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -10339,12 +10339,21 @@ Contents of probable licence file $GOMODCACHE/github.com/golang/mock@v1.6.0/LICE
 
 
 --------------------------------------------------------------------------------
+<<<<<<< HEAD
 Dependency : github.com/golang/protobuf
 Version: v1.5.2
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
 Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.2/LICENSE:
+=======
+Dependency : github.com/elastic/elastic-agent-client/v7
+Version: v7.0.1
+Licence type (autodetected): Elastic
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.1/LICENSE.txt:
+>>>>>>> 4f639d7ed6 (Update test versions of NGINX, Apache and MongoDB (#33789))
 
 Copyright 2010 The Go Authors.  All rights reserved.
 

diff --git a/go.mod b/go.mod
@@ -61,9 +61,14 @@ require (
 	github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.2.0
+<<<<<<< HEAD
 	github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2
 	github.com/elastic/ecs v1.12.0
 	github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6
+=======
+	github.com/eclipse/paho.mqtt.golang v1.3.5
+	github.com/elastic/elastic-agent-client/v7 v7.0.1
+>>>>>>> 4f639d7ed6 (Update test versions of NGINX, Apache and MongoDB (#33789))
 	github.com/elastic/go-concert v0.2.0
 	github.com/elastic/go-libaudit/v2 v2.3.1
 	github.com/elastic/go-licenser v0.4.0

diff --git a/go.sum b/go.sum
@@ -323,11 +323,18 @@ github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible h1:jlUO91EFZu
 github.com/elastic/azure-sdk-for-go v59.0.0-elastic-1+incompatible/go.mod h1:4zuQekLQi489ShcqTmS1Zj1ta0qrcNBlSuGa+ziu2vM=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
+<<<<<<< HEAD
 github.com/elastic/ecs v1.12.0 h1:u6WZ2AWtxv5vHvTQ4EuVZdWZ51mKHQ2UIltRePcta5U=
 github.com/elastic/ecs v1.12.0/go.mod h1:pgiLbQsijLOJvFR8OTILLu0Ni/R/foUNg0L+T6mU9b4=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
 github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4=
+=======
+github.com/elastic/elastic-agent-autodiscover v0.4.0 h1:R1JMLHQpH2KP3GXY8zmgV4dj39uoe1asyPPWGQbGgSk=
+github.com/elastic/elastic-agent-autodiscover v0.4.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4=
+github.com/elastic/elastic-agent-client/v7 v7.0.1 h1:RDiozMRkjVfsn1vZ5tc1YzRC+g4w4QTsXIOPEsVDyIY=
+github.com/elastic/elastic-agent-client/v7 v7.0.1/go.mod h1:cHviLpA5fAwMbfBIHBVNl16qp90bO7pKHMAQaG+9raU=
+>>>>>>> 4f639d7ed6 (Update test versions of NGINX, Apache and MongoDB (#33789))
 github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
 github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0=
 github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=

diff --git a/metricbeat/docs/modules/apache.asciidoc b/metricbeat/docs/modules/apache.asciidoc
@@ -17,7 +17,7 @@ HTTPD] servers. The default metricset is `status`.
 [float]
 === Compatibility
 
-The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with
+The Apache metricsets were tested with Apache 2.4.12 and 2.4.54 and are expected to work with
 all versions >= 2.2.31 and >= 2.4.16.
 
 

diff --git a/metricbeat/docs/modules/nginx.asciidoc b/metricbeat/docs/modules/nginx.asciidoc
@@ -19,7 +19,7 @@ The default metricset is `stubstatus`.
 [float]
 === Compatibility
 
-The Nginx metricsets were tested with Nginx 1.9 and are expected to work with all version
+The Nginx metricsets were tested with Nginx 1.23.2 and are expected to work with all version
 >= 1.9.
 
 [float]

diff --git a/metricbeat/module/apache/_meta/Dockerfile b/metricbeat/module/apache/_meta/Dockerfile
@@ -1,6 +1,5 @@
 ARG APACHE_VERSION=${APACHE_VERSION}
 FROM httpd:$APACHE_VERSION
-RUN sed -i "/jessie-updates/d" /etc/apt/sources.list
-RUN apt-get update && apt-get install -y curl
+RUN apt update && yes | apt install curl
 HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost
 COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf
diff --git a/metricbeat/module/apache/_meta/docs.asciidoc b/metricbeat/module/apache/_meta/docs.asciidoc
@@ -8,7 +8,7 @@ HTTPD] servers. The default metricset is `status`.
 [float]
 === Compatibility
 
-The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with
+The Apache metricsets were tested with Apache 2.4.12 and 2.4.54 and are expected to work with
 all versions >= 2.2.31 and >= 2.4.16.
 
 

diff --git a/metricbeat/module/apache/_meta/httpd.conf b/metricbeat/module/apache/_meta/httpd.conf
@@ -63,6 +63,7 @@ Listen 80
 # Example:
 # LoadModule foo_module modules/mod_foo.so
 #
+LoadModule mpm_event_module modules/mod_mpm_event.so
 LoadModule authn_file_module modules/mod_authn_file.so
 #LoadModule authn_dbm_module modules/mod_authn_dbm.so
 #LoadModule authn_anon_module modules/mod_authn_anon.so
@@ -190,7 +191,7 @@ ServerAdmin you@example.com
 #
 # If your host doesn't have a registered DNS name, enter its IP address here.
 #
-#ServerName www.example.com:80
+ServerName localhost
 
 #
 # Deny access to the entirety of your server's filesystem. You must

diff --git a/metricbeat/module/apache/_meta/supported-versions.yml b/metricbeat/module/apache/_meta/supported-versions.yml
@@ -1,3 +1,5 @@
+# all the versions have to be the valid semantic version syntax x.x.x
+# because of the test check for comparing against older field structure
 variants:
-  - APACHE_VERSION: 2.4.12
-  - APACHE_VERSION: 2.4.20
+  - APACHE_VERSION: "2.4.54"
+  - APACHE_VERSION: "2.4.12"
diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml
@@ -1,11 +1,11 @@
-version: '2.3'
+version: "2.3"
 
 services:
   apache:
-    image: docker.elastic.co/integrations-ci/beats-apache:${APACHE_VERSION:-2.4.20}-1
+    image: docker.elastic.co/integrations-ci/beats-apache:${APACHE_VERSION:-2.4.54}-1
     build:
       context: ./_meta
       args:
-        APACHE_VERSION: ${APACHE_VERSION:-2.4.20}
+        APACHE_VERSION: "${APACHE_VERSION:-2.4.54}"
     ports:
       - 80
diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go
@@ -25,11 +25,13 @@ import (
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/tests/compose"
+	"github.com/elastic/beats/v7/metricbeat/mb"
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 	"github.com/elastic/beats/v7/metricbeat/module/logstash"
 	_ "github.com/elastic/beats/v7/metricbeat/module/logstash/node"
@@ -49,7 +51,15 @@ func TestFetch(t *testing.T) {
 		t.Run(metricSet, func(t *testing.T) {
 			config := getConfig(metricSet, service.Host())
 			f := mbtest.NewReportingMetricSetV2Error(t, config)
-			events, errs := mbtest.ReportingFetchV2Error(f)
+			var (
+				events []mb.Event
+				errs   []error
+			)
+
+			require.Eventually(t, func() bool {
+				events, errs = mbtest.ReportingFetchV2Error(f)
+				return len(events) > 0 && len(errs) == 0
+			}, 30*time.Second, time.Second, "events cannot be empty: %v, %v", events, errs)
 
 			require.Empty(t, errs)
 			require.NotEmpty(t, events)

diff --git a/metricbeat/module/mongodb/_meta/Dockerfile b/metricbeat/module/mongodb/_meta/Dockerfile
@@ -1,5 +1,4 @@
 ARG MONGODB_VERSION
 FROM mongo:${MONGODB_VERSION}
-RUN sed -i "/jessie-updates/d" /etc/apt/sources.list
-RUN apt-get update && apt-get install -y netcat
+RUN apt update && yes | apt install netcat
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 27017
diff --git a/metricbeat/module/mongodb/docker-compose.yml b/metricbeat/module/mongodb/docker-compose.yml
@@ -1,12 +1,12 @@
-version: '2.3'
+version: "2.3"
 
 services:
   mongodb:
     image: docker.elastic.co/integrations-ci/beats-mongodb:${MONGODB_VERSION:-3.4}-1
     build:
       context: ./_meta
       args:
-        MONGODB_VERSION: ${MONGODB_VERSION:-3.4}
+        MONGODB_VERSION: "${MONGODB_VERSION:-3.4}"
     command: mongod --replSet beats
     ports:
       - 27017
diff --git a/metricbeat/module/nginx/_meta/Dockerfile b/metricbeat/module/nginx/_meta/Dockerfile
@@ -1,6 +1,5 @@
 ARG NGINX_VERSION
 FROM nginx:${NGINX_VERSION}
-RUN sed -i "/jessie-updates/d" /etc/apt/sources.list
-RUN apt-get update && apt-get install -y curl
+RUN apt update && apt install -y curl
 HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost/nginx_status
 COPY ./nginx.conf /etc/nginx/
diff --git a/metricbeat/module/nginx/_meta/docs.asciidoc b/metricbeat/module/nginx/_meta/docs.asciidoc
@@ -10,7 +10,7 @@ The default metricset is `stubstatus`.
 [float]
 === Compatibility
 
-The Nginx metricsets were tested with Nginx 1.9 and are expected to work with all version
+The Nginx metricsets were tested with Nginx 1.23.2 and are expected to work with all version
 >= 1.9.
 
 [float]

diff --git a/metricbeat/module/nginx/docker-compose.yml b/metricbeat/module/nginx/docker-compose.yml
@@ -1,11 +1,11 @@
-version: '2.3'
+version: "2.3"
 
 services:
   nginx:
-    image: docker.elastic.co/integrations-ci/beats-nginx:${NGINX_VERSION:-1.9}-1
-    build: 
+    image: docker.elastic.co/integrations-ci/beats-nginx:${NGINX_VERSION:-1.23.2}-1
+    build:
       context: ./_meta
       args:
-        NGINX_VERSION: ${NGINX_VERSION:-1.9}
+        NGINX_VERSION: "${NGINX_VERSION:-1.23.2}"
     ports:
       - 80
-Original file line number
+Diff line change
@@ Expand Up / @@ -19,7 +19,7 @@ The default metricset is `stubstatus`. @@
     [float]
     === Compatibility
-    The Nginx metricsets were tested with Nginx 1.9 and are expected to work with all version
+    The Nginx metricsets were tested with Nginx 1.23.2 and are expected to work with all version
     >= 1.9.
     [float]
@@ Expand Down @@