Fix testing for 2x releases (#3057)

* Update docker files to the last major with the most recent minor and bugfix version
* Renamed files to Dockerfile-2x to not have to be renamed every time a new bugfix is released
* Remove scripts and config files which are not needed anymore

To run testsuite for 2x releases, run: `TESTING_ENVIRONMENT=2x make testsuite`

filebeat/tests/system/test_registrar.py

@@ -1146,7 +1146,7 @@ def test_restart_state_reset_ttl(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/test.log",
             clean_inactive="20s",
-            ignore_older="5s"
+            ignore_older="15s"
         )
         os.mkdir(self.working_dir + "/log/")
 
@@ -1200,7 +1200,7 @@ def test_restart_state_reset_ttl_with_space(self):
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/test file.log",
             clean_inactive="20s",
-            ignore_older="5s"
+            ignore_older="15s"
         )
         os.mkdir(self.working_dir + "/log/")
 

libbeat/docker-compose.yml

@@ -60,3 +60,4 @@ services:
     build:
       context: ${PWD}/../testing/environments/
       dockerfile: Dockerfile
+      args: []

testing/environments/snapshot-2x.yml → testing/environments/2x.yml

@@ -6,15 +6,21 @@ services:
   elasticsearch:
     build:
       context: ./docker/elasticsearch
-      dockerfile: Dockerfile-2.3.3
+      dockerfile: Dockerfile-2x
+      args:
+        ELASTICSEARCH_VERSION: 2.4.2
     command: elasticsearch -Des.network.host=0.0.0.0
 
   logstash:
     build:
       context: ./docker/logstash
-      dockerfile: Dockerfile-2.3.2
+      dockerfile: Dockerfile-2x
+      args:
+        LOGSTASH_VERSION: 2.4.1
 
   kibana:
     build:
       context: ./docker/kibana
-      dockerfile: Dockerfile-4.5.1
+      dockerfile: Dockerfile-4.6
+      args:
+        KIBANA_VERSION: 4.6.3

testing/environments/Dockerfile

@@ -8,3 +8,4 @@ RUN apt-get update && \
     apt-get clean
 
 
+ARG KIBANA_VERSION

testing/environments/docker/elasticsearch/Dockerfile-2.4.0 → testing/environments/docker/elasticsearch/Dockerfile-2x

@@ -12,12 +12,9 @@ RUN arch="$(dpkg --print-architecture)" \
 
 RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
 
-ENV ELASTICSEARCH_MAJOR 2.4
-ENV ELASTICSEARCH_VERSION 2.4.0
+ARG ELASTICSEARCH_VERSION
 
-RUN wget https://download.elasticsearch.org/elasticsearch/staging/2.4.0-ce9f0c7/org/elasticsearch/distribution/deb/elasticsearch/2.4.0/elasticsearch-2.4.0.deb
-
-#RUN wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/${ELASTICSEARCH_VERSION}/elasticsearch-${ELASTICSEARCH_VERSION}.deb
+RUN wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/${ELASTICSEARCH_VERSION}/elasticsearch-${ELASTICSEARCH_VERSION}.deb
 
 RUN dpkg -i elasticsearch-${ELASTICSEARCH_VERSION}.deb
 

testing/environments/docker/kibana/Dockerfile-4.6.0 → testing/environments/docker/kibana/Dockerfile-4.6

@@ -15,10 +15,10 @@ RUN arch="$(dpkg --print-architecture)" \
 	&& rm /usr/local/bin/gosu.asc \
 	&& chmod +x /usr/local/bin/gosu
 
-ENV KIBANA_VERSION 4.6.0
+ARG KIBANA_VERSION
 
 RUN set -x \
-	&& curl -fSL "https://download.elastic.co/kibana/kibana/kibana-4.6.1-linux-x86_64.tar.gz" -o kibana.tar.gz \
+	&& curl -fSL "https://download.elastic.co/kibana/kibana/kibana-${KIBANA_VERSION}-linux-x86_64.tar.gz" -o kibana.tar.gz \
 	&& mkdir -p /opt/kibana \
 	&& tar -xz --strip-components=1 -C /opt/kibana -f kibana.tar.gz \
 	&& chown -R kibana:kibana /opt/kibana \

testing/environments/docker/logstash/Dockerfile-2.4.0 → testing/environments/docker/logstash/Dockerfile-2x

@@ -1,8 +1,8 @@
 FROM java:8-jre
 
-ENV LS_VERSION 2
+ARG LOGSTASH_VERSION
 
-ENV DEB_URL https://download.elastic.co/logstash/logstash/packages/debian/logstash-2.4.0.snapshot4_all.deb
+ENV DEB_URL https://download.elastic.co/logstash/logstash/packages/debian/logstash-${LOGSTASH_VERSION}_all.deb
 
 ENV PATH $PATH:/opt/logstash/bin:/opt/logstash/vendor/jruby/bin
 
@@ -18,7 +18,7 @@ RUN set -x && \
 
 RUN plugin install logstash-input-beats
 
-COPY logstash.conf.2.tmpl /logstash.conf.2.tmpl
+COPY logstash.conf.tmpl /logstash.conf.tmpl
 COPY docker-entrypoint.sh /entrypoint.sh
 
 COPY pki /etc/pki

testing/environments/docker/logstash/Dockerfile-5.0.0

@@ -1,6 +1,5 @@
 FROM java:8-jre
 
-ENV LS_VERSION 5
 ENV VERSION 5.0.0
 ENV URL https://artifacts.elastic.co/downloads/logstash/logstash-${VERSION}.tar.gz
 ENV PATH $PATH:/opt/logstash-$VERSION/bin
@@ -11,7 +10,7 @@ RUN set -x && \
     wget -qO logstash.tar.gz $URL && \
     tar xzf logstash.tar.gz
 
-COPY logstash.conf.2.tmpl /logstash.conf.2.tmpl
+COPY logstash.conf.tmpl /logstash.conf.tmpl
 COPY docker-entrypoint.sh /entrypoint.sh
 
 COPY pki /etc/pki

testing/environments/docker/logstash/Dockerfile-snapshot

@@ -16,7 +16,7 @@ RUN set -x && \
     tar xzf logstash.tar.gz
 
 
-COPY logstash.conf.2.tmpl /logstash.conf.2.tmpl
+COPY logstash.conf.tmpl /logstash.conf.tmpl
 COPY docker-entrypoint.sh /entrypoint.sh
 
 COPY pki /etc/pki

testing/environments/docker/logstash/docker-entrypoint.sh

@@ -58,21 +58,12 @@ waitForElasticsearch() {
   exit 1
 }
 
-updateConfigFile_1_5() {
-    sed -e "s/host .*/host => \"$ES_HOST\"/" /logstash.conf.1.5.tmpl > /logstash.conf
-}
-
-updateConfigFile_2() {
-    sed -e "s/hosts.*/hosts => [\"$ES_HOST:$ES_PORT\"]/" /logstash.conf.2.tmpl > /logstash.conf
+updateConfigFile() {
+    sed -e "s/hosts.*/hosts => [\"$ES_HOST:$ES_PORT\"]/" /logstash.conf.tmpl > /logstash.conf
 }
 
 # Main
-readParams
-if [ "$LS_VERSION" == "1.5" ]; then
-    updateConfigFile_1_5
-else
-    updateConfigFile_2
-fi
 
+updateConfigFile
 waitForElasticsearch
 exec "$@"

testing/environments/ports.yml

@@ -1,4 +1,7 @@
-# Defines if ports should be exported
+# Defines if ports should be exported.
+# This is useful for testing locally with a full elastic stack setup.
+# All services can be reached through localhost like localhost:5601 for Kibana
+# This is not used for CI as otherwise ports conflicts could happen.
 version: '2'
 services:
   elasticsearch: