Update beats to bd6981d

Important changes / additions: * Add support for cloud id. See elastic/beats#4964 for more details. This could simplify the onboard for users that use apm-server with the Cloud in the future. * Be able to overwrite index pattern in the index template. See elastic/beats#4769 for more details. This allows to clean up our system tests.
ruflin · Aug 23, 2017 · ad28890 · ad28890
1 parent 449df0f
commit ad28890
Show file tree

Hide file tree

Showing 26 changed files with 476 additions and 187 deletions.
diff --git a/NOTICE b/NOTICE
@@ -182,7 +182,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: a7103d1125adfb68bdf193484c5a2aaf476516e1
+Revision: bd6981df3e461c5ae34bf62516637f7c03701efd
 License type (autodetected): Apache License 2.0
 ./vendor/github.com/elastic/beats/LICENSE:
 --------------------------------------------------------------------

diff --git a/_beats/dev-tools/generate_notice.py b/_beats/dev-tools/generate_notice.py
@@ -112,7 +112,7 @@ def write_notice_file(f, beat, copyright, dependencies):
                 # it's an Apache License, so include only the NOTICE file
                 f.write("Apache License 2.0\n\n")
 
-                # Skipe NOTICE files not needed
+                # Skip NOTICE files which are not needed
                 if os.path.join(os.path.dirname(lib["license_file"])) in SKIP_NOTICE:
                     continue
 

diff --git a/_beats/dev-tools/packer/platforms/centos/run.sh.j2 b/_beats/dev-tools/packer/platforms/centos/run.sh.j2
@@ -33,7 +33,7 @@ FPM_ARGS=(
         --after-install /tmp/systemd-daemon-reload.sh
         --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
         homedir/=/usr/share/{{.beat_name}}
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
         {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml

diff --git a/_beats/dev-tools/packer/platforms/debian/run.sh.j2 b/_beats/dev-tools/packer/platforms/debian/run.sh.j2
@@ -30,7 +30,7 @@ FPM_ARGS=(
         --after-install /tmp/systemd-daemon-reload.sh
         --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
         homedir/=/usr/share/{{.beat_name}}
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
         {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml

diff --git a/_beats/libbeat/_meta/config.reference.yml b/_beats/libbeat/_meta/config.reference.yml
@@ -110,6 +110,19 @@
 #processors:
 #- add_docker_metadata: ~
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs ======================================
 
 # Configure what outputs to use when sending the data collected by the beat.
@@ -144,6 +157,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "beatname" plus date
   # and generates [beatname-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "beatname-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -601,10 +615,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is beatname.
-# The version of the beat will always be appended to the given name
-# so the final name is beatname-%{[beat.version]}.
-#setup.template.name: "beatname"
+# Template name. By default the template name is "beatname-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "beatname-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "beatname-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

diff --git a/_beats/libbeat/_meta/config.yml b/_beats/libbeat/_meta/config.yml
@@ -29,7 +29,7 @@
 
 #============================== Kibana =====================================
 
-# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. 
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
 # This requires a Kibana endpoint configuration.
 setup.kibana:
 
@@ -39,6 +39,19 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

diff --git a/_beats/libbeat/tests/system/beat/beat.py b/_beats/libbeat/tests/system/beat/beat.py
@@ -11,6 +11,7 @@
 import yaml
 from datetime import datetime, timedelta
 
+
 BEAT_REQUIRED_FIELDS = ["@timestamp",
                         "beat.name", "beat.hostname", "beat.version"]
 
@@ -168,7 +169,7 @@ def start_beat(self,
                 "-test.coverprofile",
                 os.path.join(self.working_dir, "coverage.cov"),
                 "-path.home", os.path.normpath(self.working_dir),
-                "-c", os.path.join(self.working_dir, config)
+                "-c", os.path.join(self.working_dir, config),
                 ]
 
         if logging_args:
@@ -262,6 +263,11 @@ def setUp(self):
             shutil.rmtree(self.working_dir)
         os.makedirs(self.working_dir)
 
+        fields_yml = os.path.join(self.beat_path, "fields.yml")
+        # Only add it if it exists
+        if os.path.isfile(fields_yml):
+            shutil.copyfile(fields_yml, os.path.join(self.working_dir, "fields.yml"))
+
         try:
             # update the last_run link
             if os.path.islink(self.build_path + "last_run"):

diff --git a/_beats/script/build_docs.sh b/_beats/script/build_docs.sh
@@ -20,13 +20,14 @@ fi
 
 index="${GOPATH%%:*}/src/${path}/index.asciidoc"
 
-if [ -f "$index" ]; then
-  echo "Building docs for ${name}..."
-  dest_dir="$html_dir/${name}"
-  mkdir -p "$dest_dir"
-  params=""
-  if [ "$PREVIEW" = "1" ]; then
-    params="--chunk=1 -open chunk=1 -open"
-  fi
-  $docs_dir/build_docs.pl $params --doc "$index" -out "$dest_dir"
+echo "Building docs for ${name}..."
+echo "Index document: ${index}"
+
+dest_dir="$html_dir/${name}"
+mkdir -p "$dest_dir"
+params=""
+if [ "$PREVIEW" = "1" ]; then
+  params="--chunk=1 -open chunk=1 -open"
 fi
+$docs_dir/build_docs.pl $params --doc "$index" -out "$dest_dir"
+
diff --git a/_beats/testing/environments/5x.yml b/_beats/testing/environments/5x.yml
@@ -3,7 +3,7 @@
 version: '2.1'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.5.1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -18,13 +18,13 @@ services:
       context: docker/logstash
       dockerfile: Dockerfile
       args:
-        ELASTIC_VERSION: 5.4.1
+        ELASTIC_VERSION: 5.5.1
         DOWNLOAD_URL: https://artifacts.elastic.co/downloads
     environment:
       - ES_HOST=elasticsearch
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:5.4.1
+    image: docker.elastic.co/kibana/kibana:5.5.1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 6
diff --git a/_beats/testing/environments/latest.yml b/_beats/testing/environments/latest.yml
@@ -1,9 +1,9 @@
-# This is the latest stable release environment.
+# This is the latest released environment.
 
 version: '2.1'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0-beta1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -18,13 +18,13 @@ services:
       context: docker/logstash
       dockerfile: Dockerfile
       args:
-        ELASTIC_VERSION: 5.4.1
+        ELASTIC_VERSION: 6.0.0-beta1
         DOWNLOAD_URL: https://artifacts.elastic.co/downloads
     environment:
       - ES_HOST=elasticsearch
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:5.4.1
+    image: docker.elastic.co/kibana/kibana:6.0.0-beta1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 6
diff --git a/apm-server.reference.yml b/apm-server.reference.yml
@@ -126,6 +126,19 @@ apm-server:
 #processors:
 #- add_docker_metadata: ~
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using apm-server with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs ======================================
 
 # Configure what outputs to use when sending the data collected by the beat.
@@ -160,6 +173,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "apm-server" plus date
   # and generates [apm-server-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "apm-server-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -617,10 +631,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is apm-server.
-# The version of the beat will always be appended to the given name
-# so the final name is apm-server-%{[beat.version]}.
-#setup.template.name: "apm-server"
+# Template name. By default the template name is "apm-server-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "apm-server-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "apm-server-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

diff --git a/apm-server.yml b/apm-server.yml
@@ -45,7 +45,7 @@ apm-server:
 
 #============================== Kibana =====================================
 
-# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. 
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
 # This requires a Kibana endpoint configuration.
 setup.kibana:
 
@@ -55,6 +55,19 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using apm-server with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.