Update beats to bd6981d (#92)

Important changes / additions:

* Add support for cloud id. See elastic/beats#4964 for more details. This could simplify the onboard for users that use apm-server with the Cloud in the future.
* Be able to overwrite index pattern in the index template. See elastic/beats#4769 for more details. This allows to clean up our system tests.

NOTICE

@@ -182,7 +182,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: a7103d1125adfb68bdf193484c5a2aaf476516e1
+Revision: bd6981df3e461c5ae34bf62516637f7c03701efd
 License type (autodetected): Apache License 2.0
 ./vendor/github.com/elastic/beats/LICENSE:
 --------------------------------------------------------------------

_beats/dev-tools/generate_notice.py

@@ -112,7 +112,7 @@ def write_notice_file(f, beat, copyright, dependencies):
                 # it's an Apache License, so include only the NOTICE file
                 f.write("Apache License 2.0\n\n")
 
-                # Skipe NOTICE files not needed
+                # Skip NOTICE files which are not needed
                 if os.path.join(os.path.dirname(lib["license_file"])) in SKIP_NOTICE:
                     continue
 

_beats/dev-tools/packer/platforms/centos/run.sh.j2

@@ -33,7 +33,7 @@ FPM_ARGS=(
         --after-install /tmp/systemd-daemon-reload.sh
         --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
         homedir/=/usr/share/{{.beat_name}}
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
         {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml

_beats/dev-tools/packer/platforms/debian/run.sh.j2

@@ -30,7 +30,7 @@ FPM_ARGS=(
         --after-install /tmp/systemd-daemon-reload.sh
         --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
         homedir/=/usr/share/{{.beat_name}}
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
         {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml

_beats/libbeat/_meta/config.reference.yml

@@ -110,6 +110,19 @@
 #processors:
 #- add_docker_metadata: ~
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs ======================================
 
 # Configure what outputs to use when sending the data collected by the beat.
@@ -144,6 +157,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "beatname" plus date
   # and generates [beatname-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "beatname-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -601,10 +615,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is beatname.
-# The version of the beat will always be appended to the given name
-# so the final name is beatname-%{[beat.version]}.
-#setup.template.name: "beatname"
+# Template name. By default the template name is "beatname-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "beatname-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "beatname-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

_beats/libbeat/_meta/config.yml

@@ -29,7 +29,7 @@
 
 #============================== Kibana =====================================
 
-# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. 
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
 # This requires a Kibana endpoint configuration.
 setup.kibana:
 
@@ -39,6 +39,19 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

_beats/libbeat/tests/system/beat/beat.py

@@ -11,6 +11,7 @@
 import yaml
 from datetime import datetime, timedelta
 
+
 BEAT_REQUIRED_FIELDS = ["@timestamp",
                         "beat.name", "beat.hostname", "beat.version"]
 
@@ -168,7 +169,7 @@ def start_beat(self,
                 "-test.coverprofile",
                 os.path.join(self.working_dir, "coverage.cov"),
                 "-path.home", os.path.normpath(self.working_dir),
-                "-c", os.path.join(self.working_dir, config)
+                "-c", os.path.join(self.working_dir, config),
                 ]
 
         if logging_args:
@@ -262,6 +263,11 @@ def setUp(self):
             shutil.rmtree(self.working_dir)
         os.makedirs(self.working_dir)
 
+        fields_yml = os.path.join(self.beat_path, "fields.yml")
+        # Only add it if it exists
+        if os.path.isfile(fields_yml):
+            shutil.copyfile(fields_yml, os.path.join(self.working_dir, "fields.yml"))
+
         try:
             # update the last_run link
             if os.path.islink(self.build_path + "last_run"):

_beats/script/build_docs.sh

@@ -20,13 +20,14 @@ fi
 
 index="${GOPATH%%:*}/src/${path}/index.asciidoc"
 
-if [ -f "$index" ]; then
-  echo "Building docs for ${name}..."
-  dest_dir="$html_dir/${name}"
-  mkdir -p "$dest_dir"
-  params=""
-  if [ "$PREVIEW" = "1" ]; then
-    params="--chunk=1 -open chunk=1 -open"
-  fi
-  $docs_dir/build_docs.pl $params --doc "$index" -out "$dest_dir"
+echo "Building docs for ${name}..."
+echo "Index document: ${index}"
+
+dest_dir="$html_dir/${name}"
+mkdir -p "$dest_dir"
+params=""
+if [ "$PREVIEW" = "1" ]; then
+  params="--chunk=1 -open chunk=1 -open"
 fi
+$docs_dir/build_docs.pl $params --doc "$index" -out "$dest_dir"
+

_beats/testing/environments/5x.yml

@@ -3,7 +3,7 @@
 version: '2.1'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.5.1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -18,13 +18,13 @@ services:
       context: docker/logstash
       dockerfile: Dockerfile
       args:
-        ELASTIC_VERSION: 5.4.1
+        ELASTIC_VERSION: 5.5.1
         DOWNLOAD_URL: https://artifacts.elastic.co/downloads
     environment:
       - ES_HOST=elasticsearch
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:5.4.1
+    image: docker.elastic.co/kibana/kibana:5.5.1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 6

_beats/testing/environments/latest.yml

@@ -1,9 +1,9 @@
-# This is the latest stable release environment.
+# This is the latest released environment.
 
 version: '2.1'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0-beta1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -18,13 +18,13 @@ services:
       context: docker/logstash
       dockerfile: Dockerfile
       args:
-        ELASTIC_VERSION: 5.4.1
+        ELASTIC_VERSION: 6.0.0-beta1
         DOWNLOAD_URL: https://artifacts.elastic.co/downloads
     environment:
       - ES_HOST=elasticsearch
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:5.4.1
+    image: docker.elastic.co/kibana/kibana:6.0.0-beta1
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 6

apm-server.reference.yml

@@ -126,6 +126,19 @@ apm-server:
 #processors:
 #- add_docker_metadata: ~
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using apm-server with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs ======================================
 
 # Configure what outputs to use when sending the data collected by the beat.
@@ -160,6 +173,7 @@ output.elasticsearch:
 
   # Optional index name. The default is "apm-server" plus date
   # and generates [apm-server-]YYYY.MM.DD keys.
+  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
   #index: "apm-server-%{[beat.version]}-%{+yyyy.MM.dd}"
 
   # Optional ingest node pipeline. By default no pipeline will be used.
@@ -617,10 +631,14 @@ output.elasticsearch:
 # Set to false to disable template loading.
 #setup.template.enabled: true
 
-# Template name. By default the template name is apm-server.
-# The version of the beat will always be appended to the given name
-# so the final name is apm-server-%{[beat.version]}.
-#setup.template.name: "apm-server"
+# Template name. By default the template name is "apm-server-%{[beat.version]}"
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.name: "apm-server-%{[beat.version]}"
+
+# Template patttern. By default the template patter is "-%{[beat.version]}-*" to apply to the default index settings.
+# The first part is the version of the beat and then -* is used to match all daily indicies.
+# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
+#setup.template.pattern: "apm-server-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"

apm-server.yml

@@ -45,7 +45,7 @@ apm-server:
 
 #============================== Kibana =====================================
 
-# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. 
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
 # This requires a Kibana endpoint configuration.
 setup.kibana:
 
@@ -55,6 +55,19 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+#============================= Elastic Cloud ==================================
+
+# These settings simplify using apm-server with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
 #================================ Outputs =====================================
 
 # Configure what outputs to use when sending the data collected by the beat.

tests/system/apmserver.py

@@ -73,16 +73,13 @@ class ElasticTest(ServerBaseTest):
     @classmethod
     def setUpClass(cls):
         super(ElasticTest, cls).setUpClass()
-        cls.base_name = "apm-server-tests"
-        cls.beat_version = "0.1.1"
-        cls.index_name = cls.base_name + "-" + cls.beat_version + "-1"
+        cls.index_name = "apm-server-tests"
 
     def config(self):
         cfg = super(ElasticTest, self).config()
         cfg.update({"elasticsearch_host": self.get_elasticsearch_url(),
                     "file_enabled": "false",
-                    "index_name": self.index_name,
-                    "template_base_name": self.base_name})
+                    "index_name": self.index_name})
         return cfg
 
     def setUp(self):
@@ -95,11 +92,11 @@ def setUp(self):
             self.es.indices.delete(index=self.index_name)
         except:
             pass
-        self.wait_until(lambda: not self.es.indices.exists(self.base_name))
+        self.wait_until(lambda: not self.es.indices.exists(self.index_name))
 
         try:
             self.es.indices.delete_template(
-                name=self.base_name + "-" + self.beat_version)
+                name=self.index_name)
         except:
             pass
 

tests/system/config/apm-server.yml.j2

@@ -7,6 +7,15 @@ apm-server:
   ssl.certificate:  {{ ssl_cert }}
   ssl.key: {{ ssl_key }}
 
+############################# Setup ##########################################
+
+{% if index_name %}
+
+setup.template.name: {{ index_name }}
+setup.template.pattern: {{ index_name }}*
+
+{% endif %}
+
 ############################# Output ##########################################
 
 # File as output
@@ -30,7 +39,6 @@ output.file:
 {% if elasticsearch_host %}
 output.elasticsearch.hosts: ["{{ elasticsearch_host }}"]
 output.elasticsearch.index: {{ index_name }}
-setup.template.name: {{ template_base_name }}
 {% endif %}
 
 ############################# Beat #########################################

tests/system/test_integration.py

@@ -37,7 +37,6 @@ def test_load_docs_with_template_and_add_error(self):
         self.load_docs_with_template(f, 'errors', 4)
 
     def load_docs_with_template(self, data_path, endpoint, expected_events_count):
-        # TODO Needs cleanup when https://github.com/elastic/beats/pull/4769 merged
 
         payload = json.loads(open(data_path).read())
         url = 'http://localhost:8200/v1/' + endpoint
@@ -46,7 +45,7 @@ def load_docs_with_template(self, data_path, endpoint, expected_events_count):
 
         # make sure template is loaded
         self.wait_until(
-            lambda: self.log_contains("Elasticsearch template with name 'apm-server-tests-0.1.1' loaded"))
+            lambda: self.log_contains("Elasticsearch template with name 'apm-server-tests' loaded"))
 
         self.wait_until(lambda: self.es.indices.exists(self.index_name))
         self.es.indices.refresh(index=self.index_name)