Remove defaults from fields.common.yml for more flexibility (#3602)

* Remove defaults from fields.common.yml for more flexibility Having the defaults in code instead of the libbeat fields.yml will allow to reuse the same script also for fields.yml files from a metricbeat module for example without having to merge in the libbeat file. This is a prerequisite for dynamic index template generation in Golang which can be only for a module. The top level `fields` entry was removed as without the defaults, it is not needed anymore. All scripts were adjusted for the new format. * * Switch dict to object and list to array type. This is to be in line with the elasticsearch naming: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html * Remove object-type from array as arrays cannot be predefined.
elastic · Feb 16, 2017 · 1b6af55 · 1b6af55
1 parent 221fe7d
commit 1b6af55
Show file tree

Hide file tree

Showing 22 changed files with 110 additions and 114 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -333,7 +333,7 @@ Arbitrary tags that can be set per Beat and per transaction type.
 [float]
 === fields
 
-type: dict
+type: object
 
 Contains user configurable fields.
 

diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -65,7 +65,7 @@ Arbitrary tags that can be set per Beat and per transaction type.
 [float]
 === fields
 
-type: dict
+type: object
 
 Contains user configurable fields.
 

diff --git a/libbeat/_meta/fields.common.yml b/libbeat/_meta/fields.common.yml
@@ -1,11 +1,4 @@
-defaults:
-  type: keyword
-  required: false
-  index: true
-  doc_values: true
-  ignore_above: 1024
 
-fields:
 - key: beat
   title: Beat
   description: >
@@ -40,8 +33,8 @@ fields:
         type.
 
     - name: fields
-      type: dict
-      dict-type: keyword
+      type: object
+      object-type: keyword
       description: >
         Contains user configurable fields.
 
diff --git a/libbeat/scripts/generate_fields_docs.py b/libbeat/scripts/generate_fields_docs.py
@@ -84,15 +84,15 @@ def fields_to_asciidoc(input, output, beat):
 
     # Create sections from available fields
     sections = {}
-    for v in docs["fields"]:
+    for v in docs:
         sections[v["key"]] = v["title"]
 
     for key in sorted(sections):
         output.write("* <<exported-fields-{}>>\n".format(key))
     output.write("\n--\n")
 
     # Sort alphabetically by key
-    for section in sorted(docs["fields"], key=lambda field: field["key"]):
+    for section in sorted(docs, key=lambda field: field["key"]):
         section["name"] = section["title"]
         section["anchor"] = section["key"]
         document_fields(output, section, sections, "")

diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py
@@ -97,7 +97,7 @@ def fields_to_index_pattern(args, input):
 
     }
 
-    for k, section in enumerate(docs["fields"]):
+    for k, section in enumerate(docs):
         fields_to_json(section, "", output)
 
     # add meta fields
@@ -149,9 +149,10 @@ def get_index_pattern_name(index):
     with open(fields_yml, 'r') as f:
         fields = f.read()
 
-        # Prepend beat fields from libbeat
-        with open(args.libbeat + "/_meta/fields.generated.yml") as f:
-            fields = f.read() + fields
+        if os.path.basename(args.beat) != "libbeat":
+            # Prepend beat fields from libbeat
+            with open(args.libbeat + "/_meta/fields.generated.yml") as f:
+                fields = f.read() + fields
 
         # with open(target, 'w') as output:
         output = fields_to_index_pattern(args, fields)

diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py
@@ -30,16 +30,16 @@ def fields_to_es_template(args, input, output, index, version):
         print("fields.yml is empty. Cannot generate template.")
         return
 
-    # Each template needs defaults
-    if "defaults" not in docs.keys():
-        print("No defaults are defined. Each template needs at" +
-              " least defaults defined.")
-        return
-
-    defaults = docs["defaults"]
+    defaults = {
+        "type": "keyword",
+        "required": False,
+        "index": True,
+        "doc_values": True,
+        "ignore_above": 1024,
+    }
 
-    for k, section in enumerate(docs["fields"]):
-        docs["fields"][k] = dedot(section)
+    for k, section in enumerate(docs):
+        docs[k] = dedot(section)
 
     # skeleton
     template = {
@@ -105,9 +105,8 @@ def fields_to_es_template(args, input, output, index, version):
             }
         })
 
-    for section in docs["fields"]:
-        prop, dynamic = fill_section_properties(args, section,
-                                                defaults, "")
+    for section in docs:
+        prop, dynamic = fill_section_properties(args, section, defaults, "")
         properties.update(prop)
         dynamic_templates.extend(dynamic)
 
@@ -253,10 +252,15 @@ def fill_field_properties(args, field, defaults, path):
             properties[field["name"]]["scaling_factor"] = \
                 field.get("scaling_factor", 1000)
 
-    elif field["type"] in ["dict", "list"]:
-        if field.get("dict-type") == "text":
+    elif field["type"] in ["array"]:
+        properties[field["name"]] = {
+            "properties": {}
+        }
+
+    elif field["type"] in ["object"]:
+        if field.get("object-type") == "text":
             # add a dynamic template to set all members of
-            # the dict as text
+            # the object as text
             if len(path) > 0:
                 name = path + "." + field["name"]
             else:
@@ -284,7 +288,7 @@ def fill_field_properties(args, field, defaults, path):
                     }
                 })
 
-        if field.get("dict-type") == "long":
+        if field.get("object-type") == "long":
             if len(path) > 0:
                 name = path + "." + field["name"]
             else:

diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
@@ -408,7 +408,7 @@ def extract_fields(doc_list, name):
                     dictfields.extend(subdictfields)
                 else:
                     fields.append(newName)
-                    if field.get("type") in ["dict", "geo_point"]:
+                    if field.get("type") in ["object", "geo_point"]:
                         dictfields.append(newName)
             return fields, dictfields
 
@@ -429,7 +429,7 @@ def extract_fields(doc_list, name):
             fields = []
             dictfields = []
 
-            for item in doc["fields"]:
+            for item in doc:
                 subfields, subdictfields = extract_fields(item["fields"], "")
                 fields.extend(subfields)
                 dictfields.extend(subdictfields)

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -409,7 +409,7 @@ Arbitrary tags that can be set per Beat and per transaction type.
 [float]
 === fields
 
-type: dict
+type: object
 
 Contains user configurable fields.
 
@@ -1335,15 +1335,15 @@ Size of the files that have been created or changed since creation.
 [float]
 === docker.container.labels
 
-type: dict
+type: object
 
 Image labels.
 
 
 [float]
 === docker.container.tags
 
-type: list
+type: array
 
 Image tags.
 
@@ -1574,15 +1574,15 @@ Total size of the all cached images associated to the current image.
 [float]
 === docker.image.labels
 
-type: dict
+type: object
 
 Image labels.
 
 
 [float]
 === docker.image.tags
 
-type: list
+type: array
 
 Image tags.
 
@@ -2990,7 +2990,7 @@ Leader id (broker).
 [float]
 === kafka.partition.partition.isr
 
-type: list
+type: array
 
 List of isr ids.
 
@@ -6313,7 +6313,7 @@ The username of the user that created the process. If the username cannot be det
 [float]
 === system.process.env
 
-type: dict
+type: object
 
 The environment variables used to start the process. The data is available on FreeBSD, Linux, and OS X.
 
@@ -6597,7 +6597,7 @@ CPU time consumed by tasks in user (kernel) mode.
 [float]
 === system.process.cgroup.cpuacct.percpu
 
-type: dict
+type: object
 
 CPU time (in nanoseconds) consumed on each CPU by all tasks in this cgroup.
 

diff --git a/metricbeat/module/docker/container/_meta/fields.yml b/metricbeat/module/docker/container/_meta/fields.yml
@@ -41,12 +41,11 @@
           description: >
              Size of the files that have been created or changed since creation.
     - name: labels
-      type: dict
-      dict-type: keyword
+      type: object
+      object-type: keyword
       description: >
         Image labels.
     - name: tags
-      type: list
-      dict-type: keyword
+      type: array
       description: >
         Image tags.
diff --git a/metricbeat/module/docker/cpu/_meta/fields.yml b/metricbeat/module/docker/cpu/_meta/fields.yml
@@ -35,6 +35,6 @@
         Total CPU usage.
     # TODO: how to document cpu list?
     #- name: core
-    #  type: list
+    #  type: array
     #  description: >
     #    Dictionary with list of cpu and usage inside.
diff --git a/metricbeat/module/docker/image/_meta/fields.yml b/metricbeat/module/docker/image/_meta/fields.yml
@@ -35,13 +35,12 @@
              Total size of the all cached images associated to the current image.
 
     - name: labels
-      type: dict
-      dict-type: keyword
+      type: object
+      object-type: keyword
       description: >
         Image labels.
 
     - name: tags
-      type: list
-      dict-type: keyword
+      type: array
       description: >
         Image tags.
diff --git a/metricbeat/module/kafka/partition/_meta/fields.yml b/metricbeat/module/kafka/partition/_meta/fields.yml
@@ -32,7 +32,7 @@
           description: >
             Leader id (broker).
         - name: isr
-          type: list
+          type: array
           description: >
             List of isr ids.
         - name: replica

diff --git a/metricbeat/module/system/process/_meta/fields.yml b/metricbeat/module/system/process/_meta/fields.yml
@@ -36,8 +36,8 @@
         numeric identifier (UID). On Windows, this field includes the user's
         domain and is formatted as `domain\username`.
     - name: env
-      type: dict
-      dict-type: keyword
+      type: object
+      object-type: keyword
       description: >
         The environment variables used to start the process. The data is
         available on FreeBSD, Linux, and OS X.
@@ -235,8 +235,8 @@
               description: CPU time consumed by tasks in user (kernel) mode.
 
             - name: percpu
-              type: dict
-              dict-type: long
+              type: object
+              object-type: long
               description: >
                 CPU time (in nanoseconds) consumed on each CPU by all tasks in
                 this cgroup.