[parser] switching to using a list for optional top level keys

airbnb · Jun 8, 2017 · 8f9b55b · 8f9b55b
1 parent 06b470f
commit 8f9b55b
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/conf/logs.json b/conf/logs.json
@@ -265,10 +265,10 @@
           "ingress.event.filemod"
         ]
       },
-      "optional_top_level_keys": {
-        "filetype": "integer",
-        "filetype_name": "string"
-      }
+      "optional_top_level_keys": [
+        "filetype",
+        "filetype_name"
+      ]
     }
   },
   "carbonblack:binaryinfo.host.observed": {

diff --git a/stream_alert/rule_processor/parsers.py b/stream_alert/rule_processor/parsers.py
@@ -214,17 +214,19 @@ def default_optional_values(key):
                 elif key == OrderedDict():
                     return dict()
 
-            for key_name, value_type in optional_keys.iteritems():
+            for key_name in optional_keys:
                 # Instead of doing a schema.update() here with a default value type,
                 # we should enforce having any optional keys declared within the schema
                 # and log an error if that is not the case
                 if key_name not in schema:
-                    LOGGER.error('Optional key \'%s\' not found in schema', key_name)
+                    LOGGER.error('Optional top level key \'%s\' '
+                                 'not found in declared log schema', key_name)
+                    continue
                 # If the optional key isn't in our parsed json payload
                 for record in json_records:
                     if key_name not in record:
                         # Set default value
-                        record[key_name] = default_optional_values(value_type)
+                        record[key_name] = default_optional_values(schema[key_name])
 
         return json_records