Refactored JSON output modules to use event data #2697 (#2698)

plaso/output/interface.py

@@ -152,11 +152,6 @@ def WriteHeader(self):
 class LinearOutputModule(OutputModule):
   """Linear output module."""
 
-  # Need to suppress this since these classes do not implement the
-  # abstract method WriteEventBody, classes that inherit from one of these
-  # classes need to implement that function.
-  # pylint: disable=abstract-method
-
   def __init__(self, output_mediator):
     """Initializes a linear output module.
 
@@ -170,6 +165,16 @@ def __init__(self, output_mediator):
     super(LinearOutputModule, self).__init__(output_mediator)
     self._output_writer = None
 
+  @abc.abstractmethod
+  def WriteEventBody(self, event, event_data, event_tag):
+    """Writes event values to the output.
+
+    Args:
+      event (EventObject): event.
+      event_data (EventData): event data.
+      event_tag (EventTag): event tag.
+    """
+
   def SetOutputWriter(self, output_writer):
     """Set the output writer.
 

plaso/output/json_line.py

@@ -7,25 +7,16 @@
 
 from __future__ import unicode_literals
 
-import codecs
-import copy
-import json
-
-from plaso.lib import errors
-from plaso.lib import py2to3
-from plaso.output import interface
 from plaso.output import manager
-from plaso.serializer import json_serializer
+from plaso.output import shared_json
 
 
-class JSONLineOutputModule(interface.LinearOutputModule):
+class JSONLineOutputModule(shared_json.SharedJSONOutputModule):
   """Output module for the JSON line format."""
 
   NAME = 'json_line'
   DESCRIPTION = 'Saves the events into a JSON line format.'
 
-  _JSON_SERIALIZER = json_serializer.JSONAttributeContainerSerializer
-
   def WriteEventBody(self, event, event_data, event_tag):
     """Writes event values to the output.
 
@@ -34,32 +25,7 @@ def WriteEventBody(self, event, event_data, event_tag):
       event_data (EventData): event data.
       event_tag (EventTag): event tag.
     """
-    # TODO: since the internal serializer can change move functionality
-    # to serialize into a shared json output module class.
-    # TODO: refactor to separately serialize event and event data
-    copy_of_event = copy.deepcopy(event)
-    for attribute_name, attribute_value in event_data.GetAttributes():
-      setattr(copy_of_event, attribute_name, attribute_value)
-
-    copy_of_event.tag = event_tag
-
-    inode = getattr(event_data, 'inode', None)
-    if inode is None:
-      copy_of_event.inode = 0
-
-    try:
-      message, _ = self._output_mediator.GetFormattedMessages(event_data)
-    except errors.WrongFormatter:
-      message = None
-
-    if message:
-      copy_of_event.message = message
-
-    json_dict = self._JSON_SERIALIZER.WriteSerializedDict(copy_of_event)
-    json_string = json.dumps(json_dict, sort_keys=True)
-    # dumps() returns an ascii-encoded byte string in Python 2.
-    if py2to3.PY_2:
-      json_string = codecs.decode(json_string, 'ascii')
+    json_string = self._WriteSerialized(event, event_data, event_tag)
 
     self._output_writer.Write(json_string)
     self._output_writer.Write('\n')

plaso/output/json_out.py

@@ -3,22 +3,16 @@
 
 from __future__ import unicode_literals
 
-import copy
-import json
-
-from plaso.output import interface
 from plaso.output import manager
-from plaso.serializer import json_serializer
+from plaso.output import shared_json
 
 
-class JSONOutputModule(interface.LinearOutputModule):
+class JSONOutputModule(shared_json.SharedJSONOutputModule):
   """Output module for the JSON format."""
 
   NAME = 'json'
   DESCRIPTION = 'Saves the events into a JSON format.'
 
-  _JSON_SERIALIZER = json_serializer.JSONAttributeContainerSerializer
-
   def __init__(self, output_mediator):
     """Initializes the output module object.
 
@@ -37,21 +31,7 @@ def WriteEventBody(self, event, event_data, event_tag):
       event_data (EventData): event data.
       event_tag (EventTag): event tag.
     """
-    # TODO: since the internal serializer can change move functionality
-    # to serialize into a shared json output module class.
-    # TODO: refactor to separately serialize event and event data
-    copy_of_event = copy.deepcopy(event)
-    for attribute_name, attribute_value in event_data.GetAttributes():
-      setattr(copy_of_event, attribute_name, attribute_value)
-
-    copy_of_event.tag = event_tag
-
-    inode = getattr(event_data, 'inode', None)
-    if inode is None:
-      copy_of_event.inode = 0
-
-    json_dict = self._JSON_SERIALIZER.WriteSerializedDict(copy_of_event)
-    json_string = json.dumps(json_dict, sort_keys=True)
+    json_string = self._WriteSerialized(event, event_data, event_tag)
 
     if self._event_counter != 0:
       self._output_writer.Write(', ')

plaso/output/shared_elastic.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Shared code for Elasticsearch based output modules."""
+"""Shared functionality for Elasticsearch output modules."""
 
 from __future__ import unicode_literals
 

plaso/output/shared_json.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""Shared functionality for JSON based output modules."""
+
+from __future__ import unicode_literals
+
+import abc
+import json
+
+from plaso.lib import errors
+from plaso.lib import py2to3
+from plaso.output import interface
+from plaso.serializer import json_serializer
+
+
+class SharedJSONOutputModule(interface.LinearOutputModule):
+  """Shared functionality for a JSON output module."""
+
+  _JSON_SERIALIZER = json_serializer.JSONAttributeContainerSerializer
+
+  def _WriteSerialized(self, event, event_data, event_tag):
+    """Writes an event, event data and event tag to serialized form.
+
+    Args:
+      event (EventObject): event.
+      event_data (EventData): event data.
+      event_tag (EventTag): event tag.
+
+    Returns:
+      str: A JSON string containing the serialized form.
+    """
+    json_dict = self._WriteSerializedDict(event, event_data, event_tag)
+
+    json_string = json.dumps(json_dict, sort_keys=True)
+    # json.dumps() returns an ascii-encoded byte string in Python 2.
+    if py2to3.PY_2:
+      json_string = json_string.decode('ascii')
+
+    return json_string
+
+  def _WriteSerializedDict(self, event, event_data, event_tag):
+    """Writes an event, event data and event tag to serialized form.
+
+    Args:
+      event (EventObject): event.
+      event_data (EventData): event data.
+      event_tag (EventTag): event tag.
+
+    Returns:
+      dict[str, object]: JSON serialized objects.
+    """
+    event_data_json_dict = self._JSON_SERIALIZER.WriteSerializedDict(event_data)
+    del event_data_json_dict['__container_type__']
+    del event_data_json_dict['__type__']
+
+    inode = event_data_json_dict.get('inode', None)
+    if inode is None:
+      event_data_json_dict['inode'] = 0
+
+    try:
+      message, _ = self._output_mediator.GetFormattedMessages(event_data)
+      event_data_json_dict['message'] = message
+    except errors.WrongFormatter:
+      pass
+
+    event_json_dict = self._JSON_SERIALIZER.WriteSerializedDict(event)
+    event_json_dict['__container_type__'] = 'event'
+
+    event_json_dict.update(event_data_json_dict)
+
+    if event_tag:
+      event_tag_json_dict = self._JSON_SERIALIZER.WriteSerializedDict(event_tag)
+
+      event_json_dict['tag'] = event_tag_json_dict
+
+    return event_json_dict
+
+  @abc.abstractmethod
+  def WriteEventBody(self, event, event_data, event_tag):
+    """Writes event values to the output.
+
+    Args:
+      event (EventObject): event.
+      event_data (EventData): event data.
+      event_tag (EventTag): event tag.
+    """