Code review: 276290043: Added initial version of NTFS UsnJrnl parser #…

…316

config/dpkg/changelog

@@ -2,4 +2,4 @@ python-plaso (1.3.1-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline <log2timeline-dev@googlegroups.com>  Tue, 08 Dec 2015 14:54:03 +0100
+ -- Log2Timeline <log2timeline-dev@googlegroups.com>  Tue, 08 Dec 2015 22:46:30 +0100

plaso/dependencies.py

@@ -16,7 +16,7 @@
     u'pyevt': 20120410,
     u'pyevtx': 20141112,
     u'pyewf': 20131210,
-    u'pyfsntfs': 20150831,
+    u'pyfsntfs': 20151130,
     u'pyfwsi': 20150606,
     u'pylnk': 20150830,
     u'pymsiecf': 20150314,

plaso/engine/worker.py

@@ -47,6 +47,7 @@ class BaseEventExtractionWorker(queue.ItemQueueConsumer):
       u'/$Extend/$Reparse',
       u'/$Extend/$RmMetadata/$Repair',
       u'/$Extend/$RmMetadata/$TxfLog/$Tops',
+      u'/$Extend/$UsnJrnl',
       u'/$LogFile',
       u'/$MFT',
       u'/$MFTMirr',
@@ -92,6 +93,7 @@ def __init__(
     self._produced_number_of_path_specs = 0
     self._resolver_context = resolver_context
     self._specification_store = None
+    self._usnjrnl_parser_object = None
 
     self._event_queue_producer = event_queue_producer
     self._parse_error_queue_producer = parse_error_queue_producer
@@ -486,36 +488,53 @@ def _ProcessFileEntry(self, file_entry, data_stream_name=u''):
 
     self._parser_mediator.SetFileEntry(file_entry)
 
-    logging.debug(u'[ProcessFileEntry] parsing file: {0:s}'.format(
+    logging.debug(u'[ProcessFileEntry] processing file entry: {0:s}'.format(
         self._current_display_name))
 
-    is_metadata_file = self._IsMetadataFile(file_entry)
-
-    # Not every file entry has a data stream. In such cases we want to
-    # extract the metadata only.
-    has_data_stream = file_entry.HasDataStream(data_stream_name)
 
     try:
-      if has_data_stream:
-        self._HashDataStream(file_entry, data_stream_name=data_stream_name)
+      if self._IsMetadataFile(file_entry):
+        parent_path_spec = getattr(file_entry.path_spec, u'parent', None)
+        if (self._usnjrnl_parser_object and parent_path_spec and
+            file_entry.name == u'$UsnJrnl' and data_stream_name == u'$J'):
+
+          # To be able to ignore the sparse data ranges the UsnJrnl parser
+          # needs to read directly from the volume.
+          volume_file_object = path_spec_resolver.Resolver.OpenFileObject(
+              parent_path_spec, resolver_context=self._resolver_context)
+
+          try:
+            self._ParseFileEntryWithParser(
+                self._usnjrnl_parser_object, file_entry,
+                file_object=volume_file_object)
+          finally:
+            volume_file_object.close()
+
+      else:
+        # Not every file entry has a data stream. In such cases we want to
+        # extract the metadata only.
+        has_data_stream = file_entry.HasDataStream(data_stream_name)
+
+        if has_data_stream:
+          self._HashDataStream(file_entry, data_stream_name=data_stream_name)
 
-      # We always want to use the filestat parser if set but we only want
-      # to invoke it once per file entry, so we only use it if we are
-      # processing the default (nameless) data stream.
-      if not data_stream_name and self._filestat_parser_object:
-        self._ParseFileEntryWithParser(self._filestat_parser_object, file_entry)
+        # We always want to use the filestat parser if set but we only want
+        # to invoke it once per file entry, so we only use it if we are
+        # processing the default (nameless) data stream.
+        if not data_stream_name and self._filestat_parser_object:
+          self._ParseFileEntryWithParser(
+              self._filestat_parser_object, file_entry)
 
-      is_archive = False
-      is_compressed_stream = False
+        is_archive = False
+        is_compressed_stream = False
 
-      if not is_metadata_file and file_entry.IsFile():
-        is_compressed_stream = self._ProcessCompressedStreamFile(file_entry)
-        if not is_compressed_stream:
-          is_archive = self._ProcessArchiveFile(file_entry)
+        if file_entry.IsFile():
+          is_compressed_stream = self._ProcessCompressedStreamFile(file_entry)
+          if not is_compressed_stream:
+            is_archive = self._ProcessArchiveFile(file_entry)
 
-      if (has_data_stream and not is_archive and not is_compressed_stream and
-          not is_metadata_file):
-        self._ProcessDataStream(file_entry, data_stream_name=data_stream_name)
+        if has_data_stream and not is_archive and not is_compressed_stream:
+          self._ProcessDataStream(file_entry, data_stream_name=data_stream_name)
 
     finally:
       if reference_count != self._resolver_context.GetFileObjectReferenceCount(
@@ -537,8 +556,9 @@ def _ProcessFileEntry(self, file_entry, data_stream_name=u''):
     if self._enable_profiling:
       self._ProfilingSampleMemory()
 
-    logging.debug(u'[ParseFileEntry] done processing: {0:s}'.format(
-        self._current_display_name))
+    logging.debug(
+        u'[ProcessFileEntry] done processing file entry: {0:s}'.format(
+            self._current_display_name))
 
   def _ProcessPathSpec(self, path_spec):
     """Processes a path specification.
@@ -656,7 +676,7 @@ def InitializeParserObjects(self, parser_filter_string=None):
 
     self._non_sigscan_parser_names = []
     for parser_name in non_sigscan_parser_names:
-      if parser_name == u'filestat':
+      if parser_name in [u'filestat', u'usnjrnl']:
         continue
       self._non_sigscan_parser_names.append(parser_name)
 
@@ -670,6 +690,10 @@ def InitializeParserObjects(self, parser_filter_string=None):
     if u'filestat' in self._parser_objects:
       del self._parser_objects[u'filestat']
 
+    self._usnjrnl_parser_object = self._parser_objects.get(u'usnjrnl', None)
+    if u'usnjrnl' in self._parser_objects:
+      del self._parser_objects[u'usnjrnl']
+
   def Run(self):
     """Extracts event objects from file entries."""
     self._parser_mediator.ResetCounters()

plaso/events/file_system_events.py

@@ -2,6 +2,7 @@
 """This file contains the file system specific event object classes."""
 
 from plaso.events import time_events
+from plaso.lib import eventdata
 
 
 class FileStatEvent(time_events.TimestampEvent):
@@ -43,17 +44,17 @@ class NTFSFileStatEvent(time_events.FiletimeEvent):
   Attributes:
     attribute_type: the attribute type e.g. 0x00000030 which represents
                     $FILE_NAME.
-    file_attribute_flags: the NTFS file attribute flags, set to None
-                          if not available.
-    file_reference: NTFS file reference.
+    file_attribute_flags: the NTFS file attribute flags or None if
+                          not available.
+    file_reference: integer containing the NTFS file reference.
     file_system_type: the file system type.
     is_allocated: boolean value to indicate the MFT entry is allocated
                  (marked as in use).
     name: string containing the name associated with the stat event, e.g.
-          that of a $FILE_NAME attribute, set to None if not available.
-    offset: the offset of the stat data.
-    parent_file_reference: NTFS file reference of the parent, set to None
-                           if not available.
+          that of a $FILE_NAME attribute or None if not available.
+    offset: integer containing the offset of the stat data.
+    parent_file_reference: optional integer containing the NTFS file
+                           reference of the parent or None if not available.
   """
 
   DATA_TYPE = u'fs:stat:ntfs'
@@ -67,25 +68,77 @@ def __init__(
     Args:
       timestamp: the FILETIME value for the timestamp.
       timestamp_description: the usage string for the timestamp value.
-      file_reference: NTFS file reference.
+      file_reference: integer containing the NTFS file reference.
       attribute_type: the attribute type e.g. 0x00000030 which represents
                       $FILE_NAME.
-      file_attribute_flags: optional NTFS file attribute flags, set to None
+      file_attribute_flags: optional NTFS file attribute flags or None
                             if not available.
       is_allocated: optional boolean value to indicate the MFT entry is
                     is allocated (marked as in use).
       name: optional string containing the name associated with the stat event,
-            e.g.  that of a $FILE_NAME attribute, set to None if not available.
-      parent_file_reference: optional NTFS file reference of the parent, set
-                             to None if not available.
+            e.g. that of a $FILE_NAME attribute or None if not available.
+      parent_file_reference: optional integer containing the NTFS file
+                             reference of the parent or None if not available.
     """
     super(NTFSFileStatEvent, self).__init__(timestamp, timestamp_description)
 
     self.attribute_type = attribute_type
-    self.file_reference = file_reference
     self.file_attribute_flags = file_attribute_flags
+    self.file_reference = file_reference
     self.file_system_type = u'NTFS'
     self.is_allocated = is_allocated
     self.name = name
     self.offset = 0
     self.parent_file_reference = parent_file_reference
+
+
+class NTFSUSNChangeEvent(time_events.FiletimeEvent):
+  """NTFS USN change event.
+
+  Attributes:
+    file_attribute_flags: the NTFS file attribute flags or None if
+                          not available.
+    file_reference: integer containing the NTFS file reference.
+    update_sequence_number: integer containing the update sequence number.
+    update_source_flags: integer containing the update source flags.
+    update_reason_flags: integer containing the update reason flags.
+    filename: string containing the name of the file associated with the event.
+    file_system_type: the file system type.
+    offset: integer containing the offset of the corresponding USN record.
+    parent_file_reference: optional integer containing the NTFS file
+                           reference of the parent or None if not available.
+  """
+
+  DATA_TYPE = u'fs:ntfs:usn_change'
+
+  def __init__(
+      self, timestamp, offset, filename, file_reference, update_sequence_number,
+      update_source_flags, update_reason_flags, file_attribute_flags=None,
+      parent_file_reference=None):
+    """Initializes the event object.
+
+    Args:
+      timestamp: the FILETIME value for the timestamp.
+      offset: integer containing the offset of the corresponding USN record.
+      filename: string containing the name of the file associated with
+                the event.
+      file_reference: integer containing the NTFS file reference.
+      update_sequence_number: integer containing the update sequence number.
+      update_source_flags: integer containing the update source flags.
+      update_reason_flags: integer containing the update reason flags.
+      file_attribute_flags: optional NTFS file attribute flags or None
+                            if not available.
+      parent_file_reference: optional integer containing the NTFS file
+                             reference of the parent or None if not available.
+    """
+    super(NTFSUSNChangeEvent, self).__init__(
+        timestamp, eventdata.EventTimestamp.ENTRY_MODIFICATION_TIME)
+
+    self.file_attribute_flags = file_attribute_flags
+    self.file_reference = file_reference
+    self.filename = filename
+    self.offset = offset
+    self.parent_file_reference = parent_file_reference
+    self.update_reason_flags = update_reason_flags
+    self.update_sequence_number = update_sequence_number
+    self.update_source_flags = update_source_flags

plaso/formatters/file_system.py

@@ -120,11 +120,12 @@ def GetMessages(self, unused_formatter_mediator, event_object):
     event_values[u'attribute_name'] = self._ATTRIBUTE_NAMES.get(
         attribute_type, u'UNKNOWN')
 
-    file_reference = event_values.get(u'file_reference', 0)
-    event_values[u'file_reference'] = u'{0:d}-{1:d}'.format(
-        file_reference & 0xffffffffffff, file_reference >> 48)
+    file_reference = event_values.get(u'file_reference', None)
+    if file_reference:
+      event_values[u'file_reference'] = u'{0:d}-{1:d}'.format(
+          file_reference & 0xffffffffffff, file_reference >> 48)
 
-    parent_file_reference = event_values.get(u'parent_file_reference', 0)
+    parent_file_reference = event_values.get(u'parent_file_reference', None)
     if parent_file_reference:
       event_values[u'parent_file_reference'] = u'{0:d}-{1:d}'.format(
           parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
@@ -135,5 +136,103 @@ def GetMessages(self, unused_formatter_mediator, event_object):
     return self._ConditionalFormatMessages(event_values)
 
 
+class NTFSUSNChangeEventFormatter(interface.ConditionalEventFormatter):
+  """The NTFS USN change event formatter."""
+
+  DATA_TYPE = u'fs:ntfs:usn_change'
+
+  FORMAT_STRING_PIECES = [
+      u'{filename}',
+      u'File reference: {file_reference}',
+      u'Parent file reference: {parent_file_reference}',
+      u'Update source: {update_source}',
+      u'Update reason: {update_reason}']
+
+  FORMAT_STRING_SHORT_PIECES = [
+      u'{filename}',
+      u'{file_reference}',
+      u'{update_reason}']
+
+  SOURCE_SHORT = u'FILE'
+
+  _USN_REASON_FLAGS = {
+      0x00000001: u'USN_REASON_DATA_OVERWRITE',
+      0x00000002: u'USN_REASON_DATA_EXTEND',
+      0x00000004: u'USN_REASON_DATA_TRUNCATION',
+      0x00000010: u'USN_REASON_NAMED_DATA_OVERWRITE',
+      0x00000020: u'USN_REASON_NAMED_DATA_EXTEND',
+      0x00000040: u'USN_REASON_NAMED_DATA_TRUNCATION',
+      0x00000100: u'USN_REASON_FILE_CREATE',
+      0x00000200: u'USN_REASON_FILE_DELETE',
+      0x00000400: u'USN_REASON_EA_CHANGE',
+      0x00000800: u'USN_REASON_SECURITY_CHANGE',
+      0x00001000: u'USN_REASON_RENAME_OLD_NAME',
+      0x00002000: u'USN_REASON_RENAME_NEW_NAME',
+      0x00004000: u'USN_REASON_INDEXABLE_CHANGE',
+      0x00008000: u'USN_REASON_BASIC_INFO_CHANGE',
+      0x00010000: u'USN_REASON_HARD_LINK_CHANGE',
+      0x00020000: u'USN_REASON_COMPRESSION_CHANGE',
+      0x00040000: u'USN_REASON_ENCRYPTION_CHANGE',
+      0x00080000: u'USN_REASON_OBJECT_ID_CHANGE',
+      0x00100000: u'USN_REASON_REPARSE_POINT_CHANGE',
+      0x00200000: u'USN_REASON_STREAM_CHANGE',
+      0x00400000: u'USN_REASON_TRANSACTED_CHANGE',
+      0x80000000: u'USN_REASON_CLOSE'}
+
+  _USN_SOURCE_FLAGS = {
+      0x00000001: u'USN_SOURCE_DATA_MANAGEMENT',
+      0x00000002: u'USN_SOURCE_AUXILIARY_DATA',
+      0x00000004: u'USN_SOURCE_REPLICATION_MANAGEMENT'}
+
+  def GetMessages(self, unused_formatter_mediator, event_object):
+    """Determines the formatted message strings for an event object.
+
+    Args:
+      formatter_mediator: the formatter mediator object (instance of
+                          FormatterMediator).
+      event_object: the event object (instance of EventObject).
+
+    Returns:
+      A tuple containing the formatted message string and short message string.
+
+    Raises:
+      WrongFormatter: if the event object cannot be formatted by the formatter.
+    """
+    if self.DATA_TYPE != event_object.data_type:
+      raise errors.WrongFormatter(u'Unsupported data type: {0:s}.'.format(
+          event_object.data_type))
+
+    event_values = event_object.GetValues()
+
+    file_reference = event_values.get(u'file_reference', None)
+    if file_reference:
+      event_values[u'file_reference'] = u'{0:d}-{1:d}'.format(
+          file_reference & 0xffffffffffff, file_reference >> 48)
+
+    parent_file_reference = event_values.get(u'parent_file_reference', None)
+    if parent_file_reference:
+      event_values[u'parent_file_reference'] = u'{0:d}-{1:d}'.format(
+          parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
+
+    update_reason_flags = event_values.get(u'update_reason_flags', 0)
+    update_reasons = []
+    for bitmask, description in sorted(self._USN_REASON_FLAGS.items()):
+      if bitmask & update_reason_flags:
+        update_reasons.append(description)
+
+    event_values[u'update_reason'] = u', '.join(update_reasons)
+
+    update_source_flags = event_values.get(u'update_source_flags', 0)
+    update_sources = []
+    for bitmask, description in sorted(self._USN_SOURCE_FLAGS.items()):
+      if bitmask & update_source_flags:
+        update_sources.append(description)
+
+    event_values[u'update_source'] = u', '.join(update_sources)
+
+    return self._ConditionalFormatMessages(event_values)
+
+
 manager.FormattersManager.RegisterFormatters([
-    FileStatEventFormatter, NTFSFileStatEventFormatter])
+    FileStatEventFormatter, NTFSFileStatEventFormatter,
+    NTFSUSNChangeEventFormatter])