Merge pull request #1 from Velocidex/i30

Implement I30 carving.

.gitignore

@@ -1,3 +1,4 @@
 *~
 vendor/
-ntfs
+ntfs
+ntfs.exe

attribute.go

@@ -467,6 +467,37 @@ type INDEX_NODE_HEADER struct {
 	vtypes.Object
 }
 
+func (self *INDEX_NODE_HEADER) GetRecords() []*INDEX_RECORD_ENTRY {
+	result := []*INDEX_RECORD_ENTRY{}
+
+	end := self.Get("offset_to_end_index_entry").AsInteger() + self.Offset()
+	start := self.Get("offset_to_index_entry").AsInteger() + self.Offset()
+
+	// Need to fit the last entry in - it should be at least size of FILE_NAME
+	dummy_record, _ := self.Profile().Create(
+		"FILE_NAME", 0, self.Reader(), nil)
+
+	for i := start; i+dummy_record.Size() < end; {
+		record, err := self.Profile().Create(
+			"INDEX_RECORD_ENTRY", i, self.Reader(), nil)
+		if err != nil {
+			return result
+		}
+
+		result = append(result, &INDEX_RECORD_ENTRY{record})
+
+		// Records have varied sizes.
+		size_of_record := record.Get("sizeOfIndexEntry").AsInteger()
+		if size_of_record == 0 {
+			break
+		}
+
+		i += size_of_record
+	}
+
+	return result
+}
+
 type INDEX_RECORD_ENTRY struct {
 	vtypes.Object
 }

bin/stat.go

@@ -15,6 +15,9 @@ var (
 	stat_command_detailed = stat_command.Flag(
 		"verbose", "Show verbose information").Bool()
 
+	stat_command_i30 = stat_command.Flag(
+		"i30", "Carve out $I30 entries").Bool()
+
 	stat_command_file_arg = stat_command.Arg(
 		"file", "The image file to inspect",
 	).Required().File()
@@ -50,6 +53,7 @@ func doSTAT() {
 		if err != nil {
 			fmt.Printf("FullPath error: %s\n", err)
 		}
+
 	} else {
 		stat, err := ntfs.ModelMFTEntry(mft_entry)
 		kingpin.FatalIfError(err, "Can not open path")
@@ -59,6 +63,16 @@ func doSTAT() {
 
 		fmt.Println(string(serialized))
 	}
+
+	if *stat_command_i30 {
+		i30_list := ntfs.ExtractI30List(mft_entry)
+		kingpin.FatalIfError(err, "Can not extract $I30")
+
+		serialized, err := json.MarshalIndent(i30_list, " ", " ")
+		kingpin.FatalIfError(err, "Marshal")
+
+		fmt.Println(string(serialized))
+	}
 }
 
 func init() {

fixtures/TestNTFS.golden

@@ -131,6 +131,18 @@
   "00000060  0c 00 00 00                                       |....|",
   ""
  ],
+ "02.1 I30": [
+  {
+   "MFTId": "0x2e",
+   "Mtime": "2018-09-24T07:56:35Z",
+   "Atime": "2018-09-24T07:56:35Z",
+   "Ctime": "2018-09-24T07:56:35Z",
+   "Name": "Hello world text document.txt",
+   "NameType": "POSIX",
+   "IsDir": false,
+   "Size": 0
+  }
+ ],
  "03 Hello world.txt": "12: Hello world!",
  "04 Hello world.txt:goodbye.txt": "20: Goodbye cruel world.",
  "05 Compressed ones.bin hash": "2949120: f581eebdb9a49a622c305d4e44977bc91a4a1204"

go.mod

@@ -0,0 +1,14 @@
+module www.velocidex.com/golang/go-ntfs
+
+require (
+	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc
+	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf
+	github.com/davecgh/go-spew v1.1.1
+	github.com/mattn/go-runewidth v0.0.3
+	github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561
+	github.com/stretchr/testify v1.2.2
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6
+	www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b
+)

go.sum

@@ -0,0 +1,20 @@
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc h1:rQ1O4ZLYR2xXHXgBCCfIIGnuZ0lidMQw2S5n1oOv+Wg=
+github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
+github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b h1:z5v5o1dhtzaxvlWm6qSTYZ4OTr56Ol2JpM1Y5Wu9zQE=
+www.velocidex.com/golang/vtypes v0.0.0-20180924145839-b0d509f8925b/go.mod h1:tXxIx8UJuI81Hoxcv0DTq2a1Pi1H6l1uCf4dhqUSUkw=

i30.go

@@ -0,0 +1,78 @@
+package ntfs
+
+import (
+	"time"
+)
+
+func ExtractI30List(mft_entry *MFT_ENTRY) []*FileInfo {
+	records := []*INDEX_RECORD_ENTRY{}
+
+	for _, node := range mft_entry.DirNodes() {
+		records = append(records, node.GetRecords()...)
+		records = append(records, node.ScanSlack()...)
+	}
+
+	result := []*FileInfo{}
+	for _, record := range records {
+		if !record.IsValid() {
+			continue
+		}
+
+		filename := &FILE_NAME{record.Get("file")}
+		result = append(result, &FileInfo{
+			MFTId: record.Get("mftReference").AsString(),
+			Mtime: time.Unix(
+				filename.Get("file_modified").
+					AsInteger(), 0),
+			Atime: time.Unix(
+				filename.Get("file_accessed").
+					AsInteger(), 0),
+			Ctime: time.Unix(
+				filename.Get("mft_modified").
+					AsInteger(), 0),
+			Name:     filename.Name(),
+			NameType: filename.Get("name_type").AsString(),
+		})
+	}
+
+	return result
+}
+
+const (
+	earliest_valid_time = 1000000000 // Sun Sep  9 11:46:40 2001
+	latest_valid_time   = 2000000000 // Wed May 18 13:33:20 2033
+)
+
+func (self *INDEX_RECORD_ENTRY) IsValid() bool {
+	test_filename := &FILE_NAME{self.Get("file")}
+
+	for _, field := range []string{
+		"file_modified", "file_accessed",
+		"mft_modified", "created"} {
+		test_time := test_filename.Get(field).AsInteger()
+		if test_time < earliest_valid_time || test_time > latest_valid_time {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (self *INDEX_NODE_HEADER) ScanSlack() []*INDEX_RECORD_ENTRY {
+	result := []*INDEX_RECORD_ENTRY{}
+
+	// start at the last record and carve until the end of the
+	// allocation.
+	start := self.Get("offset_to_end_index_entry").AsInteger()
+	end := self.Get("sizeOfEntriesAlloc").AsInteger() - 0x52
+	for off := start; off < end; off++ {
+		test_struct_obj, _ := self.Profile().Create(
+			"INDEX_RECORD_ENTRY", off, self.Reader(), nil)
+		test_struct := &INDEX_RECORD_ENTRY{test_struct_obj}
+		if test_struct.IsValid() {
+			result = append(result, test_struct)
+		}
+	}
+
+	return result
+}

mft.go

@@ -339,27 +339,7 @@ func (self *MFT_ENTRY) Dir() []*INDEX_RECORD_ENTRY {
 	result := []*INDEX_RECORD_ENTRY{}
 
 	for _, node := range self.DirNodes() {
-		start := node.Get("offset_to_index_entry").AsInteger() + node.Offset()
-		end := node.Get("offset_to_end_index_entry").AsInteger() + node.Offset()
-
-		// Need to fit the last entry in - it should be at least size of FILE_NAME
-		for i := start; i+66 < end; {
-			record, err := self.Profile().Create(
-				"INDEX_RECORD_ENTRY", i, node.Reader(), nil)
-			if err != nil {
-				return result
-			}
-
-			result = append(result, &INDEX_RECORD_ENTRY{record})
-
-			// Records have varied sizes.
-			size_of_record := record.Get("sizeOfIndexEntry").AsInteger()
-			if size_of_record == 0 {
-				break
-			}
-
-			i += size_of_record
-		}
+		result = append(result, node.GetRecords()...)
 	}
 	return result
 }

model.go

@@ -1,6 +1,9 @@
 package ntfs
 
-import "time"
+import (
+	"fmt"
+	"time"
+)
 
 // This file defines a model for MFT entry.
 
@@ -17,21 +20,37 @@ type FilenameInfo struct {
 	Name  string
 }
 
+type Attribute struct {
+	Type   string
+	TypeId int64
+	Id     int64
+	Inode  string
+	Size   int64
+	Name   string
+}
+
+// Describe a single MFT entry.
 type NTFSFileInformation struct {
 	FullPath  string
 	MFTID     int64
 	Size      int64
 	Allocated bool
 	IsDir     bool
 	SI_Times  *TimeStamps
+
+	// If multiple filenames are given, we list them here.
 	Filenames []*FilenameInfo
+
+	Attributes []*Attribute
 }
 
 func ModelMFTEntry(mft_entry *MFT_ENTRY) (*NTFSFileInformation, error) {
 	full_path, _ := GetFullPath(mft_entry)
+	mft_id := mft_entry.Get("record_number").AsInteger()
+
 	result := &NTFSFileInformation{
 		FullPath:  full_path,
-		MFTID:     mft_entry.Get("record_number").AsInteger(),
+		MFTID:     mft_id,
 		Allocated: mft_entry.Get("flags").Get("ALLOCATED").AsInteger() != 0,
 		IsDir:     mft_entry.Get("flags").Get("DIRECTORY").AsInteger() != 0,
 	}
@@ -69,14 +88,21 @@ func ModelMFTEntry(mft_entry *MFT_ENTRY) (*NTFSFileInformation, error) {
 
 	for _, attr := range mft_entry.Attributes() {
 		// $DATA attribute = 128.
-		if attr.Get("type").AsInteger() == 128 {
-			if attr.IsResident() {
-				result.Size = attr.Get("content_size").AsInteger()
-			} else {
-				result.Size = attr.Get("actual_size").AsInteger()
-			}
-			break
+		if attr.Get("type").AsInteger() == 128 && result.Size == 0 {
+			result.Size = attr.Size()
 		}
+
+		attr_type := attr.Get("type")
+		attr_id := attr.Get("attribute_id").AsInteger()
+		result.Attributes = append(result.Attributes, &Attribute{
+			Type:   attr_type.AsString(),
+			TypeId: attr_type.AsInteger(),
+			Inode: fmt.Sprintf("%v-%v-%v",
+				mft_id, attr_type.AsInteger(), attr_id),
+			Size: attr.Size(),
+			Id:   attr_id,
+			Name: attr.Name(),
+		})
 	}
 
 	return result, nil

ntfs.go

@@ -53,6 +53,8 @@ const NTFS_PROFILE = `
                 "160": "$INDEX_ALLOCATION",
                 "176": "$BITMAP",
                 "192": "$REPARSE_POINT",
+                "208": "$EA_INFORMATION",
+                "224": "$EA",
                 "256": "$LOGGED_UTILITY_STREAM"
             }
         }]],
@@ -153,7 +155,7 @@ const NTFS_PROFILE = `
         "usn": [64, ["unsigned int"]]
     }],
 
-    "FILE_NAME": [0, {
+    "FILE_NAME": [66, {
         "mftReference": [0, ["BitField", {
             "target": "unsigned long long",
             "start_bit": 0,
@@ -232,7 +234,8 @@ const NTFS_PROFILE = `
 
     "INDEX_NODE_HEADER": [16, {
         "offset_to_index_entry": [0, ["unsigned int"]],
-        "offset_to_end_index_entry": [4, ["unsigned int"]]
+        "offset_to_end_index_entry": [4, ["unsigned int"]],
+        "sizeOfEntriesAlloc": [8, ["unsigned int"]]
     }],
 
     "ATTRIBUTE_LIST_ENTRY": [0, {

ntfs_test.go

@@ -40,6 +40,7 @@ func TestNTFS(t *testing.T) {
 
 	result["01 Open by path"] = ntfs.ListDir(dir)
 	result["02 Folder B stat"] = split(dir.DebugString())
+	result["02.1 I30"] = ntfs.ExtractI30List(dir)
 
 	// Open by mft id
 	mft_idx, attr, id, err := ntfs.ParseMFTId("46-128-5")