From a119a0653cfb43bf03a33b7306bc7a20758bc29b Mon Sep 17 00:00:00 2001 From: aianlinb Date: Mon, 16 Mar 2020 20:08:52 +0800 Subject: [PATCH] Add Serialization To Make Parsing GGPK Faster Add a "Serialize" button in the upper right corner of VisualGGPK. It serializes RecordOffsets into a Records.bin file. Next time you open VisualGGPK, select Records.bin and then Content.ggpk to deserialize it. Note that you cannot use the same Records.bin after you modify Content.ggpk. Other programs can do the same thing with SerializeRecords() and DeserializeRecords() in GrindingGearsPackageContainer. --- LibGGPK/GrindingGearsPackageContainer.cs | 171 ++++++++++++++++++-- LibGGPK/Records/DirectoryRecord.cs | 10 ++ LibGGPK/Records/FileRecord.cs | 12 +- LibGGPK/Records/GGPKRecord.cs | 7 + VisualGGPK/MainWindow.xaml | 9 +- VisualGGPK/MainWindow.xaml.cs | 151 +++++++++++++++-- VisualGGPK/Properties/Resources.Designer.cs | 4 +- VisualGGPK/Properties/Resources.resx | 2 +- 8 files changed, 340 insertions(+), 26 deletions(-) diff --git a/LibGGPK/GrindingGearsPackageContainer.cs b/LibGGPK/GrindingGearsPackageContainer.cs index da2cadf..36db641 100644 --- a/LibGGPK/GrindingGearsPackageContainer.cs +++ b/LibGGPK/GrindingGearsPackageContainer.cs @@ -29,7 +29,7 @@ public class GrindingGearsPackageContainer /// An estimation of the number of records in the Contents.GGPK file. This is only /// used to inform the users of the parsing progress. /// - private const int EstimatedFileCount = 175000; + private const int EstimatedFileCount = 700000; public bool IsReadOnly { get { return _isReadOnly; } } private bool _isReadOnly; @@ -48,9 +48,6 @@ public GrindingGearsPackageContainer() } #region Read GGPK - - - /// /// Parses the GGPK pack file and builds a directory tree from it. /// @@ -74,6 +71,163 @@ public void Read(string pathToGgpk, Action output) } CreateDirectoryTree(output); + + if (output != null) + { + output(Environment.NewLine); + output("Finished!" + Environment.NewLine); + } + } + + public void Read(string pathToGgpk, string pathToBin, Action output) + { + _pathToGppk = pathToGgpk; + if (output != null) + { + output("Parsing GGPK..." + Environment.NewLine); + output("Reading bin file records:" + Environment.NewLine); + } + + DeserializeRecords(pathToBin, output); + + if (output != null) + { + output(Environment.NewLine); + output("Building directory tree..." + Environment.NewLine); + } + + CreateDirectoryTree(output); + + if (output != null) + { + output(Environment.NewLine); + output("Finished!" + Environment.NewLine); + } + } + + public void SerializeRecords(string pathToBin, Action output) + { + if (output != null) + { + output(Environment.NewLine); + output("Serializing... "); + } + + var Serialized = File.Create(pathToBin); + var s = new BinaryWriter(Serialized); + foreach (var record in RecordOffsets) + { + s.Write(record.Key); + var baseRecord = record.Value; + if (baseRecord is FileRecord) + { + s.Write((byte)1); + FileRecord fr = (FileRecord)baseRecord; + s.Write(fr.RecordBegin); + s.Write(fr.Length); + s.Write(fr.Hash); + s.Write(fr.Name); + s.Write(fr.DataBegin); + s.Write(fr.DataLength); + } + else if (baseRecord is GgpkRecord) + { + s.Write((byte)2); + GgpkRecord gr = (GgpkRecord)baseRecord; + s.Write(gr.RecordBegin); + s.Write(gr.Length); + s.Write(gr.RecordOffsets.Length); + foreach (long l in gr.RecordOffsets) + { + s.Write(l); + } + } + else if (baseRecord is FreeRecord) + { + s.Write((byte)3); + FreeRecord fr = (FreeRecord)baseRecord; + s.Write(fr.RecordBegin); + s.Write(fr.Length); + s.Write(fr.NextFreeOffset); + } + else if (baseRecord is DirectoryRecord) + { + s.Write((byte)4); + DirectoryRecord dr = (DirectoryRecord)baseRecord; + s.Write(dr.RecordBegin); + s.Write(dr.Length); + s.Write(dr.Hash); + s.Write(dr.Name); + s.Write(dr.EntriesBegin); + s.Write(dr.Entries.Count); + foreach (var directoryEntry in dr.Entries) + { + s.Write(directoryEntry.EntryNameHash); + s.Write(directoryEntry.Offset); + } + } + } + Serialized.Flush(); + Serialized.Close(); + + output?.Invoke("Done!" + Environment.NewLine); + } + + public void DeserializeRecords(string pathToBin, Action output) + { + if (output != null) + { + output(Environment.NewLine); + output("Deserializing... "); + } + + var Serialized = File.OpenRead(pathToBin); + var s = new BinaryReader(Serialized); + while (Serialized.Length - Serialized.Position > 1) + { + long offset = s.ReadInt64(); + switch (s.ReadByte()) + { + case 1: + RecordOffsets.Add(offset, new FileRecord(s.ReadInt64(), s.ReadUInt32(), s.ReadBytes(32), s.ReadString(), s.ReadInt64(), s.ReadInt64())); + break; + case 2: + long recordBegin = s.ReadInt64(); + uint length = s.ReadUInt32(); + long[] recordOffsets = new long[s.ReadInt32()]; + for (int i = 0; i < recordOffsets.Length; i++) + { + recordOffsets[i] = s.ReadInt64(); + } + RecordOffsets.Add(offset, new GgpkRecord(recordBegin, length, recordOffsets)); + break; + case 3: + RecordOffsets.Add(offset, new FreeRecord(s.ReadUInt32(), s.ReadInt64(), s.ReadInt64())); + break; + case 4: + long recordBegin2 = s.ReadInt64(); + uint length2 = s.ReadUInt32(); + byte[] hash = s.ReadBytes(32); + string name = s.ReadString(); + long entriesBegin = s.ReadInt64(); + int entriesCount = s.ReadInt32(); + var entries = new List(entriesCount); + for (int i = 0; i < entriesCount; i++) + { + entries.Add(new DirectoryRecord.DirectoryEntry + { + EntryNameHash = s.ReadUInt32(), + Offset = s.ReadInt64(), + }); + } + RecordOffsets.Add(offset, new DirectoryRecord(recordBegin2, length2, hash, name, entriesBegin, entries)); + break; + } + } + Serialized.Flush(); + Serialized.Close(); + + output?.Invoke("Done!" + Environment.NewLine); } /// @@ -99,8 +253,7 @@ private void ReadRecordOffsets(string pathToGgpk, Action output) var percentComplete = currentOffset / (float)streamLength; if (percentComplete - previousPercentComplete >= 0.10f) { - if (output != null) - output(String.Format("\t{0:00.00}%{1}", 100.0 * percentComplete, Environment.NewLine)); + output?.Invoke(String.Format("\t{0:00.00}%{1}", 100.0 * percentComplete, Environment.NewLine)); previousPercentComplete = percentComplete; } } @@ -329,11 +482,11 @@ public void Save(string pathToGgpkNew, Action output) if (!(percentComplete - previousPercentComplete >= 0.05f)) return; if (output != null) - output(String.Format(" {0:00.00}%", 100.0*percentComplete)); + output(String.Format(" {0:00.00}%", 100.0 * percentComplete)); previousPercentComplete = percentComplete; }); if (output != null) output(" 100%"); - + // write root directory var rootDirectoryOffset = writer.BaseStream.Position; DirectoryRoot.Record.Write(writer, changedOffsets); @@ -349,7 +502,7 @@ public void Save(string pathToGgpkNew, Action output) ggpkRecordNew.RecordOffsets[0] = rootDirectoryOffset; ggpkRecordNew.RecordOffsets[1] = firstFreeRecordOffset; ggpkRecordNew.Write(writer, changedOffsets); - if (output != null) + if (output != null) output("Finished !!!"); } } diff --git a/LibGGPK/Records/DirectoryRecord.cs b/LibGGPK/Records/DirectoryRecord.cs index c234354..45a887d 100644 --- a/LibGGPK/Records/DirectoryRecord.cs +++ b/LibGGPK/Records/DirectoryRecord.cs @@ -51,6 +51,16 @@ public DirectoryRecord(uint length, BinaryReader br) Read(br); } + public DirectoryRecord(long recordBegin, uint length, byte[] hash, string name, long entriesBegin, List entries) + { + RecordBegin = recordBegin; + Length = length; + Hash = hash; + Name = name; + EntriesBegin = entriesBegin; + Entries = entries; + } + /// /// Reads the PDIR record entry from the specified stream /// diff --git a/LibGGPK/Records/FileRecord.cs b/LibGGPK/Records/FileRecord.cs index 7fa3d25..3f8e028 100644 --- a/LibGGPK/Records/FileRecord.cs +++ b/LibGGPK/Records/FileRecord.cs @@ -132,6 +132,16 @@ public enum DataFormat /// public DirectoryTreeNode ContainingDirectory; + public FileRecord(long recordBegin, uint length, byte[] hash, string name, long dataBegin, long dataLength) + { + RecordBegin = recordBegin; + Length = length; + Hash = hash; + Name = name; + DataBegin = dataBegin; + DataLength = dataLength; + } + public FileRecord(uint length, BinaryReader br) { RecordBegin = br.BaseStream.Position - 8; @@ -248,7 +258,7 @@ public DataFormat FileFormat { if (Name.Equals("GameObjectRegister")) return DataFormat.Unicode; - return KnownFileFormats[Path.GetExtension(Name).ToLower()]; + return KnownFileFormats.ContainsKey(Path.GetExtension(Name).ToLower()) ? KnownFileFormats[Path.GetExtension(Name).ToLower()] : DataFormat.Unknown; } } diff --git a/LibGGPK/Records/GGPKRecord.cs b/LibGGPK/Records/GGPKRecord.cs index 276cdb4..4e9c050 100644 --- a/LibGGPK/Records/GGPKRecord.cs +++ b/LibGGPK/Records/GGPKRecord.cs @@ -32,6 +32,13 @@ public GgpkRecord(uint length, BinaryReader br) Read(br); } + public GgpkRecord(long recordBegin, uint length, long[] recordOffsets) + { + RecordBegin = recordBegin; + Length = length; + RecordOffsets = recordOffsets; + } + /// /// Reads the GGPK record entry from the specified stream /// diff --git a/VisualGGPK/MainWindow.xaml b/VisualGGPK/MainWindow.xaml index 315bf03..2faa5f6 100644 --- a/VisualGGPK/MainWindow.xaml +++ b/VisualGGPK/MainWindow.xaml @@ -5,7 +5,7 @@ xmlns:Properties="clr-namespace:VisualGGPK.Properties" x:Class="VisualGGPK.MainWindow" Title="VisualGGPK" - Height="500" Width="1000" + Height="600" Width="1100" Loaded="Window_Loaded" AllowDrop="True" Drop="Window_Drop_1" @@ -47,7 +47,7 @@ -