Skip to content

Commit

Permalink
Add Serialization To Make Parsing GGPK Faster
Browse files Browse the repository at this point in the history 
Add a "Serialize" button in the upper right corner of VisualGGPK.
It serializes RecordOffsets into a Records.bin file.
Next time you open VisualGGPK, select Records.bin and then Content.ggpk to deserialize it.
Note that you cannot use the same Records.bin after you modify Content.ggpk.

Other programs can do the same thing with SerializeRecords() and DeserializeRecords() in GrindingGearsPackageContainer.
  • Loading branch information
@aianlinb
aianlinb authored Mar 16, 2020
1 parent a3fdf3f commit a119a06
Show file tree
Hide file tree
Showing 8 changed files with 340 additions and 26 deletions.
171 changes: 162 additions & 9 deletions LibGGPK/GrindingGearsPackageContainer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class GrindingGearsPackageContainer
/// An estimation of the number of records in the Contents.GGPK file. This is only
/// used to inform the users of the parsing progress.
/// </summary>
private const int EstimatedFileCount = 175000;
private const int EstimatedFileCount = 700000;

public bool IsReadOnly { get { return _isReadOnly; } }
private bool _isReadOnly;
Expand All @@ -48,9 +48,6 @@ public GrindingGearsPackageContainer()
}

#region Read GGPK



/// <summary>
/// Parses the GGPK pack file and builds a directory tree from it.
/// </summary>
Expand All @@ -74,6 +71,163 @@ public void Read(string pathToGgpk, Action<string> output)
}

CreateDirectoryTree(output);

if (output != null)
{
output(Environment.NewLine);
output("Finished!" + Environment.NewLine);
}
}

public void Read(string pathToGgpk, string pathToBin, Action<string> output)
{
_pathToGppk = pathToGgpk;
if (output != null)
{
output("Parsing GGPK..." + Environment.NewLine);
output("Reading bin file records:" + Environment.NewLine);
}

DeserializeRecords(pathToBin, output);

if (output != null)
{
output(Environment.NewLine);
output("Building directory tree..." + Environment.NewLine);
}

CreateDirectoryTree(output);

if (output != null)
{
output(Environment.NewLine);
output("Finished!" + Environment.NewLine);
}
}

public void SerializeRecords(string pathToBin, Action<string> output)
{
if (output != null)
{
output(Environment.NewLine);
output("Serializing... ");
}

var Serialized = File.Create(pathToBin);
var s = new BinaryWriter(Serialized);
foreach (var record in RecordOffsets)
{
s.Write(record.Key);
var baseRecord = record.Value;
if (baseRecord is FileRecord)
{
s.Write((byte)1);
FileRecord fr = (FileRecord)baseRecord;
s.Write(fr.RecordBegin);
s.Write(fr.Length);
s.Write(fr.Hash);
s.Write(fr.Name);
s.Write(fr.DataBegin);
s.Write(fr.DataLength);
}
else if (baseRecord is GgpkRecord)
{
s.Write((byte)2);
GgpkRecord gr = (GgpkRecord)baseRecord;
s.Write(gr.RecordBegin);
s.Write(gr.Length);
s.Write(gr.RecordOffsets.Length);
foreach (long l in gr.RecordOffsets)
{
s.Write(l);
}
}
else if (baseRecord is FreeRecord)
{
s.Write((byte)3);
FreeRecord fr = (FreeRecord)baseRecord;
s.Write(fr.RecordBegin);
s.Write(fr.Length);
s.Write(fr.NextFreeOffset);
}
else if (baseRecord is DirectoryRecord)
{
s.Write((byte)4);
DirectoryRecord dr = (DirectoryRecord)baseRecord;
s.Write(dr.RecordBegin);
s.Write(dr.Length);
s.Write(dr.Hash);
s.Write(dr.Name);
s.Write(dr.EntriesBegin);
s.Write(dr.Entries.Count);
foreach (var directoryEntry in dr.Entries)
{
s.Write(directoryEntry.EntryNameHash);
s.Write(directoryEntry.Offset);
}
}
}
Serialized.Flush();
Serialized.Close();

output?.Invoke("Done!" + Environment.NewLine);
}

public void DeserializeRecords(string pathToBin, Action<string> output)
{
if (output != null)
{
output(Environment.NewLine);
output("Deserializing... ");
}

var Serialized = File.OpenRead(pathToBin);
var s = new BinaryReader(Serialized);
while (Serialized.Length - Serialized.Position > 1)
{
long offset = s.ReadInt64();
switch (s.ReadByte())
{
case 1:
RecordOffsets.Add(offset, new FileRecord(s.ReadInt64(), s.ReadUInt32(), s.ReadBytes(32), s.ReadString(), s.ReadInt64(), s.ReadInt64()));
break;
case 2:
long recordBegin = s.ReadInt64();
uint length = s.ReadUInt32();
long[] recordOffsets = new long[s.ReadInt32()];
for (int i = 0; i < recordOffsets.Length; i++)
{
recordOffsets[i] = s.ReadInt64();
}
RecordOffsets.Add(offset, new GgpkRecord(recordBegin, length, recordOffsets));
break;
case 3:
RecordOffsets.Add(offset, new FreeRecord(s.ReadUInt32(), s.ReadInt64(), s.ReadInt64()));
break;
case 4:
long recordBegin2 = s.ReadInt64();
uint length2 = s.ReadUInt32();
byte[] hash = s.ReadBytes(32);
string name = s.ReadString();
long entriesBegin = s.ReadInt64();
int entriesCount = s.ReadInt32();
var entries = new List<DirectoryRecord.DirectoryEntry>(entriesCount);
for (int i = 0; i < entriesCount; i++)
{
entries.Add(new DirectoryRecord.DirectoryEntry
{
EntryNameHash = s.ReadUInt32(),
Offset = s.ReadInt64(),
});
}
RecordOffsets.Add(offset, new DirectoryRecord(recordBegin2, length2, hash, name, entriesBegin, entries));
break;
}
}
Serialized.Flush();
Serialized.Close();

output?.Invoke("Done!" + Environment.NewLine);
}

/// <summary>
Expand All @@ -99,8 +253,7 @@ private void ReadRecordOffsets(string pathToGgpk, Action<string> output)
var percentComplete = currentOffset / (float)streamLength;
if (percentComplete - previousPercentComplete >= 0.10f)
{
if (output != null)
output(String.Format("\t{0:00.00}%{1}", 100.0 * percentComplete, Environment.NewLine));
output?.Invoke(String.Format("\t{0:00.00}%{1}", 100.0 * percentComplete, Environment.NewLine));
previousPercentComplete = percentComplete;
}
}
Expand Down Expand Up @@ -329,11 +482,11 @@ public void Save(string pathToGgpkNew, Action<string> output)
if (!(percentComplete - previousPercentComplete >= 0.05f)) return;
if (output != null)
output(String.Format(" {0:00.00}%", 100.0*percentComplete));
output(String.Format(" {0:00.00}%", 100.0 * percentComplete));
previousPercentComplete = percentComplete;
});
if (output != null) output(" 100%");

// write root directory
var rootDirectoryOffset = writer.BaseStream.Position;
DirectoryRoot.Record.Write(writer, changedOffsets);
Expand All @@ -349,7 +502,7 @@ public void Save(string pathToGgpkNew, Action<string> output)
ggpkRecordNew.RecordOffsets[0] = rootDirectoryOffset;
ggpkRecordNew.RecordOffsets[1] = firstFreeRecordOffset;
ggpkRecordNew.Write(writer, changedOffsets);
if (output != null)
if (output != null)
output("Finished !!!");
}
}
Expand Down
10 changes: 10 additions & 0 deletions LibGGPK/Records/DirectoryRecord.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,16 @@ public DirectoryRecord(uint length, BinaryReader br)
Read(br);
}

public DirectoryRecord(long recordBegin, uint length, byte[] hash, string name, long entriesBegin, List<DirectoryEntry> entries)
{
RecordBegin = recordBegin;
Length = length;
Hash = hash;
Name = name;
EntriesBegin = entriesBegin;
Entries = entries;
}

/// <summary>
/// Reads the PDIR record entry from the specified stream
/// </summary>
Expand Down
12 changes: 11 additions & 1 deletion LibGGPK/Records/FileRecord.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,16 @@ public enum DataFormat
/// </summary>
public DirectoryTreeNode ContainingDirectory;

public FileRecord(long recordBegin, uint length, byte[] hash, string name, long dataBegin, long dataLength)
{
RecordBegin = recordBegin;
Length = length;
Hash = hash;
Name = name;
DataBegin = dataBegin;
DataLength = dataLength;
}

public FileRecord(uint length, BinaryReader br)
{
RecordBegin = br.BaseStream.Position - 8;
Expand Down Expand Up @@ -248,7 +258,7 @@ public DataFormat FileFormat
{
if (Name.Equals("GameObjectRegister"))
return DataFormat.Unicode;
return KnownFileFormats[Path.GetExtension(Name).ToLower()];
return KnownFileFormats.ContainsKey(Path.GetExtension(Name).ToLower()) ? KnownFileFormats[Path.GetExtension(Name).ToLower()] : DataFormat.Unknown;
}
}

Expand Down
7 changes: 7 additions & 0 deletions LibGGPK/Records/GGPKRecord.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ public GgpkRecord(uint length, BinaryReader br)
Read(br);
}

public GgpkRecord(long recordBegin, uint length, long[] recordOffsets)
{
RecordBegin = recordBegin;
Length = length;
RecordOffsets = recordOffsets;
}

/// <summary>
/// Reads the GGPK record entry from the specified stream
/// </summary>
Expand Down
9 changes: 5 additions & 4 deletions VisualGGPK/MainWindow.xaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
xmlns:Properties="clr-namespace:VisualGGPK.Properties"
x:Class="VisualGGPK.MainWindow"
Title="VisualGGPK"
Height="500" Width="1000"
Height="600" Width="1100"
Loaded="Window_Loaded"
AllowDrop="True"
Drop="Window_Drop_1"
Expand Down Expand Up @@ -47,7 +47,7 @@
</TreeView.ContextMenu>
</TreeView>
<StackPanel Grid.Row="1" >
<Button Content="Save" Click="OnSaveClicked" Width ="50"/>
<Button x:Name="SaveButton" Content="Save" Click="OnSaveClicked" Width ="50"/>
</StackPanel>
</Grid>

Expand All @@ -71,8 +71,9 @@
<TextBox x:Name="TextBoxNameHash" Width="100" IsReadOnly="True"/>

<Label x:Name="LabelFileHash" Content="{x:Static Properties:Resources.MainWindow_Label_FileHash}"/>
<TextBox x:Name="TextBoxHash" Width="100" IsReadOnly="True"/>
<TextBox x:Name="TextBoxHash" Width="100" IsReadOnly="True"/>
<CheckBox x:Name="AutoScrollCheckBox" Content="AutoScroll" VerticalAlignment="Center" HorizontalAlignment="Center" Margin="10,0,0,0" IsChecked="True"/>
<Button x:Name="SerializeButton" Content="Serialize" Click="OnSerializeClicked" VerticalAlignment="Top" HorizontalAlignment="Center" Margin="10,3,0,0" Width ="65"/>
</StackPanel>

<!--different viewers for different types of files-->
Expand All @@ -90,4 +91,4 @@

</Grid>
</Grid>
</Window>
</Window>
Loading

0 comments on commit a119a06

Please sign in to comment.