Skip to content

Commit

Permalink
7thWrapperLib: Remove runtime chunk support
Browse files Browse the repository at this point in the history 
As FFNx now does support field chunk files, we do not need to rebuild the LGP in memory. This will save some RAM space and improve loading performance
  • Loading branch information
@julianxhokaxhiu
julianxhokaxhiu committed Dec 15, 2022
1 parent d4e7f27 commit 988407a
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 287 deletions.
26 changes: 0 additions & 26 deletions 7thWrapperLib/Profile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,31 +182,6 @@ public RuntimeMod(string folder, IEnumerable<ConditionalFolder> conditionalFolde

}

private void ScanChunk()
{
_chunkFiles = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string file in _archive.AllFileNames())
{
string[] parts = file.Split('\\');
if (parts.Length > 2)
{
if (ExtraFolders.Contains(parts[0]) || Conditionals.Any(cf => cf.Folder.Equals(parts[0], StringComparison.InvariantCultureIgnoreCase)))
{
parts = parts.Skip(1).ToArray();
}
else
continue;
}
if (parts.Length < 2) continue;
int chunk = parts[1].IndexOf(".chunk.", StringComparison.InvariantCultureIgnoreCase);
if (chunk > 0)
{
_chunkFiles.Add(parts[0] + "\\" + parts[1].Substring(0, chunk));
}
}
DebugLogger.WriteLine(" Finished scan for chunks, found " + String.Join(",", _chunkFiles));
}

/// <summary>
/// Ensure the <see cref="_archive"/> is loaded if mod is an .iro
/// </summary>
Expand All @@ -216,7 +191,6 @@ public void Startup()
{
DebugLogger.WriteLine(" Loading archive " + BaseFolder);
_archive = new IrosArc(BaseFolder);
ScanChunk();
}
}

Expand Down
261 changes: 0 additions & 261 deletions 7thWrapperLib/VFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,98 +145,6 @@ abstract class VRangeCleanup : VRange {
public abstract void Cleanup(DateTime cutoff);
}

class VRangeChunked : VRange {
private bool _lastHeader = false, _lastInit;
private List<RuntimeMod> _mods;
private byte[] _calculated, _header;
private IntPtr _fbHandle;
private uint _fbOffset;
private int _fbLen;

public string Name { get; set; }

public VRangeChunked(string name, List<RuntimeMod> mods, IntPtr fallbackHandle, uint fallbackOffset, int fallbackLen) {
_mods = mods;
_fbHandle = fallbackHandle;
_fbOffset = fallbackOffset;
_fbLen = fallbackLen;
_header = new byte[24];
byte[] bname = System.Text.Encoding.ASCII.GetBytes(name);
Array.Copy(bname, _header, bname.Length);
Bytes.WriteInt(_header, 20, _fbLen);
}

private unsafe void Recalculate() {
DebugLogger.WriteLine($"Chunked file {Name} recalculating contents...");
byte[] original = new byte[_fbLen];
Win32.OVERLAPPED ov = new Win32.OVERLAPPED() {
EventHandle = IntPtr.Zero,
Internal = UIntPtr.Zero,
InternalHigh = UIntPtr.Zero,
Offset = _fbOffset,
OffsetHigh = 0
};

fixed (byte* bp = &original[0]) {
uint bytesRead = 0;
//TODO should loop...
Win32.ReadFile(_fbHandle, new IntPtr(bp), (uint)_fbLen, ref bytesRead, ref ov);
}
DebugLogger.WriteLine($"Original read {_fbLen} from {_fbOffset} sig {original[0]} {original[1]} {original[2]} {original[3]}");

var orig = FieldFile.Unchunk(original);

foreach (int i in Enumerable.Range(0, orig.Count)) {
string fn = Name + ".chunk." + (i + 1);
foreach (var of in _mods.SelectMany(m => m.GetOverrides(fn))) {
if (of.CFolder == null || of.CFolder.IsActive(of.CName)) {
if (of.Archive == null) {
orig[i] = System.IO.File.ReadAllBytes(of.File);
} else {
orig[i] = of.Archive.GetBytes(of.File);
}
break;
}
}
}

_calculated = FieldFile.Chunk(orig);
Bytes.WriteInt(_header, 20, _calculated.Length);
DebugLogger.WriteLine($"New length of {Name} is {_calculated.Length}");
}

public override void Read(uint offset, uint length, IntPtr dest, ref uint bytesRead) {
DebugLogger.WriteLine($"Chunked {Name} reading from OS {offset} L {length}");

bool header = (offset < 24);
bool init = (offset == 24);

if ((!_lastHeader && header) || (init && !_lastInit) || _calculated == null) { //re-evaluate
Recalculate();
_lastInit = true; //we're ready to read file headers
} else {
_lastInit = init;
}
_lastHeader = header;

if (offset < 24) {
length = Math.Min(length, 24 - offset);
Util.CopyToIntPtr(_header, dest, (int)length, (int)offset);
bytesRead = length;
DebugLogger.WriteLine($"Chunked {Name} reading from cheader - {bytesRead} bytes read [current size is {BitConverter.ToInt32(_header, 20)}]");
return;
} else {
offset -= 24;
if (offset >= _calculated.Length) {
bytesRead = length; return; //leave with garbage...
}
int len = Math.Min((int)length, _calculated.Length - (int)offset);
Util.CopyToIntPtr(_calculated, dest, len, (int)offset);
bytesRead = (uint)len;
}
}
}

class VRangeConditional : VRangeCleanup {

private IntPtr _handle;
Expand Down Expand Up @@ -399,175 +307,6 @@ public static bool AnyOverridesFor(string path, RuntimeProfile profile) {
if (item.OverridesFolder(path)) return true;
return false;
}

public LGPWrapper(IntPtr handle, string name, RuntimeProfile profile) {
if (!AnyOverridesFor(name, profile)) {
DebugLogger.WriteLine($"LGPWrapper: no overrides for {name}, early out");
return; //optimisation, don't run anything else, if no override files
}
IsActive = true;

DebugLogger.WriteLine(" LGPWrapper: Parsing");
var fs = new System.IO.FileStream(new Microsoft.Win32.SafeHandles.SafeFileHandle(handle, false), FileAccess.Read);
ProcMonParser.DataFile df = ProcMonParser.FF7Files.LoadLGP(fs, name);
fs.Position = 0;

Dictionary<string, int> sortKeys = df.Items.ToDictionary(i => i.Name, i => i.Index, StringComparer.InvariantCulture);
Dictionary<string, long> dataKeys = df.Items.ToDictionary(i => i.Name, i => i.Start, StringComparer.InvariantCulture);
Dictionary<string, int> filesSizes = new Dictionary<string, int>(StringComparer.InvariantCultureIgnoreCase);
Dictionary<string, int> filesOptions = new Dictionary<string, int>(StringComparer.InvariantCultureIgnoreCase);
Dictionary<string, ProcMonParser.DataItem> lgpItems = df.Items.ToDictionary(i => i.Name, i => i, StringComparer.InvariantCulture);
foreach (var item in df.Items) {
filesOptions[item.Name] = 0;
filesSizes[item.Name] = item.Length - 24;
DebugLogger.DetailedWriteLine($"Checking chunk support for {name}~{item.Name}~");
if (profile.Mods.Any(m => m.SupportsChunks(System.IO.Path.Combine(name, item.Name)))) {
filesSizes[item.Name] = Math.Max(filesSizes[item.Name], 1024 * 1024 * 2); //This is a horrible hack. TODO.
filesOptions[item.Name] |= 0x1;
}
}
DebugLogger.WriteLine(" LGPWrapper: Prepared structures");
List<string> names = profile.Mods.SelectMany(m => m.GetPathOverrideNames(name)).Distinct(StringComparer.InvariantCultureIgnoreCase).ToList();
foreach (string fname in names) {
if (fname.IndexOf(".chunk.", StringComparison.InvariantCultureIgnoreCase) >= 0) continue;
if (!filesSizes.ContainsKey(fname)) {
filesSizes[fname] = 0;
DebugLogger.WriteLine($"Added LGP file {name} {fname}");
}
var overrides = profile.Mods.SelectMany(m => m.GetOverrides(System.IO.Path.Combine(name, fname)));
foreach(var over in overrides) {
filesSizes[fname] = Math.Max(filesSizes[fname], over.Size);
if (over.CFolder == null) break;
}
}
List<LGPEntry> entries = filesSizes.Select(kv => new LGPEntry(kv.Key, kv.Value)).ToList();
List<LGP.LGPEntryMade> newentries;

DebugLogger.WriteLine(" LGPWrapper: creating new headers");
byte[] headers = LGP.CalculateHeaders(entries,
s => {
int index;
sortKeys.TryGetValue(s, out index);
return index;
},
s => {
long index;
dataKeys.TryGetValue(s, out index);
return (uint)index;
},
out newentries
);
int datastart = headers.Length;
DebugLogger.WriteLine($" LGPWrapper: Calculated new LGP headers for {name} with {entries.Count} file entries");

/*
int datastart = df.Items[0].Start;
byte[] headers = new byte[datastart];
fs.Read(headers, 0, datastart);
fs.Position = 0;
*/

uint offset = (uint)datastart;
VFile = new VFile();
try {
VFile.Add(new VRangeInline() { Start = 0, Length = offset, Data = headers, Tag = "Headers" });
int count = 0;
foreach (var item in newentries.OrderBy(em => em.DataIndex)) {
DebugLogger.DetailedWriteLine($"LGPWrapper calculate {item.Entry.Name}");
//Bytes.WriteUInt(headers, 16 + 20 + 27 * item.Index, offset);
string fn = System.IO.Path.Combine(name, item.Entry.Name);
var overrides = profile.Mods.SelectMany(m => m.GetOverrides(fn));

int fOptions;
filesOptions.TryGetValue(item.Entry.Name, out fOptions);
bool chunked = (fOptions & 0x1) != 0;
//var overrides = Enumerable.Empty<OverrideFile>();
//DebugLogger.WriteLine("Virtualizing LGP entry {0} at offset {1}", item.Entry.Name, offset);
if (item.DataOffset != offset) throw new Exception("LGPWrapper mismatch on offset for " + item.Entry.Name + " offset=" + offset + " hoffset=" + item.DataOffset);

if (chunked) {
long pos = lgpItems[item.Entry.Name].Start + 24;
int len = lgpItems[item.Entry.Name].Length - 24;
VFile.Add(new VRangeChunked(item.Entry.Name, profile.Mods, handle, (uint)pos, len) {
Start = offset,
Length = (uint)(item.Entry.MaxSize + 24),
Name = fn,
Tag = "Chunked"
});
//DebugLogger.WriteLine("File {0} initialized with chunks", item.Entry.Name, 0);
offset += (uint)item.Entry.MaxSize + 24;
} else if (!overrides.Any()) { //take from original LGP
long pos = lgpItems[item.Entry.Name].Start;
VFile.Add(new VRangeHandle() { Start = offset, Length = (uint)(item.Entry.MaxSize + 24), Handle = handle, Offset = (uint)pos, Tag = item.Entry.Name });
offset += (uint)item.Entry.MaxSize + 24;
//DebugLogger.WriteLine("--VRangeHandle");
} else if (overrides.First().CFolder == null) { //only one override, replace directly
var ov = overrides.First();

byte[] fheader = new byte[24];
System.Text.Encoding.ASCII.GetBytes(item.Entry.Name, 0, item.Entry.Name.Length, fheader, 0);
Bytes.WriteUInt(fheader, 20, (uint)ov.Size);
VFile.Add(new VRangeInline() { Start = offset, Length = 24, Data = fheader, Tag = item.Entry.Name + "%header" });
offset += 24;
VRange vr;
if (ov.Archive == null) {
var vf = new VRangeFile() { Start = offset, Length = (uint)ov.Size, Filename = ov.File, Tag = ov.File };
VFile.Add(vf);
_wFiles.Add(vf);
vr = vf;
//DebugLogger.WriteLine("LGP entry {0} coming from file {1}", item.Entry.Name, ov.File);
} else {
vr = new VRangeArchive() { Start = offset, Length = (uint)ov.Size, File = ov.File, Archive = ov.Archive, Tag = ov.File };
VFile.Add(vr);
//DebugLogger.WriteLine("LGP entry {0} coming from archive file {1} with size {2}", item.Entry.Name, ov.File, ov.Size);
}
//if (vr.Length != item.Entry.MaxSize)
//DebugLogger.WriteLine("Entry {0} size difference {1} vs {2}", item.Entry.Name, vr.Length, item.Entry.MaxSize);
if (vr.Length < item.Entry.MaxSize) {
uint diff = (uint)item.Entry.MaxSize - vr.Length;
VFile.Add(new VRangeNull() { Length = diff, Start = vr.Start + vr.Length, Tag = "Padding" });
}
offset += (uint)item.Entry.MaxSize;
//DebugLogger.WriteLine("--VRangeFile");
} else { //multiple overrides; tricky!
//DebugLogger.WriteLine("Add VRangeConditional for " + item.Entry.Name);
ProcMonParser.DataItem di;
lgpItems.TryGetValue(item.Entry.Name, out di);
uint fbOffset = (di == null) ? 0 : (uint)di.Start;

var vcond = new VRangeConditional(item.Entry.Name, overrides.ToList(), handle, fbOffset) { Length = (uint)item.Entry.MaxSize + 24, Start = offset, Name = item.Entry.Name, Tag = item.Entry.Name };
VFile.Add(vcond);
_wFiles.Add(vcond);
offset += (uint)item.Entry.MaxSize + 24;
//DebugLogger.WriteLine("--VRangeConditional");
}
count++;
/*
string file = MapFile(System.IO.Path.Combine(name, item.Name), profile);
if (file != null) {
uint length = (uint)new System.IO.FileInfo(file).Length;
byte[] fheader = new byte[24];
System.Text.Encoding.ASCII.GetBytes(item.Name, 0, item.Name.Length, fheader, 0);
Bytes.WriteUInt(fheader, 20, length);
VFile.Add(new VRangeInline() { Start = offset, Length = 24, Data = fheader });
offset += 24;
var vf = new VRangeFile() { Start = offset, Length = length, Filename = file };
VFile.Add(vf);
_wFiles.Add(vf);
offset += length;
} else {
VFile.Add(new VRangeHandle() { Start = offset, Length = (uint)(item.Length), Handle = handle, Offset = (uint)item.Start });
offset += (uint)item.Length;
}*/
}
byte[] footer = System.Text.Encoding.ASCII.GetBytes("FINAL FANTASY7");
VFile.Add(new VRangeInline() { Start = offset, Length = (uint)footer.Length, Data = footer, Tag = "footer" });
DebugLogger.WriteLine("Created: " + VFile.ToString());
} catch {
VFile.Dump();
throw;
}
}
}

class VArchiveData
Expand Down

0 comments on commit 988407a

Please sign in to comment.