FF8: Add LZ4 support for FS archives + minor bugfix (#741)

CMakeLists.txt

@@ -90,6 +90,7 @@ find_package(OPENPSF REQUIRED)
 find_package(tomlplusplus CONFIG REQUIRED)
 find_package(STEAMWORKSSDK CONFIG REQUIRED)
 find_package(xxHash CONFIG REQUIRED)
+find_package(LZ4 REQUIRED)
 find_package(CMakeRC CONFIG REQUIRED)
 find_package(HWINFO REQUIRED)
 
@@ -154,6 +155,7 @@ target_link_libraries(
   tomlplusplus::tomlplusplus
   STEAMWORKSSDK::STEAMWORKSSDK
   xxHash::xxhash
+  lz4::lz4
   Vorbis::vorbisfile
   Vorbis::vorbis
   MPG123::libmpg123

Changelog.md

@@ -4,9 +4,11 @@
 
 ## FF8
 
+- Core: Fix crash for non-us versions introduced in 1.20.3 ( https://github.com/julianxhokaxhiu/FFNx/pull/741 )
 - Ambient: Fix missing Battle ID for battles triggered using field opcodes
 - Ambient: Fix Battle ID detection for random encounters in Field
 - Modding: Allow modding card names hardcoded in exe ( https://github.com/julianxhokaxhiu/FFNx/pull/739 )
+- Modding: Add compatibility to LZ4 compression in FS archives ( https://github.com/julianxhokaxhiu/FFNx/pull/741 )
 
 # 1.20.3
 

misc/FFNx.toml

@@ -246,6 +246,7 @@ external_music_sync = false
 # - FF7 1998: music/vgmstream ( 7h-era compatibility )
 # - FF7 eStore: data/music_ogg
 # - FF7 Steam: data/music_ogg
+# - FF8 2000/Steam: data/music/dmusic/ogg
 #~~~~~~~~~~~~~~~~~~~~~~~~~~~
 external_music_path = ""
 

src/common.cpp

@@ -902,6 +902,11 @@ int common_create_window(HINSTANCE hInstance, struct game_obj* game_object)
 				if (more_debug)
 				{
 					replace_function(common_externals.debug_print2, external_debug_print2);
+
+					if (ff8)
+					{
+						patch_code_dword(ff8_externals.outputdebugstringa, DWORD(external_debug_print));
+					}
 				}
 
 #ifdef NO_EXT_HEAP
@@ -2789,7 +2794,7 @@ void get_userdata_path(PCHAR buffer, size_t bufSize, bool isSavegameFile)
 			else
 			{
 				// Search for the first "user_" match in the game path
-				CHAR searchPath[260];
+				CHAR searchPath[MAX_PATH];
 				WIN32_FIND_DATA pathFound;
 				HANDLE hFind;
 
@@ -3107,6 +3112,7 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
 				patch_code_dword(mciSendCommandA, (DWORD)dotemuMciSendCommandA);
 			}
 
+			if (external_music_path.empty()) external_music_path = "data/music/dmusic/ogg";
 			if (external_music_volume < 0) external_music_volume = 100;
 			if (external_sfx_volume < 0) external_sfx_volume = 100;
 			if (external_voice_volume < 0) external_voice_volume = 100;
@@ -3298,7 +3304,7 @@ __declspec(dllexport) HANDLE __stdcall dotemuCreateFileA(LPCSTR lpFileName, DWOR
 	if (strstr(lpFileName, "CD:") != NULL)
 	{
 		CHAR newPath[MAX_PATH]{ 0 };
-		uint8_t requiredDisk = *(uint8_t*)(*(DWORD*)ff8_externals.requiredDisk + 0xCC);
+		uint8_t requiredDisk = *(uint8_t*)(*(DWORD*)ff8_externals.savemap + 0xCC);
 		CHAR diskAsChar[2];
 
 		itoa(requiredDisk, diskAsChar, 10);

src/ff8.h

@@ -151,8 +151,8 @@ struct struc_38
 
 struct ff8_file_fi_infos
 {
-	int pos;
 	int size;
+	int pos;
 	int compression;
 };
 
@@ -1253,8 +1253,8 @@ struct ff8_externals
 	uint32_t sub_545EA0;
 	uint32_t sub_545F10;
 	uint32_t sub_465720;
-	uint32_t requiredDisk;
 	uint32_t sm_battle_sound;
+	uint32_t outputdebugstringa;
 	uint32_t sdmusicplay;
 	uint32_t(*sd_music_play)(uint32_t, char*, uint32_t);
 	uint32_t sd_music_play_at;
@@ -1353,10 +1353,12 @@ struct ff8_externals
 	uint32_t ff8input_cfg_read;
 	uint32_t ff8input_cfg_reset;
 	char *(*strcpy_with_malloc)(const char *);
-	uint32_t moriya_filesytem_open;
-	uint32_t moriya_filesytem_seek;
-	uint32_t moriya_filesytem_read;
-	uint32_t moriya_filesytem_close;
+	uint32_t moriya_filesystem_open;
+	uint32_t moriya_filesystem_seek;
+	uint32_t moriya_filesystem_read;
+	uint32_t moriya_filesystem_close;
+	uint32_t read_or_uncompress_fs_data;
+	uint32_t lzs_uncompress;
 	void(*free_file_container)(ff8_file_container *);
 	uint32_t field_get_dialog_string;
 	uint32_t set_window_object;

src/ff8/file.cpp

@@ -28,9 +28,13 @@
 
 #include <fcntl.h>
 #include <io.h>
+#include <lz4.h>
 
 char next_direct_file[MAX_PATH] = "";
 bool last_fopen_is_redirected = false;
+uint32_t last_compression_type = 0;
+size_t last_compressed_size = 0;
+size_t last_uncompressed_size = 0;
 
 size_t get_fl_prefix_size()
 {
@@ -67,7 +71,7 @@ int ff8_fs_archive_search_filename2(const char *fullpath, ff8_file_fi_infos *fi_
 		// Lookup without the language in the path
 		size_t prefix_size = get_fl_prefix_size();
 
-		if (trace_all || trace_files) ffnx_trace("%s: file not found, searching again with another language %s...\n", __func__, fullpath + prefix_size);
+		if (trace_all || trace_files) ffnx_warning("%s: file not found, searching again with another language %s...\n", __func__, fullpath + prefix_size);
 
 		for (int id = 0; id < file_container->fl_infos->file_count; ++id)
 		{
@@ -118,6 +122,66 @@ void ff8_fs_archive_free_file_container_sub_archive(ff8_file_container *file_con
 	return ff8_externals.free_file_container(file_container);
 }
 
+void ff8_fs_archive_patch_compression(uint32_t compression_type)
+{
+	if (trace_all || trace_files) ffnx_trace("%s compression_type=%d\n", __func__, compression_type);
+
+	last_compression_type = compression_type;
+}
+
+uint8_t *ff8_fs_archive_malloc_source_data(size_t size, char *source_code_path, int line)
+{
+	if (trace_all || trace_files) ffnx_trace("%s size=%d\n", __func__, size);
+
+	last_compressed_size = size;
+
+	return (uint8_t *)common_externals.assert_malloc(size, source_code_path, line);
+}
+
+uint8_t *ff8_fs_archive_malloc_target_data(size_t size, char *source_code_path, int line)
+{
+	if (trace_all || trace_files) ffnx_trace("%s size=%d\n", __func__, size);
+
+	if (last_compression_type == 2) // LZ4 compression
+	{
+		size += 10;
+
+		last_uncompressed_size = size;
+	}
+
+	return (uint8_t *)common_externals.assert_malloc(size, source_code_path, line);
+}
+
+void ff8_fs_archive_uncompress_data(const uint8_t *source_data, uint8_t *target_data)
+{
+	if (trace_all || trace_files) ffnx_trace("%s\n", __func__);
+
+	if (last_compression_type == 2) // LZ4 compression
+	{
+		if (trace_all || trace_files) ffnx_trace("%s LZ4 compression detected\n", __func__);
+
+		int uncompressed_size = LZ4_decompress_safe((const char *)source_data + 8, (char *)target_data, last_compressed_size - 8, last_uncompressed_size);
+
+		if (uncompressed_size < 0)
+		{
+			ffnx_error("%s: cannot uncompress lz4 file data (compressed_size=%d, error=%d)\n", __func__, last_compressed_size, uncompressed_size);
+
+			return;
+		}
+
+		if (uncompressed_size != last_uncompressed_size)
+		{
+			ffnx_warning("%s: uncompressed size is different than expected: %d != %d\n", __func__, uncompressed_size, last_uncompressed_size);
+
+			return;
+		}
+	}
+	else
+	{
+		((void(*)(const uint8_t*, uint8_t*))ff8_externals.lzs_uncompress)(source_data, target_data);
+	}
+}
+
 bool ff8_attempt_redirection(const char *in, char *out, size_t size)
 {
 	// Remove AppPath from input
@@ -136,15 +200,16 @@ int ff8_open(const char *fileName, int oflag, ...)
 	va_list va;
 
 	va_start(va, oflag);
-	int pmode = va_arg(va, int);
+	int pmode = va_arg(va, DWORD);
+	const int shflag = _SH_DENYNO;
 
 	if (trace_all || trace_files) ffnx_trace("%s: %s oflag=%X pmode=%X\n", __func__, fileName, oflag, pmode);
 
 	if (next_direct_file && *next_direct_file != '\0')
 	{
 		if (trace_all || trace_direct) ffnx_info("Direct file using %s\n", next_direct_file);
 
-		int ret = ff8_externals._sopen(next_direct_file, oflag, 64, pmode);
+		int ret = ff8_externals._sopen(next_direct_file, oflag, shflag, pmode);
 
 		*next_direct_file = '\0';
 
@@ -156,7 +221,7 @@ int ff8_open(const char *fileName, int oflag, ...)
 
 	last_fopen_is_redirected = is_redirected;
 
-	int ret = ff8_externals._sopen(is_redirected ? _filename : fileName, oflag, 64, pmode);
+	int ret = ff8_externals._sopen(is_redirected ? _filename : fileName, oflag, shflag, pmode);
 
 	last_fopen_is_redirected = false;
 
@@ -167,11 +232,13 @@ FILE *ff8_fopen(const char *fileName, const char *mode)
 {
 	if (trace_all || trace_files) ffnx_trace("%s: %s mode=%s\n", __func__, fileName, mode);
 
+	const int shflag = _SH_DENYNO;
+
 	if (next_direct_file && *next_direct_file != '\0')
 	{
 		if (trace_all || trace_direct) ffnx_info("Direct file using %s\n", next_direct_file);
 
-		FILE *file = ff8_externals._fsopen(next_direct_file, mode, 64);
+		FILE *file = ff8_externals._fsopen(next_direct_file, mode, shflag);
 
 		*next_direct_file = '\0';
 
@@ -183,7 +250,7 @@ FILE *ff8_fopen(const char *fileName, const char *mode)
 
 	last_fopen_is_redirected = is_redirected;
 
-	FILE *file = ff8_externals._fsopen(is_redirected ? _filename : fileName, mode, 64);
+	FILE *file = ff8_externals._fsopen(is_redirected ? _filename : fileName, mode, shflag);
 
 	last_fopen_is_redirected = false;
 
@@ -195,7 +262,8 @@ ff8_file *ff8_open_file(ff8_file_context *infos, const char *fs_path)
 	if (trace_all || trace_files) ffnx_trace("%s: %s mode=%d callback=%p noOpen=%d archive=%p field_4=%d\n", __func__, fs_path, infos->mode, infos->filename_callback, infos->field_4, infos->file_container, infos->field_4);
 
 	ff8_file *file;
-	char fullpath[256];
+	char fullpath[MAX_PATH];
+	const int shflag = _SH_DENYNO;
 
 	if (infos->filename_callback != nullptr)
 	{
@@ -249,7 +317,7 @@ ff8_file *ff8_open_file(ff8_file_context *infos, const char *fs_path)
 
 				set_direct_path(fullpath, direct_path, sizeof(direct_path));
 
-				file->fd = ff8_externals._sopen(direct_path, oflag, 64, pmode);
+				file->fd = ff8_externals._sopen(direct_path, oflag, shflag, pmode);
 
 				if (file->fd != -1)
 				{
@@ -274,7 +342,7 @@ ff8_file *ff8_open_file(ff8_file_context *infos, const char *fs_path)
 				last_fopen_is_redirected = is_redirected;
 
 				// We need to use the external _open, and not the official one
-				file->fd = ff8_externals._sopen(is_redirected ? _filename : fullpath, oflag, 64, pmode);
+				file->fd = ff8_externals._sopen(is_redirected ? _filename : fullpath, oflag, shflag, pmode);
 
 				last_fopen_is_redirected = false;
 			}

src/ff8/file.h

@@ -29,6 +29,10 @@
 int ff8_fs_archive_search_filename2(const char *fullpath, ff8_file_fi_infos *fi_infos_for_the_path, const ff8_file_container *file_container);
 int ff8_fs_archive_search_filename_sub_archive(const char *fullpath, ff8_file_fi_infos *fi_infos_for_the_path, const ff8_file_container *file_container);
 void ff8_fs_archive_free_file_container_sub_archive(ff8_file_container *file_container);
+void ff8_fs_archive_patch_compression(uint32_t compression_type);
+uint8_t *ff8_fs_archive_malloc_source_data(size_t size, char *source_code_path, int line);
+uint8_t *ff8_fs_archive_malloc_target_data(size_t size, char *source_code_path, int line);
+void ff8_fs_archive_uncompress_data(const uint8_t *source_data, uint8_t *target_data);
 
 int ff8_open(const char *fileName, int oflag, ...);
 FILE *ff8_fopen(const char *fileName, const char *mode);

src/ff8_data.cpp

@@ -151,11 +151,13 @@ void ff8_find_externals()
 	ff8_externals.engine_set_init_time = get_relative_call(ff8_externals.battle_enter, 0x35);
 
 	common_externals.debug_print2 = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0x16);
-	ff8_externals.moriya_filesytem_open = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0x21);
-	ff8_externals.moriya_filesytem_seek = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0x77);
-	ff8_externals.moriya_filesytem_read = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0xB7);
-	ff8_externals.moriya_filesytem_close = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0xDD);
-	ff8_externals.free_file_container = (void(*)(ff8_file_container*))get_relative_call(ff8_externals.moriya_filesytem_close, 0x1F);
+	ff8_externals.moriya_filesystem_open = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0x21);
+	ff8_externals.moriya_filesystem_seek = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0x77);
+	ff8_externals.moriya_filesystem_read = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0xB7);
+	ff8_externals.moriya_filesystem_close = get_relative_call(uint32_t(ff8_externals.sm_pc_read), 0xDD);
+	ff8_externals.read_or_uncompress_fs_data = get_relative_call(ff8_externals.moriya_filesystem_read, 0x5C);
+	ff8_externals.lzs_uncompress = get_relative_call(ff8_externals.read_or_uncompress_fs_data, 0x1E6);
+	ff8_externals.free_file_container = (void(*)(ff8_file_container*))get_relative_call(ff8_externals.moriya_filesystem_close, 0x1F);
 
 	ff8_externals.cdcheck_sub_52F9E0 = get_relative_call(ff8_externals.cdcheck_main_loop, 0x95);
 
@@ -184,7 +186,7 @@ void ff8_find_externals()
 	ff8_externals.cardgame_tim_texture_icons = (uint8_t *)get_absolute_value(ff8_externals.sub_534640, 0x125);
 	ff8_externals.sub_539500 = get_relative_call(ff8_externals.sub_534640, 0x110);
 	ff8_externals.cardgame_tim_texture_font = (uint8_t *)get_absolute_value(ff8_externals.sub_539500, 0x1);
-	ff8_externals.is_card_game = (uint32_t*)get_absolute_value(ff8_externals.sub_47CCB0, 0xB7991);
+	ff8_externals.is_card_game = (uint32_t*)get_absolute_value(ff8_externals.sub_47CCB0, *(uint32_t *)(ff8_externals.sub_47CCB0 + 0xF2) + 0xF7);
 
 	ff8_externals.loc_47D490 = ff8_externals.sub_47CCB0 + 0xDA + 0x4 + *((int32_t *)(ff8_externals.sub_47CCB0 + 0xDA));
 	ff8_externals.sub_500870 = get_relative_call(ff8_externals.loc_47D490, 0x85);
@@ -482,6 +484,7 @@ void ff8_find_externals()
 	ff8_externals.volume_update = get_relative_call(ff8_externals.field_main_loop, 0x28C);
 	ff8_externals.volume_music_update = get_relative_call(ff8_externals.volume_update, 0x6);
 
+	ff8_externals.outputdebugstringa = get_absolute_value(ff8_externals.sm_battle_sound, 0x1B);
 	ff8_externals.sdmusicplay = get_relative_call(ff8_externals.sm_battle_sound, 0x164);
 	ff8_externals.sd_music_play = (uint32_t(*)(uint32_t, char*, uint32_t))get_relative_call(ff8_externals.sdmusicplay, 0x17);
 	ff8_externals.current_music_ids = (uint32_t*)get_absolute_value(uint32_t(ff8_externals.sd_music_play), 0x1AA);
@@ -844,29 +847,6 @@ void ff8_find_externals()
 
 	common_externals.current_triangle_id = 0x0;
 	common_externals.field_game_moment = (WORD*)(ff8_externals.field_vars_stack_1CFE9B8 + 0x100); //0x1CFEAB8
-
-	// Required by Steam edition
-	switch (version)
-	{
-	case VERSION_FF8_12_US_NV:
-		ff8_externals.requiredDisk = 0xB8EE90;
-		break;
-	case VERSION_FF8_12_FR_NV:
-		ff8_externals.requiredDisk = 0xB8EDB8;
-		break;
-	case VERSION_FF8_12_DE_NV:
-		ff8_externals.requiredDisk = 0xB8EDC0;
-		break;
-	case VERSION_FF8_12_SP_NV:
-		ff8_externals.requiredDisk = 0xB8EDC0;
-		break;
-	case VERSION_FF8_12_IT_NV:
-		ff8_externals.requiredDisk = 0xB8EDB8;
-		break;
-	case VERSION_FF8_12_JP:
-		ff8_externals.requiredDisk = 0xD92BB0;
-		break;
-	}
 }
 
 void ff8_data()

src/ff8_opengl.cpp

@@ -994,16 +994,33 @@ void ff8_init_hooks(struct game_obj *_game_object)
 	replace_function(common_externals.open_file, ff8_open_file);
 	replace_call(uint32_t(ff8_externals.fs_archive_search_filename) + 0x10, ff8_fs_archive_search_filename2);
 	// Search file in temp.fs archive (field)
-	replace_call(ff8_externals.moriya_filesytem_open + 0x776, ff8_fs_archive_search_filename_sub_archive);
+	replace_call(ff8_externals.moriya_filesystem_open + 0x776, ff8_fs_archive_search_filename_sub_archive);
 	// Search file in FS archive
-	replace_call(ff8_externals.moriya_filesytem_open + 0x83C, ff8_fs_archive_search_filename_sub_archive);
+	replace_call(ff8_externals.moriya_filesystem_open + 0x83C, ff8_fs_archive_search_filename_sub_archive);
 	replace_function(ff8_externals._open, ff8_open);
 	replace_function(ff8_externals.fopen, ff8_fopen);
-	replace_call(ff8_externals.moriya_filesytem_close + 0x1F, ff8_fs_archive_free_file_container_sub_archive);
+	replace_call(ff8_externals.moriya_filesystem_close + 0x1F, ff8_fs_archive_free_file_container_sub_archive);
 
 	ff8_read_file = (uint32_t(*)(uint32_t, void *, struct ff8_file *))common_externals.read_file;
 	ff8_close_file = (void (*)(struct ff8_file *))common_externals.close_file;
 
+	// #####################
+	// Adding LZ4 support to FS archives
+	// #####################
+
+	// Insert a call to ff8_fs_archive_patch_compression to pass the compression type
+	// 83 BD D4 FD FF FF 01|0F 84(addrCompre)|E9(addrUnkComp)
+	// 51|E8 (addrPatch)|83 C4 04|E9(addrCompre) 90 90 90 90
+	uint32_t read_or_uncompress_fs_data_jump_to_uncompress = *(uint32_t *)(ff8_externals.read_or_uncompress_fs_data + 0x54 + 2);
+	memcpy_code(ff8_externals.read_or_uncompress_fs_data + 0x4D, "\x51\xE8\x00\x00\x00\x00\x83\xC4\x04\xE9\x00\x00\x00\x00\x90\x90\x90\x90", 18);
+	replace_call(ff8_externals.read_or_uncompress_fs_data + 0x4D + 1, ff8_fs_archive_patch_compression);
+	patch_code_dword(ff8_externals.read_or_uncompress_fs_data + 0x4D + 10, read_or_uncompress_fs_data_jump_to_uncompress - 1);
+	// Obtain the compressed and the uncompressed sizes
+	replace_call(ff8_externals.read_or_uncompress_fs_data + 0x153, ff8_fs_archive_malloc_source_data);
+	replace_call(ff8_externals.read_or_uncompress_fs_data + 0x188, ff8_fs_archive_malloc_target_data);
+	// Replace the LZS algorithm by LZ4 if compression type is 2
+	replace_call(ff8_externals.read_or_uncompress_fs_data + 0x1E6, ff8_fs_archive_uncompress_data);
+
 	memset_code(ff8_externals.movie_hack1, 0x90, 14);
 	memset_code(ff8_externals.movie_hack2, 0x90, 8);