Add "Use strong CRC for texture dump" option. (#2820)

* [GLideNHQ] Add checksum64strong(...) function for strong checksum calculation.

* Add "Use strong CRC for texture dump" option.
When this option is on, textures dumped with alternative strong CRC.

Dumps with strong CRC are stored in "strong_crc" subfolder of dump folder.

Hires texture load is changed:
if HD texture with RiceCRC is not found, the plugin tries to load HD texture with alternative strong CRC.

src/Config.cpp

@@ -103,6 +103,7 @@ void Config::resetToDefaults()
 	textureFilter.txCacheCompression = 1;
 	textureFilter.txSaveCache = 1;
 	textureFilter.txDump = 0;
+	textureFilter.txStrongCRC = 0;
 
 	textureFilter.txEnhancedTextureFileStorage = 0;
 	textureFilter.txHiresTextureFileStorage = 0;
@@ -207,6 +208,8 @@ const char* Config::hotkeyIniName(u32 _idx)
 		return "hkForceGammaCorrection";
 	case Config::HotKey::hkInaccurateTexCords:
 		return "hkInaccurateTexCords";
+	case Config::HotKey::hkStrongCRC:
+		return "hkStrongCRC";
 	}
 	return nullptr;
 }
@@ -245,6 +248,8 @@ const char* Config::enabledHotkeyIniName(u32 _idx)
 		return "hkForceGammaCorrectionEnabled";
 	case Config::HotKey::hkInaccurateTexCords:
 		return "hkInaccurateTexCordsEnabled";
+	case Config::HotKey::hkStrongCRC:
+		return "hkStrongCRCEnabled";
 	}
 	return nullptr;
 }

src/Config.h

@@ -177,6 +177,7 @@ struct Config
 		u32 txCacheCompression;			// Zip textures cache
 		u32 txSaveCache;				// Save texture cache to hard disk
 		u32 txDump;                     // Dump textures
+		u32 txStrongCRC;                // Dump textures with alternative (strong) CRC
 
 		u32 txEnhancedTextureFileStorage;	// Use file storage instead of memory cache for enhanced textures.
 		u32 txHiresTextureFileStorage;		// Use file storage instead of memory cache for hires textures.
@@ -239,6 +240,7 @@ struct Config
 		hkOsdRenderingResolution,
 		hkForceGammaCorrection,
 		hkInaccurateTexCords,
+		hkStrongCRC,
 		hkTotal
 	};
 

src/GLideNHQ/TxFilter.cpp

@@ -580,8 +580,18 @@ TxFilter::checksum64(uint8 *src, int width, int height, int size, int rowStride,
 	return 0;
 }
 
+uint64
+TxFilter::checksum64strong(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette)
+{
+	if (_options & (HIRESTEXTURES_MASK | DUMP_TEX))
+		return TxUtil::checksum64strong(src, width, height, size, rowStride, palette);
+
+	return 0;
+}
+
 boolean
-TxFilter::dmptx(uint8 *src, int width, int height, int rowStridePixel, ColorFormat gfmt, N64FormatSize n64FmtSz, Checksum r_crc64)
+TxFilter::dmptx(uint8 *src, int width, int height, int rowStridePixel,
+				ColorFormat gfmt, N64FormatSize n64FmtSz, Checksum r_crc64, boolean isStrongCrc)
 {
 	assert(gfmt != graphics::colorFormat::RGBA);
 	if (!_initialized)
@@ -609,7 +619,7 @@ TxFilter::dmptx(uint8 *src, int width, int height, int rowStridePixel, ColorForm
 		tmpbuf.assign(_dumpPath);
 		tmpbuf.append(wst("/"));
 		tmpbuf.append(_ident);
-		tmpbuf.append(wst("/GLideNHQ"));
+		isStrongCrc ? tmpbuf.append(wst("/GLideNHQ_strong_crc")) : tmpbuf.append(wst("/GLideNHQ"));
 		if (!osal_path_existsW(tmpbuf.c_str()) && osal_mkdirp(tmpbuf.c_str()) != 0)
 			return 0;
 

src/GLideNHQ/TxFilter.h

@@ -78,7 +78,9 @@ class TxFilter
 				   N64FormatSize n64FmtSz,
 				   GHQTexInfo *info);
   uint64 checksum64(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette);
-  boolean dmptx(uint8 *src, int width, int height, int rowStridePixel, ColorFormat gfmt, N64FormatSize n64FmtSz, Checksum r_crc64);
+  uint64 checksum64strong(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette);
+  boolean dmptx(uint8 *src, int width, int height, int rowStridePixel,
+				ColorFormat gfmt, N64FormatSize n64FmtSz, Checksum r_crc64, boolean isStrongCrc);
   boolean reloadhirestex();
   void dumpcache();
 };

src/GLideNHQ/TxFilterExport.cpp

@@ -84,15 +84,33 @@ txfilter_checksum(uint8 *src, int width, int height, int size, int rowStride, ui
   return 0;
 }
 
+TAPI uint64 TAPIENTRY
+txfilter_checksum_strong(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette)
+{
+	if (txFilter)
+		return txFilter->checksum64strong(src, width, height, size, rowStride, palette);
+
+	return 0;
+}
+
 TAPI boolean TAPIENTRY
 txfilter_dmptx(uint8 *src, int width, int height, int rowStridePixel, uint16 gfmt, N64FormatSize n64FmtSz, Checksum r_crc64)
 {
   if (txFilter)
-	return txFilter->dmptx(src, width, height, rowStridePixel, ColorFormat(u32(gfmt)), n64FmtSz, r_crc64);
+	return txFilter->dmptx(src, width, height, rowStridePixel, ColorFormat(u32(gfmt)), n64FmtSz, r_crc64, FALSE);
 
   return 0;
 }
 
+TAPI boolean TAPIENTRY
+txfilter_dmptx_strong(uint8 *src, int width, int height, int rowStridePixel, uint16 gfmt, N64FormatSize n64FmtSz, Checksum r_crc64)
+{
+	if (txFilter)
+		return txFilter->dmptx(src, width, height, rowStridePixel, ColorFormat(u32(gfmt)), n64FmtSz, r_crc64, TRUE);
+
+	return 0;
+}
+
 TAPI boolean TAPIENTRY
 txfilter_reloadhirestex()
 {

src/GLideNHQ/TxFilterExport.h

@@ -92,7 +92,7 @@ typedef unsigned char boolean;
 #define GZ_HIRESTEXCACHE    0x00800000
 #define DUMP_TEXCACHE       0x01000000
 #define DUMP_HIRESTEXCACHE  0x02000000
-#define UNDEFINED_0         0x04000000
+#define DUMP_STRONG_CRC     0x04000000
 #define UNDEFINED_1         0x08000000
 #define FORCE16BPP_HIRESTEX 0x10000000
 #define FORCE16BPP_TEX      0x20000000
@@ -225,9 +225,15 @@ txfilter_hirestex(uint64 g64crc, Checksum r_crc64, uint16 *palette, N64FormatSiz
 TAPI uint64 TAPIENTRY
 txfilter_checksum(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette);
 
+TAPI uint64 TAPIENTRY
+txfilter_checksum_strong(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette);
+
 TAPI boolean TAPIENTRY
 txfilter_dmptx(uint8 *src, int width, int height, int rowStridePixel, uint16 gfmt, N64FormatSize n64FmtSz, Checksum r_crc64);
 
+TAPI boolean TAPIENTRY
+txfilter_dmptx_strong(uint8 *src, int width, int height, int rowStridePixel, uint16 gfmt, N64FormatSize n64FmtSz, Checksum r_crc64);
+
 TAPI boolean TAPIENTRY
 txfilter_reloadhirestex();
 

src/GLideNHQ/TxUtil.cpp

@@ -38,21 +38,22 @@
 #include <malloc.h>
 #endif
 
+#define XXH_INLINE_ALL
+#include "xxHash/xxhash.h"
+
 /*
  * Utilities
  ******************************************************************************/
+static uint32 Uint64ToUint32(uint64_t t)
+{
+	return static_cast<uint32>((t & 0xFFFFFFFF) ^ (t >> 32));
+}
+
 uint32
 TxUtil::checksumTx(uint8 *src, int width, int height, ColorFormat format)
 {
 	int dataSize = sizeofTx(width, height, format);
-
-	/* for now we use adler32 if something else is better
-   * we can simply swtich later
-   */
-	/* return (dataSize ? Adler32(src, dataSize, 1) : 0); */
-
-	/* zlib crc32 */
-	return (dataSize ? crc32(crc32(0L, Z_NULL, 0), src, dataSize) : 0);
+	return Uint64ToUint32(XXH3_64bits(src, dataSize));
 }
 
 int
@@ -77,18 +78,6 @@ TxUtil::sizeofTx(int width, int height, ColorFormat format)
 	return dataSize;
 }
 
-uint32
-TxUtil::checksum(uint8 *src, int width, int height, int size, int rowStride)
-{
-	/* Rice CRC32 for now. We can switch this to Jabo MD5 or
-   * any other custom checksum.
-   * TODO: use *_HIRESTEXTURE option. */
-
-	if (!src) return 0;
-
-	return RiceCRC32(src, width, height, size, rowStride);
-}
-
 uint64
 TxUtil::checksum64(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette)
 {
@@ -126,6 +115,43 @@ TxUtil::checksum64(uint8 *src, int width, int height, int size, int rowStride, u
 	return crc64Ret;
 }
 
+uint64
+TxUtil::checksum64strong(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette)
+{
+	/* XXH3_64bits for strong 32bit texture hash. */
+    /* Returned value is 64bits: hi=palette crc32 low=texture crc32 */
+
+	if (!src)
+		return 0;
+
+	uint64 crc64Ret = 0;
+
+	if (palette) {
+		uint32 crc32 = 0, cimax = 0;
+		switch (size & 0xff) {
+		case 1:
+			if (StrongCRC32_CI8(src, width, height, rowStride, &crc32, &cimax)) {
+				crc64Ret = StrongCRC32(palette, cimax + 1, 1, 2, 512);
+				crc64Ret <<= 32;
+				crc64Ret |= crc32;
+			}
+			break;
+		case 0:
+			if (StrongCRC32_CI4(src, width, height, rowStride, &crc32, &cimax)) {
+				crc64Ret = StrongCRC32(palette, cimax + 1, 1, 2, 32);
+				crc64Ret <<= 32;
+				crc64Ret |= crc32;
+			}
+		}
+	}
+
+	if (!crc64Ret) {
+		crc64Ret = StrongCRC32(src, width, height, size, rowStride);
+	}
+
+	return crc64Ret;
+}
+
 /* Rice CRC32 for hires texture packs */
 /* NOTE: The following is used in Glide64 to calculate the CRC32
  * for Rice hires texture packs.
@@ -484,6 +510,77 @@ TxUtil::RiceCRC32_CI8(const uint8* src, int width, int height, int rowStride,
 	return 1;
 }
 
+uint32
+TxUtil::StrongCRC32(const uint8* src, int width, int height, int size, int rowStride)
+{
+	/* NOTE: bytesPerLine must be equal or larger than 4 */
+	const uint32 bytesPerLine = width << size >> 1;
+
+	u64 crc = UINT64_MAX;
+	std::vector<uint8> buf(static_cast<uint32>(height) * std::max(bytesPerLine, static_cast<uint32>(rowStride)));
+	uint8* pData = buf.data();
+	try {
+		for (int y = 0; y < height; ++y) {
+			if (bytesPerLine < 4) {
+				// bytesPerLine must be >= 4, but if it less than 4, reproduce RiceCRC behavior,
+				// that is read bytes before provided source address.
+				memcpy(pData, src - 4 + bytesPerLine, 4);
+				pData += 4;
+			}
+			else {
+				memcpy(pData, src, bytesPerLine);
+				pData += bytesPerLine;
+			}
+			src += rowStride;
+		}
+		crc = XXH3_64bits(buf.data(), static_cast<size_t>(pData - buf.data()));
+	}
+	catch (...) {
+		DBG_INFO(80, wst("Error: StrongCRC32 exception!\n"));
+	}
+
+	return Uint64ToUint32(crc);
+}
+
+boolean
+TxUtil::StrongCRC32_CI4(const uint8* src, int width, int height, int rowStride,
+					  uint32* crc32, uint32* cimax)
+{
+	/* NOTE: bytes_per_width must be equal or larger than 4 */
+
+	/* 4bit CI */
+	try {
+		uint32 crc32Ret = StrongCRC32(src, width, height, 0, rowStride);
+		uint32 cimaxRet = CalculateMaxCI4b(src, width, height, rowStride);
+		*crc32 = crc32Ret;
+		*cimax = cimaxRet;
+		return 1;
+	} catch(...) {
+		DBG_INFO(80, wst("Error: RiceCRC32 exception!\n"));
+	}
+	return 0;
+}
+
+boolean
+TxUtil::StrongCRC32_CI8(const uint8* src, int width, int height, int rowStride,
+	uint32* crc32, uint32* cimax)
+{
+	/* NOTE: bytes_per_width must be equal or larger than 4 */
+
+	/* 8bit CI */
+	try {
+		uint32 crc32Ret = StrongCRC32(src, width, height, 1, rowStride);
+		uint32 cimaxRet = CalculateMaxCI8b(src, width, height, rowStride);
+		*crc32 = crc32Ret;
+		*cimax = cimaxRet;
+		return 1;
+	}
+	catch (...) {
+		DBG_INFO(80, wst("Error: RiceCRC32 exception!\n"));
+	}
+	return 0;
+}
+
 uint32 TxUtil::getNumberofProcessors()
 {
 	uint32 numcore = std::thread::hardware_concurrency();

src/GLideNHQ/TxUtil.h

@@ -43,14 +43,16 @@ class TxUtil
 						  uint32* crc32, uint32* cimax);
 	static boolean RiceCRC32_CI8(const uint8* src, int width, int height, int rowStride,
 						  uint32* crc32, uint32* cimax);
+	static uint32 StrongCRC32(const uint8* src, int width, int height, int size, int rowStride);
+	static boolean StrongCRC32_CI4(const uint8* src, int width, int height, int rowStride,
+		uint32* crc32, uint32* cimax);
+	static boolean StrongCRC32_CI8(const uint8* src, int width, int height, int rowStride,
+		uint32* crc32, uint32* cimax);
 public:
 	static int sizeofTx(int width, int height, ColorFormat format);
 	static uint32 checksumTx(uint8 *data, int width, int height, ColorFormat format);
-#if 0 /* unused */
-	static uint32 chkAlpha(uint32* src, int width, int height);
-#endif
-	static uint32 checksum(uint8 *src, int width, int height, int size, int rowStride);
 	static uint64 checksum64(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette);
+	static uint64 checksum64strong(uint8 *src, int width, int height, int size, int rowStride, uint8 *palette);
 	static uint32 getNumberofProcessors();
 };
 

src/GLideNUI/ConfigDialog.cpp

@@ -105,6 +105,8 @@ QString ConfigDialog::_hotkeyDescription(quint32 _idx) const
 		return tr("Toggle force gamma correction");
 	case Config::HotKey::hkInaccurateTexCords:
 		return tr("Toggle inaccurate texture coordinates");
+	case Config::HotKey::hkStrongCRC:
+		return tr("Toggle strong CRC for textures dump");
 	}
 	return tr("Unknown hotkey");
 }
@@ -373,6 +375,7 @@ void ConfigDialog::_init(bool reInit, bool blockCustomSettings)
 	ui->texturePackGroupBox->setChecked(config.textureFilter.txHiresEnable != 0);
 	ui->alphaChannelCheckBox->setChecked(config.textureFilter.txHiresFullAlphaChannel != 0);
 	ui->alternativeCRCCheckBox->setChecked(config.textureFilter.txHresAltCRC != 0);
+	ui->strongCRCCheckBox->setChecked(config.textureFilter.txStrongCRC != 0);
 	ui->force16bppCheckBox->setChecked(config.textureFilter.txForce16bpp != 0);
 	ui->compressCacheCheckBox->setChecked(config.textureFilter.txCacheCompression != 0);
 	ui->saveTextureCacheCheckBox->setChecked(config.textureFilter.txSaveCache != 0);
@@ -689,6 +692,7 @@ void ConfigDialog::accept(bool justSave) {
 	config.textureFilter.txHiresEnable = ui->texturePackGroupBox->isChecked() ? 1 : 0;
 	config.textureFilter.txHiresFullAlphaChannel = ui->alphaChannelCheckBox->isChecked() ? 1 : 0;
 	config.textureFilter.txHresAltCRC = ui->alternativeCRCCheckBox->isChecked() ? 1 : 0;
+	config.textureFilter.txStrongCRC = ui->strongCRCCheckBox->isChecked() ? 1 : 0;
 
 	config.textureFilter.txCacheCompression = ui->compressCacheCheckBox->isChecked() ? 1 : 0;
 	config.textureFilter.txForce16bpp = ui->force16bppCheckBox->isChecked() ? 1 : 0;

src/GLideNUI/Settings.cpp

@@ -103,6 +103,7 @@ void _loadSettings(QSettings & settings)
 	config.textureFilter.txHiresEnable = settings.value("txHiresEnable", config.textureFilter.txHiresEnable).toInt();
 	config.textureFilter.txHiresFullAlphaChannel = settings.value("txHiresFullAlphaChannel", config.textureFilter.txHiresFullAlphaChannel).toInt();
 	config.textureFilter.txHresAltCRC = settings.value("txHresAltCRC", config.textureFilter.txHresAltCRC).toInt();
+	config.textureFilter.txStrongCRC = settings.value("txStrongCRC", config.textureFilter.txStrongCRC).toInt();
 	config.textureFilter.txForce16bpp = settings.value("txForce16bpp", config.textureFilter.txForce16bpp).toInt();
 	config.textureFilter.txCacheCompression = settings.value("txCacheCompression", config.textureFilter.txCacheCompression).toInt();
 	config.textureFilter.txSaveCache = settings.value("txSaveCache", config.textureFilter.txSaveCache).toInt();
@@ -252,6 +253,7 @@ void _writeSettingsToFile(const QString & filename)
 	settings.setValue("txHiresEnable", config.textureFilter.txHiresEnable);
 	settings.setValue("txHiresFullAlphaChannel", config.textureFilter.txHiresFullAlphaChannel);
 	settings.setValue("txHresAltCRC", config.textureFilter.txHresAltCRC);
+	settings.setValue("txStrongCRC", config.textureFilter.txStrongCRC);
 	settings.setValue("txForce16bpp", config.textureFilter.txForce16bpp);
 	settings.setValue("txCacheCompression", config.textureFilter.txCacheCompression);
 	settings.setValue("txSaveCache", config.textureFilter.txSaveCache);
@@ -550,6 +552,7 @@ void saveCustomRomSettings(const QString & _strIniFolder, const QString & _strSh
 	WriteCustomSetting(textureFilter, txHiresEnable);
 	WriteCustomSetting(textureFilter, txHiresFullAlphaChannel);
 	WriteCustomSetting(textureFilter, txHresAltCRC);
+	WriteCustomSetting(textureFilter, txStrongCRC);
 	WriteCustomSetting(textureFilter, txForce16bpp);
 	WriteCustomSetting(textureFilter, txCacheCompression);
 	WriteCustomSetting(textureFilter, txSaveCache);