GPUDevice: Use CompressHelpers

And compress the pipeline cache. Saves a fair bit of disk space.

src/util/d3d12_device.cpp

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
-// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
+// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "d3d12_device.h"
 #include "d3d12_builders.h"
@@ -283,10 +283,8 @@ void D3D12Device::DestroyDevice()
   m_dxgi_factory.Reset();
 }
 
-bool D3D12Device::ReadPipelineCache(const std::string& filename)
+bool D3D12Device::ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data)
 {
-  std::optional<DynamicHeapArray<u8>> data = FileSystem::ReadBinaryFile(filename.c_str());
-
   HRESULT hr =
     m_device->CreatePipelineLibrary(data.has_value() ? data->data() : nullptr, data.has_value() ? data->size() : 0,
                                     IID_PPV_ARGS(m_pipeline_library.ReleaseAndGetAddressOf()));
@@ -306,11 +304,7 @@ bool D3D12Device::ReadPipelineCache(const std::string& filename)
 
     hr = m_device->CreatePipelineLibrary(nullptr, 0, IID_PPV_ARGS(m_pipeline_library.ReleaseAndGetAddressOf()));
     if (SUCCEEDED(hr))
-    {
-      // Delete cache file, it's no longer relevant.
-      INFO_LOG("Deleting pipeline cache file {}", filename);
-      FileSystem::DeleteFile(filename.c_str());
-    }
+      return true;
   }
 
   if (FAILED(hr))
@@ -332,7 +326,7 @@ bool D3D12Device::GetPipelineCacheData(DynamicHeapArray<u8>* data)
   if (size == 0)
   {
     WARNING_LOG("Empty serialized pipeline state returned.");
-    return false;
+    return true;
   }
 
   data->resize(size);

src/util/d3d12_device.h

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
-// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
+// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #pragma once
 
@@ -190,7 +190,7 @@ class D3D12Device final : public GPUDevice
                     Error* error) override;
   void DestroyDevice() override;
 
-  bool ReadPipelineCache(const std::string& filename) override;
+  bool ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data) override;
   bool GetPipelineCacheData(DynamicHeapArray<u8>* data) override;
 
 private:

src/util/gpu_device.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "gpu_device.h"
+#include "compress_helpers.h"
 #include "core/host.h"     // TODO: Remove, needed for getting fullscreen mode.
 #include "core/settings.h" // TODO: Remove, needed for dump directory.
 #include "gpu_framebuffer_manager.h"
@@ -14,6 +15,7 @@
 #include "common/log.h"
 #include "common/path.h"
 #include "common/scoped_guard.h"
+#include "common/sha1_digest.h"
 #include "common/string_util.h"
 #include "common/timer.h"
 
@@ -43,6 +45,8 @@ Log_SetChannel(GPUDevice);
 std::unique_ptr<GPUDevice> g_gpu_device;
 
 static std::string s_pipeline_cache_path;
+static size_t s_pipeline_cache_size;
+static std::array<u8, SHA1Digest::DIGEST_SIZE> s_pipeline_cache_hash;
 size_t GPUDevice::s_total_vram_usage = 0;
 GPUDevice::Statistics GPUDevice::s_stats = {};
 
@@ -427,7 +431,7 @@ void GPUDevice::OpenShaderCache(std::string_view base_path, u32 version)
   {
     const std::string basename = GetShaderCacheBaseName("pipelines");
     std::string filename = Path::Combine(base_path, TinyString::from_format("{}.bin", basename));
-    if (ReadPipelineCache(filename))
+    if (OpenPipelineCache(filename))
       s_pipeline_cache_path = std::move(filename);
     else
       WARNING_LOG("Failed to read pipeline cache.");
@@ -444,12 +448,17 @@ void GPUDevice::CloseShaderCache()
     if (GetPipelineCacheData(&data))
     {
       // Save disk writes if it hasn't changed, think of the poor SSDs.
-      FILESYSTEM_STAT_DATA sd;
-      if (!FileSystem::StatFile(s_pipeline_cache_path.c_str(), &sd) || sd.Size != static_cast<s64>(data.size()))
+      if (s_pipeline_cache_size != static_cast<s64>(data.size()) ||
+          s_pipeline_cache_hash != SHA1Digest::GetDigest(data.cspan()))
       {
-        INFO_LOG("Writing {} bytes to '{}'", data.size(), Path::GetFileName(s_pipeline_cache_path));
-        if (!FileSystem::WriteBinaryFile(s_pipeline_cache_path.c_str(), data.data(), data.size()))
-          ERROR_LOG("Failed to write pipeline cache to '{}'", Path::GetFileName(s_pipeline_cache_path));
+        Error error;
+        INFO_LOG("Compressing and writing {} bytes to '{}'", data.size(), Path::GetFileName(s_pipeline_cache_path));
+        if (!CompressHelpers::CompressToFile(CompressHelpers::CompressType::Zstandard, s_pipeline_cache_path.c_str(),
+                                             data.cspan(), -1, true, &error))
+        {
+          ERROR_LOG("Failed to write pipeline cache to '{}': {}", Path::GetFileName(s_pipeline_cache_path),
+                    error.GetDescription());
+        }
       }
       else
       {
@@ -505,7 +514,43 @@ std::string GPUDevice::GetShaderCacheBaseName(std::string_view type) const
   return ret;
 }
 
-bool GPUDevice::ReadPipelineCache(const std::string& filename)
+bool GPUDevice::OpenPipelineCache(const std::string& filename)
+{
+  if (FileSystem::GetPathFileSize(filename.c_str()) <= 0)
+    return false;
+
+  Error error;
+  CompressHelpers::OptionalByteBuffer data =
+    CompressHelpers::DecompressFile(CompressHelpers::CompressType::Zstandard, filename.c_str(), std::nullopt, &error);
+  if (!data.has_value())
+  {
+    ERROR_LOG("Failed to load pipeline cache from '{}': {}", Path::GetFileName(filename), error.GetDescription());
+    data.reset();
+  }
+
+  if (data.has_value())
+  {
+    s_pipeline_cache_size = data->size();
+    s_pipeline_cache_hash = SHA1Digest::GetDigest(data->cspan());
+  }
+  else
+  {
+    s_pipeline_cache_size = 0;
+    s_pipeline_cache_hash = {};
+  }
+
+  if (!ReadPipelineCache(std::move(data)))
+  {
+    s_pipeline_cache_size = 0;
+    s_pipeline_cache_hash = {};
+    return false;
+  }
+
+  INFO_LOG("Pipeline cache hash: {}", SHA1Digest::DigestToString(s_pipeline_cache_hash));
+  return true;
+}
+
+bool GPUDevice::ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data)
 {
   return false;
 }
@@ -744,25 +789,27 @@ std::unique_ptr<GPUShader> GPUDevice::CreateShader(GPUShaderStage stage, GPUShad
   }
 
   const GPUShaderCache::CacheIndexKey key = m_shader_cache.GetCacheKey(stage, language, source, entry_point);
-  DynamicHeapArray<u8> binary;
-  if (m_shader_cache.Lookup(key, &binary))
+  std::optional<GPUShaderCache::ShaderBinary> binary = m_shader_cache.Lookup(key);
+  if (binary.has_value())
   {
-    shader = CreateShaderFromBinary(stage, binary, error);
+    shader = CreateShaderFromBinary(stage, binary->cspan(), error);
     if (shader)
       return shader;
 
     ERROR_LOG("Failed to create shader from binary (driver changed?). Clearing cache.");
     m_shader_cache.Clear();
+    binary.reset();
   }
 
-  shader = CreateShaderFromSource(stage, language, source, entry_point, &binary, error);
+  GPUShaderCache::ShaderBinary new_binary;
+  shader = CreateShaderFromSource(stage, language, source, entry_point, &new_binary, error);
   if (!shader)
     return shader;
 
   // Don't insert empty shaders into the cache...
-  if (!binary.empty())
+  if (!new_binary.empty())
   {
-    if (!m_shader_cache.Insert(key, binary.data(), static_cast<u32>(binary.size())))
+    if (!m_shader_cache.Insert(key, new_binary.data(), static_cast<u32>(new_binary.size())))
       m_shader_cache.Close();
   }
 

src/util/gpu_device.h

@@ -533,6 +533,7 @@ class GPUDevice
   static constexpr u32 MAX_RENDER_TARGETS = 4;
   static constexpr u32 MAX_IMAGE_RENDER_TARGETS = 2;
   static constexpr u32 DEFAULT_CLEAR_COLOR = 0xFF000000u;
+  static constexpr u32 PIPELINE_CACHE_HASH_SIZE = 20;
   static_assert(sizeof(GPUPipeline::GraphicsConfig::color_formats) == sizeof(GPUTexture::Format) * MAX_RENDER_TARGETS);
 
   GPUDevice();
@@ -741,7 +742,8 @@ class GPUDevice
   virtual void DestroyDevice() = 0;
 
   std::string GetShaderCacheBaseName(std::string_view type) const;
-  virtual bool ReadPipelineCache(const std::string& filename);
+  virtual bool OpenPipelineCache(const std::string& filename);
+  virtual bool ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data);
   virtual bool GetPipelineCacheData(DynamicHeapArray<u8>* data);
 
   virtual std::unique_ptr<GPUShader> CreateShaderFromBinary(GPUShaderStage stage, std::span<const u8> data,

src/util/gpu_shader_cache.cpp

@@ -4,6 +4,7 @@
 #include "gpu_shader_cache.h"
 #include "gpu_device.h"
 
+#include "common/error.h"
 #include "common/file_system.h"
 #include "common/heap_array.h"
 #include "common/log.h"
@@ -12,8 +13,7 @@
 
 #include "fmt/format.h"
 
-#include "zstd.h"
-#include "zstd_errors.h"
+#include "compress_helpers.h"
 
 Log_SetChannel(GPUShaderCache);
 
@@ -251,42 +251,43 @@ GPUShaderCache::CacheIndexKey GPUShaderCache::GetCacheKey(GPUShaderStage stage,
   return key;
 }
 
-bool GPUShaderCache::Lookup(const CacheIndexKey& key, ShaderBinary* binary)
+std::optional<GPUShaderCache::ShaderBinary> GPUShaderCache::Lookup(const CacheIndexKey& key)
 {
-  auto iter = m_index.find(key);
-  if (iter == m_index.end())
-    return false;
-
-  binary->resize(iter->second.uncompressed_size);
-
-  DynamicHeapArray<u8> compressed_data(iter->second.compressed_size);
+  std::optional<ShaderBinary> ret;
 
-  if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
-      std::fread(compressed_data.data(), iter->second.compressed_size, 1, m_blob_file) != 1) [[unlikely]]
+  auto iter = m_index.find(key);
+  if (iter != m_index.end())
   {
-    ERROR_LOG("Read {} byte {} shader from file failed", iter->second.compressed_size,
-              GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)));
-    return false;
-  }
+    DynamicHeapArray<u8> compressed_data(iter->second.compressed_size);
 
-  const size_t decompress_result =
-    ZSTD_decompress(binary->data(), binary->size(), compressed_data.data(), compressed_data.size());
-  if (ZSTD_isError(decompress_result)) [[unlikely]]
-  {
-    ERROR_LOG("Failed to decompress shader: {}", ZSTD_getErrorName(decompress_result));
-    return false;
+    if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
+        std::fread(compressed_data.data(), iter->second.compressed_size, 1, m_blob_file) != 1) [[unlikely]]
+    {
+      ERROR_LOG("Read {} byte {} shader from file failed", iter->second.compressed_size,
+                GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)));
+    }
+    else
+    {
+      Error error;
+      ret = CompressHelpers::DecompressBuffer(CompressHelpers::CompressType::Zstandard,
+                                              CompressHelpers::OptionalByteBuffer(std::move(compressed_data)),
+                                              iter->second.uncompressed_size, &error);
+      if (!ret.has_value()) [[unlikely]]
+        ERROR_LOG("Failed to decompress shader: {}", error.GetDescription());
+    }
   }
 
-  return true;
+  return ret;
 }
 
 bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data_size)
 {
-  DynamicHeapArray<u8> compress_buffer(ZSTD_compressBound(data_size));
-  const size_t compress_result = ZSTD_compress(compress_buffer.data(), compress_buffer.size(), data, data_size, 0);
-  if (ZSTD_isError(compress_result)) [[unlikely]]
+  Error error;
+  CompressHelpers::OptionalByteBuffer compress_buffer =
+    CompressHelpers::CompressToBuffer(CompressHelpers::CompressType::Zstandard, data, data_size, -1, &error);
+  if (!compress_buffer.has_value()) [[unlikely]]
   {
-    ERROR_LOG("Failed to compress shader: {}", ZSTD_getErrorName(compress_result));
+    ERROR_LOG("Failed to compress shader: {}", error.GetDescription());
     return false;
   }
 
@@ -295,7 +296,7 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data
 
   CacheIndexData idata;
   idata.file_offset = static_cast<u32>(std::ftell(m_blob_file));
-  idata.compressed_size = static_cast<u32>(compress_result);
+  idata.compressed_size = static_cast<u32>(compress_buffer->size());
   idata.uncompressed_size = data_size;
 
   CacheIndexEntry entry = {};
@@ -310,16 +311,17 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data
   entry.compressed_size = idata.compressed_size;
   entry.uncompressed_size = idata.uncompressed_size;
 
-  if (std::fwrite(compress_buffer.data(), compress_result, 1, m_blob_file) != 1 || std::fflush(m_blob_file) != 0 ||
-      std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 || std::fflush(m_index_file) != 0) [[unlikely]]
+  if (std::fwrite(compress_buffer->data(), compress_buffer->size(), 1, m_blob_file) != 1 ||
+      std::fflush(m_blob_file) != 0 || std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+      std::fflush(m_index_file) != 0) [[unlikely]]
   {
     ERROR_LOG("Failed to write {} byte {} shader blob to file", data_size,
               GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)));
     return false;
   }
 
   DEV_LOG("Cached compressed {} shader: {} -> {} bytes",
-          GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)), data_size, compress_result);
+          GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)), data_size, compress_buffer->size());
   m_index.emplace(key, idata);
   return true;
 }

src/util/gpu_shader_cache.h

@@ -7,6 +7,7 @@
 #include "common/heap_array.h"
 #include "common/types.h"
 
+#include <optional>
 #include <string>
 #include <string_view>
 #include <unordered_map>
@@ -56,7 +57,7 @@ class GPUShaderCache
   static CacheIndexKey GetCacheKey(GPUShaderStage stage, GPUShaderLanguage language, std::string_view shader_code,
                                    std::string_view entry_point);
 
-  bool Lookup(const CacheIndexKey& key, ShaderBinary* binary);
+  std::optional<ShaderBinary> Lookup(const CacheIndexKey& key);
   bool Insert(const CacheIndexKey& key, const void* data, u32 data_size);
   void Clear();
 

src/util/opengl_device.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "opengl_device.h"

src/util/opengl_device.h

@@ -137,7 +137,7 @@ class OpenGLDevice final : public GPUDevice
                     Error* error) override;
   void DestroyDevice() override;
 
-  bool ReadPipelineCache(const std::string& filename) override;
+  bool OpenPipelineCache(const std::string& filename) override;
   bool GetPipelineCacheData(DynamicHeapArray<u8>* data) override;
 
 private: