[host] dxgi: HLSL experiement to pre-process HDR content

host/platform/Windows/capture/DXGI/CMakeLists.txt

@@ -16,6 +16,7 @@ target_link_libraries(capture_DXGI
   d3d11
   dxgi
   dwmapi
+  d3dcompiler
 )
 
 target_include_directories(capture_DXGI

host/platform/Windows/capture/DXGI/src/dxgi.c

@@ -40,6 +40,7 @@
 #include <dxgi1_6.h>
 #include <d3d11.h>
 #include <d3dcommon.h>
+#include <d3dcompiler.h>
 #include <versionhelpers.h>
 #include <dwmapi.h>
 
@@ -59,6 +60,12 @@ DownsampleRule;
 
 static Vector downsampleRules = {0};
 
+struct Vertex
+{
+  float position[3];
+  float texCoord[2];
+};
+
 // locals
 static struct DXGIInterface * this = NULL;
 
@@ -659,12 +666,9 @@ static bool dxgi_init(void)
   {
     const DXGI_FORMAT supportedFormats[] =
     {
-      DXGI_FORMAT_R10G10B10A2_UNORM,
       DXGI_FORMAT_B8G8R8A8_UNORM,
       DXGI_FORMAT_R8G8B8A8_UNORM,
-
-      // we do not want this format as it doubles the bandwidth required
-//      DXGI_FORMAT_R16G16B16A16_FLOAT
+      DXGI_FORMAT_R16G16B16A16_FLOAT
     };
 
     // we try this twice in case we still get an error on re-initialization
@@ -742,17 +746,14 @@ static bool dxgi_init(void)
     D3D11_TEXTURE2D_DESC desc;
     ID3D11Texture2D_GetDesc(src, &desc);
 
-    // there seems to be a bug here in DXGI, asking for 10bit when restarting
-    // reports as 16bit, even though it's really 10bit.
-    if (desc.Format == DXGI_FORMAT_R16G16B16A16_FLOAT)
-      desc.Format = DXGI_FORMAT_R10G10B10A2_UNORM;
-
     DEBUG_INFO("Capture Format    : %s", GetDXGIFormatStr(desc.Format));
     this->dxgiFormat = desc.Format;
 
     ID3D11Texture2D_Release(src);
     IDXGIResource_Release(res);
     IDXGIOutputDuplication_ReleaseFrame(this->dup);
+
+    this->hdr = this->dxgiColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020;
   }
 
   this->bpp = 4;
@@ -766,18 +767,14 @@ static bool dxgi_init(void)
       this->format = CAPTURE_FMT_RGBA;
       break;
 
-    case DXGI_FORMAT_R10G10B10A2_UNORM:
-      if (this->dxgiColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020)
+    // we convert to HDR10 to save bandwidth always
+    case DXGI_FORMAT_R16G16B16A16_FLOAT:
+      if (this->hdr)
         this->format = CAPTURE_FMT_RGBA10_HDR;
       else
         this->format = CAPTURE_FMT_RGBA10_SDR;
       break;
 
-    case DXGI_FORMAT_R16G16B16A16_FLOAT:
-      this->format = CAPTURE_FMT_RGBA16F;
-      this->bpp    = 8;
-      break;
-
     default:
       DEBUG_ERROR("Unsupported source format");
       goto fail;
@@ -836,7 +833,129 @@ static bool dxgi_init(void)
   DEBUG_INFO("Damage-aware copy : %s", this->disableDamage  ? "disabled" : "enabled" );
 
   for (int i = 0; i < this->maxTextures; ++i)
+  {
     this->texture[i].texDamageCount = -1;
+    if (!this->hdr)
+      continue;
+
+    D3D11_TEXTURE2D_DESC hdrTexDesc =
+    {
+      .Width              = this->width,
+      .Height             = this->height,
+      .MipLevels          = 1,
+      .ArraySize          = 1,
+      .SampleDesc.Count   = 1,
+      .SampleDesc.Quality = 0,
+      .Usage              = D3D11_USAGE_DEFAULT,
+      .Format             = this->dxgiFormat,
+      .BindFlags          = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE,
+      .CPUAccessFlags     = 0,
+      .MiscFlags          = 0
+    };
+
+    status = ID3D11Device_CreateTexture2D(this->device, &hdrTexDesc, NULL,
+      &this->texture[i].hdrTex);
+
+    if (FAILED(status))
+    {
+      DEBUG_WINERROR("Failed to create HDR texture", status);
+      goto fail;
+    }
+
+    status = ID3D11Device_CreateRenderTargetView(this->device,
+      (ID3D11Resource *)this->texture[i].hdrTex, NULL,
+      &this->texture[i].renderTarget);
+
+    ID3D11Texture2D_Release(this->texture[i].hdrTex);
+
+    if (FAILED(status))
+    {
+      DEBUG_WINERROR("Failed to create HDR target view", status);
+      goto fail;
+    }
+  }
+
+  if (this->hdr)
+  {
+    D3D11_VIEWPORT vp =
+    {
+      .TopLeftX = 0.0f,
+      .TopLeftY = 0.0f,
+      .Width    = this->width,
+      .Height   = this->height,
+      .MinDepth = 0.0f,
+      .MaxDepth = 1.0f,
+    };
+    ID3D11DeviceContext_RSSetViewports(this->deviceContext, 1, &vp);
+
+    struct Vertex vertices[] =
+    {
+      {{-1.0f, -1.0f, 0.0f}, {0.0f, 1.0f}},
+      {{-1.0f,  1.0f, 0.0f}, {0.0f, 0.0f}},
+      {{ 1.0f, -1.0f, 0.0f}, {1.0f, 1.0f}},
+      {{ 1.0f,  1.0f, 0.0f}, {1.0f, 0.0f}}
+    };
+
+    D3D11_BUFFER_DESC bufDesc =
+    {
+      .Usage     = D3D11_USAGE_DEFAULT,
+      .ByteWidth = sizeof(vertices),
+      .BindFlags = D3D11_BIND_VERTEX_BUFFER
+    };
+
+    D3D11_SUBRESOURCE_DATA bufData = { .pSysMem = vertices };
+    status = ID3D11Device_CreateBuffer(
+      this->device, &bufDesc, &bufData, &this->vertexBuffer);
+    if (FAILED(status))
+    {
+      DEBUG_WINERROR("Failed to create the vertex buffer", status);
+      goto fail;
+    }
+
+    static const char * shader =
+      "float4 main(float4 color : COLOR) : SV_TARGET\n"
+      "{\n"
+      "  return color;\n"
+      "}";
+
+    ID3DBlob * bytecode;
+    ID3DBlob * errors;
+    status = D3DCompile(
+      shader,
+      strlen(shader),
+      NULL,
+      NULL,
+      NULL,
+      "main",
+      "ps_5_0",
+      D3DCOMPILE_DEBUG | D3DCOMPILE_SKIP_OPTIMIZATION,
+      0,
+      &bytecode,
+      &errors);
+
+    if (FAILED(status))
+    {
+      DEBUG_ERROR("Failed to compile the pixel shader");
+      DEBUG_ERROR("%s", (const char *)ID3D10Blob_GetBufferPointer(errors));
+      ID3D10Blob_Release(errors);
+      goto fail;
+    }
+
+    status = ID3D11Device_CreatePixelShader(
+      this->device,
+      ID3D10Blob_GetBufferPointer(bytecode),
+      ID3D10Blob_GetBufferSize   (bytecode),
+      NULL,
+      &this->pixelShader);
+
+    ID3D10Blob_Release(bytecode);
+
+    if (FAILED(status))
+    {
+      DEBUG_WINERROR("Failed to create the pixel shader", status);
+      goto fail;
+    }
+  }
 
   for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i)
     this->frameDamage[i].count = -1;
@@ -867,6 +986,24 @@ static bool dxgi_deinit(void)
       this->backend->unmapTexture(this->texture + i);
       this->texture[i].map = NULL;
     }
+
+    if (this->texture[i].renderTarget)
+    {
+      ID3D11RenderTargetView_Release(this->texture[i].renderTarget);
+      this->texture[i].renderTarget = NULL;
+    }
+  }
+
+  if (this->pixelShader)
+  {
+    ID3D11PixelShader_Release(this->pixelShader);
+    this->pixelShader = NULL;
+  }
+
+  if (this->vertexBuffer)
+  {
+    ID3D11Buffer_Release(this->vertexBuffer);
+    this->vertexBuffer = NULL;
   }
 
   if (this->dup)
@@ -1160,10 +1297,59 @@ static CaptureResult dxgi_capture(void)
         computeFrameDamage(tex);
       computeTexDamage(tex);
 
-      if (!this->backend->copyFrame(tex, src))
+      if (this->hdr)
       {
-        ID3D11Texture2D_Release(src);
-        return CAPTURE_RESULT_ERROR;
+        // setup the pixel shader input resource view
+        {
+          D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc =
+          {
+            .Format              = this->dxgiFormat,
+            .ViewDimension       = D3D11_SRV_DIMENSION_TEXTURE2D,
+            .Texture2D.MipLevels = 1
+          };
+
+          ID3D11ShaderResourceView * inputSRV;
+          status = ID3D11Device_CreateShaderResourceView(
+            this->device, (ID3D11Resource *)src, &srvDesc, &inputSRV);
+          if (FAILED(status))
+          {
+            DEBUG_ERROR("Failed to create the source resource view");
+            return CAPTURE_RESULT_ERROR;
+          }
+          ID3D11DeviceContext_PSSetShaderResources(
+            this->deviceContext, 0, 1, &inputSRV);
+          ID3D11ShaderResourceView_Release(inputSRV);
+        }
+
+        ID3D11DeviceContext_OMSetRenderTargets(
+          this->deviceContext, 1, &tex->renderTarget, NULL);
+        ID3D11DeviceContext_PSSetShader(
+          this->deviceContext, this->pixelShader, NULL, 0);
+
+        ID3D11DeviceContext_IASetPrimitiveTopology(this->deviceContext,
+          D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
+
+        UINT stride = sizeof(struct Vertex);
+        UINT offset = 0;
+        ID3D11DeviceContext_IASetVertexBuffers(
+          this->deviceContext, 0, 1, &this->vertexBuffer, &stride, &offset);
+
+        ID3D11DeviceContext_Draw(this->deviceContext, 4, 0);
+//        ID3D11DeviceContext_Flush(this->deviceContext);
+
+        if (!this->backend->copyFrame(tex, tex->hdrTex))
+        {
+          ID3D11Texture2D_Release(src);
+          return CAPTURE_RESULT_ERROR;
+        }
+      }
+      else
+      {
+        if (!this->backend->copyFrame(tex, src))
+        {
+          ID3D11Texture2D_Release(src);
+          return CAPTURE_RESULT_ERROR;
+        }
       }
 
       ID3D11Texture2D_Release(src);

host/platform/Windows/capture/DXGI/src/dxgi_capture.h

@@ -47,6 +47,8 @@ typedef struct Texture
   FrameDamageRect            damageRects[KVMFR_MAX_DAMAGE_RECTS];
   int32_t                    texDamageCount;
   FrameDamageRect            texDamageRects[KVMFR_MAX_DAMAGE_RECTS];
+  ID3D11RenderTargetView   * renderTarget;
+  ID3D11Texture2D          * hdrTex;
 
   void                     * impl;
 }
@@ -86,8 +88,11 @@ struct DXGIInterface
   atomic_int                 texReady;
   bool                       needsRelease;
   DXGI_FORMAT                dxgiFormat;
+  bool                       hdr;
   DXGI_COLOR_SPACE_TYPE      dxgiColorSpace;
   float                      sdrWhiteLevel;
+  ID3D11Buffer             * vertexBuffer;
+  ID3D11PixelShader        * pixelShader;
   struct DXGICopyBackend   * backend;
 
   CaptureGetPointerBuffer    getPointerBufferFn;