Enable TRT in Sdcb.PaddleInference.

build/00-common.linq

@@ -21,7 +21,7 @@ static void DotNetRun(string args) => Run("dotnet", args, Encoding.GetEncoding("
 static void Run(string exe, string args, Encoding encoding) => Util.Cmd(exe, args, encoding);
 static ProjectVersion[] Projects = new[]
 {
-	new ProjectVersion("Sdcb.PaddleInference", "2.4.0"), 
+	new ProjectVersion("Sdcb.PaddleInference", "2.4.0-rc.2"), 
 	new ProjectVersion("Sdcb.PaddleOCR", "2.6.0"), 
 	new ProjectVersion("Sdcb.PaddleOCR.Models.Online", "2.5.0"), 
 	new ProjectVersion("Sdcb.PaddleOCR.Models.LocalV3", "2.5.0"), 

src/Sdcb.PaddleInference/CompilerServices.cs

@@ -0,0 +1,14 @@
+using System.ComponentModel;
+
+namespace System.Runtime.CompilerServices
+{
+    /// <summary>
+    /// Reserved to be used by the compiler for tracking metadata.
+    /// This class should not be used by developers in source code.
+    /// This dummy class is required to compile records when targeting .NET Standard
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public static class IsExternalInit
+    {
+    }
+}

src/Sdcb.PaddleInference/PaddleConfig.cs

@@ -1,5 +1,7 @@
 using Sdcb.PaddleInference.Native;
+using Sdcb.PaddleInference.TensorRt;
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -9,7 +11,7 @@
 
 namespace Sdcb.PaddleInference
 {
-    public class PaddleConfig : IDisposable
+    public sealed class PaddleConfig : IDisposable
     {
         private IntPtr _ptr;
 
@@ -182,9 +184,12 @@ private static void SearchPathLoad()
         }
 #endif
 
+        /// <summary>Get version info.</summary>
         public static string Version => PaddleNative.PD_GetVersion().UTF8PtrToString()!;
 
+        public IntPtr UnsafeGetHandle() => _ptr;
 
+        /// <summary>A boolean state telling whether logs in Paddle inference are muted.</summary>
         public bool GLogEnabled
         {
             get => PaddleNative.PD_ConfigGlogInfoDisabled(_ptr) == 0;
@@ -201,13 +206,17 @@ public bool GLogEnabled
             }
         }
 
+        /// <summary>A boolean state telling whether the memory optimization is activated.</summary>
         public bool MemoryOptimized
         {
             get => PaddleNative.PD_ConfigMemoryOptimEnabled(_ptr) != 0;
             set => PaddleNative.PD_ConfigEnableMemoryOptim(_ptr, (sbyte)(value ? 1 : 0));
         }
 
+        /// <summary>A boolean state telling whether the model is set from the CPU memory.</summary>
         public bool IsMemoryModel => PaddleNative.PD_ConfigModelFromMemory(_ptr) != 0;
+
+        /// <summary>A boolean state telling whether to use the MKLDNN.</summary>
         public bool MkldnnEnabled
         {
             get => PaddleNative.PD_ConfigMkldnnEnabled(_ptr) != 0;
@@ -225,6 +234,8 @@ public bool MkldnnEnabled
         }
 
         private int _MkldnnCacheCapacity = 0;
+
+        /// <summary>Set the cache capacity of different input shapes for MKLDNN. Default value 0 means not caching any shape.</summary>
         public int MkldnnCacheCapacity
         {
             get => _MkldnnCacheCapacity;
@@ -235,6 +246,7 @@ public int MkldnnCacheCapacity
             }
         }
 
+        /// <summary>Turn on profiling report. If not turned on, no profiling report will be generated.</summary>
         public bool ProfileEnabled
         {
             get => PaddleNative.PD_ConfigProfileEnabled(_ptr) != 0;
@@ -257,6 +269,7 @@ public bool ProfileEnabled
         //	set => PdInvoke.PD_ConfigSetModelDir(_ptr, value);
         //}
 
+        /// <summary>A boolean state telling whether the GPU is turned on.</summary>
         public bool UseGpu
         {
             get => PaddleNative.PD_ConfigUseGpu(_ptr) != 0;
@@ -273,12 +286,66 @@ public bool UseGpu
             }
         }
 
+        /// <summary>Get the GPU device id.</summary>
         public int GpuDeviceId => PaddleNative.PD_ConfigGpuDeviceId(_ptr);
 
+        /// <summary>Get the initial size in MB of the GPU memory pool.</summary>
         public int InitialGpuMemorySizeMB => PaddleNative.PD_ConfigMemoryPoolInitSizeMb(_ptr);
 
+        /// <summary>Get the proportion of the initial memory pool size compared to the device.</summary>
         public float FractionOfGpuMemoryForPool => PaddleNative.PD_ConfigFractionOfGpuMemoryForPool(_ptr);
 
+        /// <param name="workspaceSize">The memory size(in byte) used for TensorRT workspace.</param>
+        /// <param name="maxBatchSize">The maximum batch size of this prediction task, better set as small as possible for less performance loss.</param>
+        /// <param name="minSubgraphSize">Paddle-TRT is running under subgraph, Paddle-TRT will only been enabled when subgraph node count > min_subgraph_size to avoid performance loss</param>
+        /// <param name="precision">The precision used in TensorRT.</param>
+        /// <param name="useStatic">Serialize optimization information to disk for reusing.</param>
+        /// <param name="useCalibMode">Use TRT int8 calibration(post training quantization)</param>
+        public void EnableTensorRtEngine(long workspaceSize = 1 << 20, int maxBatchSize = 1, int minSubgraphSize = 20, int precision = 0, bool useStatic = true, bool useCalibMode = false)
+            => PaddleNative.PD_ConfigEnableTensorRtEngine(_ptr, workspaceSize, maxBatchSize, minSubgraphSize, precision, (sbyte)(useStatic ? 1 : 0), (sbyte)(useCalibMode ? 1 : 0));
+
+        /// <summary>A boolean state telling whether the TensorRT engine is used.</summary>
+        public bool TensorRtEngineEnabled => PaddleNative.PD_ConfigTensorRtEngineEnabled(_ptr) != 0;
+
+        /// <summary>A boolean state telling whether the trt dynamic_shape is used.</summary>
+        public bool TensorRtDynamicShapeEnabled => PaddleNative.PD_ConfigTensorRtDynamicShapeEnabled(_ptr) != 0;
+
+        /// <summary>A boolean state telling whether to use the TensorRT DLA.</summary>
+        public bool TensorRtDlaEnabled => PaddleNative.PD_ConfigTensorRtDlaEnabled(_ptr) != 0;
+
+        /// <summary>Set the path of optimization cache directory.</summary>
+        public unsafe void SetOptimCacheDir(string path)
+        {
+            fixed (byte* cacheDirPtr = PaddleEncoding.GetBytes(path))
+            {
+                PaddleNative.PD_ConfigSetOptimCacheDir(_ptr, (IntPtr)cacheDirPtr);
+            }
+        }
+
+        /// <summary>Set min, max, opt shape for TensorRT Dynamic shape mode.</summary>
+        public unsafe void SetTrtDynamicShapeInfo(Dictionary<string, TrtShapeGroup> shapeInfo)
+        {
+            using PtrFromStringArray shapeNames = new(shapeInfo.Keys.ToArray());
+
+            long[] shape = new long[shapeInfo.Count];
+            for (int i = 0; i < shape.Length; ++i)
+            {
+                shape[i] = 4;
+            }
+            using PtrFromIntArray minShapePtr = new(shapeInfo.Select(x => x.Value.Min.ToArray()).ToArray());
+            using PtrFromIntArray maxShapePtr = new(shapeInfo.Select(x => x.Value.Max.ToArray()).ToArray());
+            using PtrFromIntArray optShapePtr = new(shapeInfo.Select(x => x.Value.Opt.ToArray()).ToArray());
+
+            fixed (long* shapePtr = shape)
+            {
+                PaddleNative.PD_ConfigSetTrtDynamicShapeInfo(_ptr, 10, shapeNames.Ptr, (IntPtr)shapePtr,
+                    minShapePtr.Ptr,
+                    maxShapePtr.Ptr,
+                    optShapePtr.Ptr, 0);
+            }
+        }
+
+        /// <summary>Get information of config.</summary>
         public unsafe string? Summary
         {
             get
@@ -299,6 +366,7 @@ public unsafe string? Summary
             }
         }
 
+        /// <summary>A boolean state telling whether the thread local CUDA stream is enabled.</summary>
         public bool EnableGpuMultiStream
         {
             get => PaddleNative.PD_ConfigThreadLocalStreamEnabled(_ptr) != 0;
@@ -315,13 +383,16 @@ public bool EnableGpuMultiStream
             }
         }
 
+        /// <summary>A boolean state telling whether the Config is valid.</summary>
         public bool Valid => PaddleNative.PD_ConfigIsValid(_ptr) != 0;
 
+        /// <summary>Turn on GPU.</summary>
         public void EnableUseGpu(int initialMemoryMB, int deviceId)
         {
             PaddleNative.PD_ConfigEnableUseGpu(_ptr, (ulong)initialMemoryMB, deviceId);
         }
 
+        /// <summary>Set the combined model with two specific pathes for program and parameters.</summary>
         public unsafe void SetModel(string programPath, string paramsPath)
         {
             if (programPath == null) throw new ArgumentNullException(nameof(programPath));
@@ -337,9 +408,13 @@ public unsafe void SetModel(string programPath, string paramsPath)
             }
         }
 
+        /// <summary>Get the program file path.</summary>
         public string? ProgramPath => PaddleNative.PD_ConfigGetProgFile(_ptr).ANSIToString();
+
+        /// <summary>Get the params file path.</summary>
         public string? ParamsPath => PaddleNative.PD_ConfigGetParamsFile(_ptr).ANSIToString();
 
+        /// <summary>Specify the memory buffer of program and parameter. Used when model and params are loaded directly from memory.</summary>
         public unsafe void SetMemoryModel(byte[] programBuffer, byte[] paramsBuffer)
         {
             fixed (byte* pprogram = programBuffer)
@@ -351,12 +426,14 @@ public unsafe void SetMemoryModel(byte[] programBuffer, byte[] paramsBuffer)
             }
         }
 
+        /// <summary>An int state telling how many threads are used in the CPU math library.</summary>
         public int CpuMathThreadCount
         {
             get => PaddleNative.PD_ConfigGetCpuMathLibraryNumThreads(_ptr);
             set => PaddleNative.PD_ConfigSetCpuMathLibraryNumThreads(_ptr, value);
         }
 
+        /// <summary>Create a new Predictor</summary>
         public PaddlePredictor CreatePredictor()
         {
             try
@@ -369,6 +446,7 @@ public PaddlePredictor CreatePredictor()
             }
         }
 
+        /// <summary>Delete all passes that has a certain type 'pass'.</summary>
         public unsafe void DeletePass(string passName)
         {
             byte[] passNameBytes = PaddleEncoding.GetBytes(passName);
@@ -378,6 +456,7 @@ public unsafe void DeletePass(string passName)
             }
         }
 
+        /// <summary>Destroy the paddle config</summary>
         public void Dispose()
         {
             if (_ptr != IntPtr.Zero)
@@ -406,4 +485,62 @@ public class PaddleConfigDefaults
         public int GpuDeviceId { get; set; } = 0;
         public bool EnableGpuMultiStream { get; set; } = false;
     }
+
+    file class PtrFromStringArray : IDisposable
+    {
+        readonly IntPtr[] internalArray;
+        readonly GCHandle[] handles;
+        readonly GCHandle mainHandle;
+
+        public PtrFromStringArray(string[] data)
+        {
+            handles = new GCHandle[data.Length];
+            internalArray = new IntPtr[data.Length];
+
+            for (int i = 0; i < data.Length; ++i)
+            {
+                byte[] byteArray = Encoding.UTF8.GetBytes(data[i] + '\0');
+                handles[i] = GCHandle.Alloc(byteArray, GCHandleType.Pinned);
+                internalArray[i] = handles[i].AddrOfPinnedObject();
+            }
+
+            mainHandle = GCHandle.Alloc(internalArray, GCHandleType.Pinned);
+        }
+
+        public IntPtr Ptr => mainHandle.AddrOfPinnedObject();
+
+        public void Dispose()
+        {
+            foreach (GCHandle handle in handles) handle.Free();
+            mainHandle.Free();
+        }
+    }
+
+    file class PtrFromIntArray : IDisposable
+    {
+        readonly IntPtr[] internalArray;
+        readonly GCHandle[] handles;
+        readonly GCHandle mainHandle;
+
+        public PtrFromIntArray(int[][] data)
+        {
+            handles = new GCHandle[data.Length];
+            internalArray = new IntPtr[data.Length];
+            mainHandle = GCHandle.Alloc(internalArray, GCHandleType.Pinned);
+
+            for (int i = 0; i < data.Length; ++i)
+            {
+                handles[i] = GCHandle.Alloc(data[i], GCHandleType.Pinned);
+                internalArray[i] = handles[i].AddrOfPinnedObject();
+            }
+        }
+
+        public IntPtr Ptr => mainHandle.AddrOfPinnedObject();
+
+        public void Dispose()
+        {
+            foreach (GCHandle handle in handles) handle.Free();
+            mainHandle.Free();
+        }
+    }
 }

src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj

@@ -3,7 +3,7 @@
     <TargetFrameworks>net6;netstandard2.0;net45</TargetFrameworks>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
-    <LangVersion>10</LangVersion>
+    <LangVersion>11</LangVersion>
     <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
     <PackageProjectUrl>https://github.com/sdcb/PaddleSharp</PackageProjectUrl>
     <RepositoryUrl>https://github.com/sdcb/PaddleSharp.git</RepositoryUrl>

src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs

@@ -0,0 +1,4 @@
+namespace Sdcb.PaddleInference.TensorRt
+{
+    public record TrtShapeGroup(TrtShape Min, TrtShape Max, TrtShape Opt);
+}

src/Sdcb.PaddleInference/TensorRt/TrtShape.cs

@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace Sdcb.PaddleInference.TensorRt
+{
+    public record struct TrtShape(int _0, int _1, int _2, int _3)
+    {
+        public int[] ToArray() => new int[] { _0, _1, _2, _3 };
+    }
+}