diff --git a/build/00-common.linq b/build/00-common.linq
index a7afe64..c8edc50 100644
--- a/build/00-common.linq
+++ b/build/00-common.linq
@@ -21,7 +21,7 @@ static void DotNetRun(string args) => Run("dotnet", args, Encoding.GetEncoding("
static void Run(string exe, string args, Encoding encoding) => Util.Cmd(exe, args, encoding);
static ProjectVersion[] Projects = new[]
{
- new ProjectVersion("Sdcb.PaddleInference", "2.4.0"),
+ new ProjectVersion("Sdcb.PaddleInference", "2.4.0-rc.2"),
new ProjectVersion("Sdcb.PaddleOCR", "2.6.0"),
new ProjectVersion("Sdcb.PaddleOCR.Models.Online", "2.5.0"),
new ProjectVersion("Sdcb.PaddleOCR.Models.LocalV3", "2.5.0"),
diff --git a/src/Sdcb.PaddleInference/CompilerServices.cs b/src/Sdcb.PaddleInference/CompilerServices.cs
new file mode 100644
index 0000000..96e5972
--- /dev/null
+++ b/src/Sdcb.PaddleInference/CompilerServices.cs
@@ -0,0 +1,14 @@
+using System.ComponentModel;
+
+namespace System.Runtime.CompilerServices
+{
+ ///
+ /// Reserved to be used by the compiler for tracking metadata.
+ /// This class should not be used by developers in source code.
+ /// This dummy class is required to compile records when targeting .NET Standard
+ ///
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public static class IsExternalInit
+ {
+ }
+}
\ No newline at end of file
diff --git a/src/Sdcb.PaddleInference/PaddleConfig.cs b/src/Sdcb.PaddleInference/PaddleConfig.cs
index 0578b5c..fd3597a 100644
--- a/src/Sdcb.PaddleInference/PaddleConfig.cs
+++ b/src/Sdcb.PaddleInference/PaddleConfig.cs
@@ -1,5 +1,7 @@
using Sdcb.PaddleInference.Native;
+using Sdcb.PaddleInference.TensorRt;
using System;
+using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
@@ -9,7 +11,7 @@
namespace Sdcb.PaddleInference
{
- public class PaddleConfig : IDisposable
+ public sealed class PaddleConfig : IDisposable
{
private IntPtr _ptr;
@@ -182,9 +184,12 @@ private static void SearchPathLoad()
}
#endif
+ /// Get version info.
public static string Version => PaddleNative.PD_GetVersion().UTF8PtrToString()!;
+ public IntPtr UnsafeGetHandle() => _ptr;
+ /// A boolean state telling whether logs in Paddle inference are muted.
public bool GLogEnabled
{
get => PaddleNative.PD_ConfigGlogInfoDisabled(_ptr) == 0;
@@ -201,13 +206,17 @@ public bool GLogEnabled
}
}
+ /// A boolean state telling whether the memory optimization is activated.
public bool MemoryOptimized
{
get => PaddleNative.PD_ConfigMemoryOptimEnabled(_ptr) != 0;
set => PaddleNative.PD_ConfigEnableMemoryOptim(_ptr, (sbyte)(value ? 1 : 0));
}
+ /// A boolean state telling whether the model is set from the CPU memory.
public bool IsMemoryModel => PaddleNative.PD_ConfigModelFromMemory(_ptr) != 0;
+
+ /// A boolean state telling whether to use the MKLDNN.
public bool MkldnnEnabled
{
get => PaddleNative.PD_ConfigMkldnnEnabled(_ptr) != 0;
@@ -225,6 +234,8 @@ public bool MkldnnEnabled
}
private int _MkldnnCacheCapacity = 0;
+
+ /// Set the cache capacity of different input shapes for MKLDNN. Default value 0 means not caching any shape.
public int MkldnnCacheCapacity
{
get => _MkldnnCacheCapacity;
@@ -235,6 +246,7 @@ public int MkldnnCacheCapacity
}
}
+ /// Turn on profiling report. If not turned on, no profiling report will be generated.
public bool ProfileEnabled
{
get => PaddleNative.PD_ConfigProfileEnabled(_ptr) != 0;
@@ -257,6 +269,7 @@ public bool ProfileEnabled
// set => PdInvoke.PD_ConfigSetModelDir(_ptr, value);
//}
+ /// A boolean state telling whether the GPU is turned on.
public bool UseGpu
{
get => PaddleNative.PD_ConfigUseGpu(_ptr) != 0;
@@ -273,12 +286,66 @@ public bool UseGpu
}
}
+ /// Get the GPU device id.
public int GpuDeviceId => PaddleNative.PD_ConfigGpuDeviceId(_ptr);
+ /// Get the initial size in MB of the GPU memory pool.
public int InitialGpuMemorySizeMB => PaddleNative.PD_ConfigMemoryPoolInitSizeMb(_ptr);
+ /// Get the proportion of the initial memory pool size compared to the device.
public float FractionOfGpuMemoryForPool => PaddleNative.PD_ConfigFractionOfGpuMemoryForPool(_ptr);
+ /// The memory size(in byte) used for TensorRT workspace.
+ /// The maximum batch size of this prediction task, better set as small as possible for less performance loss.
+ /// Paddle-TRT is running under subgraph, Paddle-TRT will only been enabled when subgraph node count > min_subgraph_size to avoid performance loss
+ /// The precision used in TensorRT.
+ /// Serialize optimization information to disk for reusing.
+ /// Use TRT int8 calibration(post training quantization)
+ public void EnableTensorRtEngine(long workspaceSize = 1 << 20, int maxBatchSize = 1, int minSubgraphSize = 20, int precision = 0, bool useStatic = true, bool useCalibMode = false)
+ => PaddleNative.PD_ConfigEnableTensorRtEngine(_ptr, workspaceSize, maxBatchSize, minSubgraphSize, precision, (sbyte)(useStatic ? 1 : 0), (sbyte)(useCalibMode ? 1 : 0));
+
+ /// A boolean state telling whether the TensorRT engine is used.
+ public bool TensorRtEngineEnabled => PaddleNative.PD_ConfigTensorRtEngineEnabled(_ptr) != 0;
+
+ /// A boolean state telling whether the trt dynamic_shape is used.
+ public bool TensorRtDynamicShapeEnabled => PaddleNative.PD_ConfigTensorRtDynamicShapeEnabled(_ptr) != 0;
+
+ /// A boolean state telling whether to use the TensorRT DLA.
+ public bool TensorRtDlaEnabled => PaddleNative.PD_ConfigTensorRtDlaEnabled(_ptr) != 0;
+
+ /// Set the path of optimization cache directory.
+ public unsafe void SetOptimCacheDir(string path)
+ {
+ fixed (byte* cacheDirPtr = PaddleEncoding.GetBytes(path))
+ {
+ PaddleNative.PD_ConfigSetOptimCacheDir(_ptr, (IntPtr)cacheDirPtr);
+ }
+ }
+
+ /// Set min, max, opt shape for TensorRT Dynamic shape mode.
+ public unsafe void SetTrtDynamicShapeInfo(Dictionary shapeInfo)
+ {
+ using PtrFromStringArray shapeNames = new(shapeInfo.Keys.ToArray());
+
+ long[] shape = new long[shapeInfo.Count];
+ for (int i = 0; i < shape.Length; ++i)
+ {
+ shape[i] = 4;
+ }
+ using PtrFromIntArray minShapePtr = new(shapeInfo.Select(x => x.Value.Min.ToArray()).ToArray());
+ using PtrFromIntArray maxShapePtr = new(shapeInfo.Select(x => x.Value.Max.ToArray()).ToArray());
+ using PtrFromIntArray optShapePtr = new(shapeInfo.Select(x => x.Value.Opt.ToArray()).ToArray());
+
+ fixed (long* shapePtr = shape)
+ {
+ PaddleNative.PD_ConfigSetTrtDynamicShapeInfo(_ptr, 10, shapeNames.Ptr, (IntPtr)shapePtr,
+ minShapePtr.Ptr,
+ maxShapePtr.Ptr,
+ optShapePtr.Ptr, 0);
+ }
+ }
+
+ /// Get information of config.
public unsafe string? Summary
{
get
@@ -299,6 +366,7 @@ public unsafe string? Summary
}
}
+ /// A boolean state telling whether the thread local CUDA stream is enabled.
public bool EnableGpuMultiStream
{
get => PaddleNative.PD_ConfigThreadLocalStreamEnabled(_ptr) != 0;
@@ -315,13 +383,16 @@ public bool EnableGpuMultiStream
}
}
+ /// A boolean state telling whether the Config is valid.
public bool Valid => PaddleNative.PD_ConfigIsValid(_ptr) != 0;
+ /// Turn on GPU.
public void EnableUseGpu(int initialMemoryMB, int deviceId)
{
PaddleNative.PD_ConfigEnableUseGpu(_ptr, (ulong)initialMemoryMB, deviceId);
}
+ /// Set the combined model with two specific pathes for program and parameters.
public unsafe void SetModel(string programPath, string paramsPath)
{
if (programPath == null) throw new ArgumentNullException(nameof(programPath));
@@ -337,9 +408,13 @@ public unsafe void SetModel(string programPath, string paramsPath)
}
}
+ /// Get the program file path.
public string? ProgramPath => PaddleNative.PD_ConfigGetProgFile(_ptr).ANSIToString();
+
+ /// Get the params file path.
public string? ParamsPath => PaddleNative.PD_ConfigGetParamsFile(_ptr).ANSIToString();
+ /// Specify the memory buffer of program and parameter. Used when model and params are loaded directly from memory.
public unsafe void SetMemoryModel(byte[] programBuffer, byte[] paramsBuffer)
{
fixed (byte* pprogram = programBuffer)
@@ -351,12 +426,14 @@ public unsafe void SetMemoryModel(byte[] programBuffer, byte[] paramsBuffer)
}
}
+ /// An int state telling how many threads are used in the CPU math library.
public int CpuMathThreadCount
{
get => PaddleNative.PD_ConfigGetCpuMathLibraryNumThreads(_ptr);
set => PaddleNative.PD_ConfigSetCpuMathLibraryNumThreads(_ptr, value);
}
+ /// Create a new Predictor
public PaddlePredictor CreatePredictor()
{
try
@@ -369,6 +446,7 @@ public PaddlePredictor CreatePredictor()
}
}
+ /// Delete all passes that has a certain type 'pass'.
public unsafe void DeletePass(string passName)
{
byte[] passNameBytes = PaddleEncoding.GetBytes(passName);
@@ -378,6 +456,7 @@ public unsafe void DeletePass(string passName)
}
}
+ /// Destroy the paddle config
public void Dispose()
{
if (_ptr != IntPtr.Zero)
@@ -406,4 +485,62 @@ public class PaddleConfigDefaults
public int GpuDeviceId { get; set; } = 0;
public bool EnableGpuMultiStream { get; set; } = false;
}
+
+ file class PtrFromStringArray : IDisposable
+ {
+ readonly IntPtr[] internalArray;
+ readonly GCHandle[] handles;
+ readonly GCHandle mainHandle;
+
+ public PtrFromStringArray(string[] data)
+ {
+ handles = new GCHandle[data.Length];
+ internalArray = new IntPtr[data.Length];
+
+ for (int i = 0; i < data.Length; ++i)
+ {
+ byte[] byteArray = Encoding.UTF8.GetBytes(data[i] + '\0');
+ handles[i] = GCHandle.Alloc(byteArray, GCHandleType.Pinned);
+ internalArray[i] = handles[i].AddrOfPinnedObject();
+ }
+
+ mainHandle = GCHandle.Alloc(internalArray, GCHandleType.Pinned);
+ }
+
+ public IntPtr Ptr => mainHandle.AddrOfPinnedObject();
+
+ public void Dispose()
+ {
+ foreach (GCHandle handle in handles) handle.Free();
+ mainHandle.Free();
+ }
+ }
+
+ file class PtrFromIntArray : IDisposable
+ {
+ readonly IntPtr[] internalArray;
+ readonly GCHandle[] handles;
+ readonly GCHandle mainHandle;
+
+ public PtrFromIntArray(int[][] data)
+ {
+ handles = new GCHandle[data.Length];
+ internalArray = new IntPtr[data.Length];
+ mainHandle = GCHandle.Alloc(internalArray, GCHandleType.Pinned);
+
+ for (int i = 0; i < data.Length; ++i)
+ {
+ handles[i] = GCHandle.Alloc(data[i], GCHandleType.Pinned);
+ internalArray[i] = handles[i].AddrOfPinnedObject();
+ }
+ }
+
+ public IntPtr Ptr => mainHandle.AddrOfPinnedObject();
+
+ public void Dispose()
+ {
+ foreach (GCHandle handle in handles) handle.Free();
+ mainHandle.Free();
+ }
+ }
}
diff --git a/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj b/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj
index 15ecc32..b091423 100644
--- a/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj
+++ b/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj
@@ -3,7 +3,7 @@
net6;netstandard2.0;net45
enable
True
- 10
+ 11
Apache-2.0
https://github.com/sdcb/PaddleSharp
https://github.com/sdcb/PaddleSharp.git
diff --git a/src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs b/src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs
new file mode 100644
index 0000000..c7a7654
--- /dev/null
+++ b/src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs
@@ -0,0 +1,4 @@
+namespace Sdcb.PaddleInference.TensorRt
+{
+ public record TrtShapeGroup(TrtShape Min, TrtShape Max, TrtShape Opt);
+}
diff --git a/src/Sdcb.PaddleInference/TensorRt/TrtShape.cs b/src/Sdcb.PaddleInference/TensorRt/TrtShape.cs
new file mode 100644
index 0000000..656fbd3
--- /dev/null
+++ b/src/Sdcb.PaddleInference/TensorRt/TrtShape.cs
@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace Sdcb.PaddleInference.TensorRt
+{
+ public record struct TrtShape(int _0, int _1, int _2, int _3)
+ {
+ public int[] ToArray() => new int[] { _0, _1, _2, _3 };
+ }
+}