diff --git a/build/00-common.linq b/build/00-common.linq index a7afe64..c8edc50 100644 --- a/build/00-common.linq +++ b/build/00-common.linq @@ -21,7 +21,7 @@ static void DotNetRun(string args) => Run("dotnet", args, Encoding.GetEncoding(" static void Run(string exe, string args, Encoding encoding) => Util.Cmd(exe, args, encoding); static ProjectVersion[] Projects = new[] { - new ProjectVersion("Sdcb.PaddleInference", "2.4.0"), + new ProjectVersion("Sdcb.PaddleInference", "2.4.0-rc.2"), new ProjectVersion("Sdcb.PaddleOCR", "2.6.0"), new ProjectVersion("Sdcb.PaddleOCR.Models.Online", "2.5.0"), new ProjectVersion("Sdcb.PaddleOCR.Models.LocalV3", "2.5.0"), diff --git a/src/Sdcb.PaddleInference/CompilerServices.cs b/src/Sdcb.PaddleInference/CompilerServices.cs new file mode 100644 index 0000000..96e5972 --- /dev/null +++ b/src/Sdcb.PaddleInference/CompilerServices.cs @@ -0,0 +1,14 @@ +using System.ComponentModel; + +namespace System.Runtime.CompilerServices +{ + /// + /// Reserved to be used by the compiler for tracking metadata. + /// This class should not be used by developers in source code. + /// This dummy class is required to compile records when targeting .NET Standard + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public static class IsExternalInit + { + } +} \ No newline at end of file diff --git a/src/Sdcb.PaddleInference/PaddleConfig.cs b/src/Sdcb.PaddleInference/PaddleConfig.cs index 0578b5c..fd3597a 100644 --- a/src/Sdcb.PaddleInference/PaddleConfig.cs +++ b/src/Sdcb.PaddleInference/PaddleConfig.cs @@ -1,5 +1,7 @@ using Sdcb.PaddleInference.Native; +using Sdcb.PaddleInference.TensorRt; using System; +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; @@ -9,7 +11,7 @@ namespace Sdcb.PaddleInference { - public class PaddleConfig : IDisposable + public sealed class PaddleConfig : IDisposable { private IntPtr _ptr; @@ -182,9 +184,12 @@ private static void SearchPathLoad() } #endif + /// Get version info. public static string Version => PaddleNative.PD_GetVersion().UTF8PtrToString()!; + public IntPtr UnsafeGetHandle() => _ptr; + /// A boolean state telling whether logs in Paddle inference are muted. public bool GLogEnabled { get => PaddleNative.PD_ConfigGlogInfoDisabled(_ptr) == 0; @@ -201,13 +206,17 @@ public bool GLogEnabled } } + /// A boolean state telling whether the memory optimization is activated. public bool MemoryOptimized { get => PaddleNative.PD_ConfigMemoryOptimEnabled(_ptr) != 0; set => PaddleNative.PD_ConfigEnableMemoryOptim(_ptr, (sbyte)(value ? 1 : 0)); } + /// A boolean state telling whether the model is set from the CPU memory. public bool IsMemoryModel => PaddleNative.PD_ConfigModelFromMemory(_ptr) != 0; + + /// A boolean state telling whether to use the MKLDNN. public bool MkldnnEnabled { get => PaddleNative.PD_ConfigMkldnnEnabled(_ptr) != 0; @@ -225,6 +234,8 @@ public bool MkldnnEnabled } private int _MkldnnCacheCapacity = 0; + + /// Set the cache capacity of different input shapes for MKLDNN. Default value 0 means not caching any shape. public int MkldnnCacheCapacity { get => _MkldnnCacheCapacity; @@ -235,6 +246,7 @@ public int MkldnnCacheCapacity } } + /// Turn on profiling report. If not turned on, no profiling report will be generated. public bool ProfileEnabled { get => PaddleNative.PD_ConfigProfileEnabled(_ptr) != 0; @@ -257,6 +269,7 @@ public bool ProfileEnabled // set => PdInvoke.PD_ConfigSetModelDir(_ptr, value); //} + /// A boolean state telling whether the GPU is turned on. public bool UseGpu { get => PaddleNative.PD_ConfigUseGpu(_ptr) != 0; @@ -273,12 +286,66 @@ public bool UseGpu } } + /// Get the GPU device id. public int GpuDeviceId => PaddleNative.PD_ConfigGpuDeviceId(_ptr); + /// Get the initial size in MB of the GPU memory pool. public int InitialGpuMemorySizeMB => PaddleNative.PD_ConfigMemoryPoolInitSizeMb(_ptr); + /// Get the proportion of the initial memory pool size compared to the device. public float FractionOfGpuMemoryForPool => PaddleNative.PD_ConfigFractionOfGpuMemoryForPool(_ptr); + /// The memory size(in byte) used for TensorRT workspace. + /// The maximum batch size of this prediction task, better set as small as possible for less performance loss. + /// Paddle-TRT is running under subgraph, Paddle-TRT will only been enabled when subgraph node count > min_subgraph_size to avoid performance loss + /// The precision used in TensorRT. + /// Serialize optimization information to disk for reusing. + /// Use TRT int8 calibration(post training quantization) + public void EnableTensorRtEngine(long workspaceSize = 1 << 20, int maxBatchSize = 1, int minSubgraphSize = 20, int precision = 0, bool useStatic = true, bool useCalibMode = false) + => PaddleNative.PD_ConfigEnableTensorRtEngine(_ptr, workspaceSize, maxBatchSize, minSubgraphSize, precision, (sbyte)(useStatic ? 1 : 0), (sbyte)(useCalibMode ? 1 : 0)); + + /// A boolean state telling whether the TensorRT engine is used. + public bool TensorRtEngineEnabled => PaddleNative.PD_ConfigTensorRtEngineEnabled(_ptr) != 0; + + /// A boolean state telling whether the trt dynamic_shape is used. + public bool TensorRtDynamicShapeEnabled => PaddleNative.PD_ConfigTensorRtDynamicShapeEnabled(_ptr) != 0; + + /// A boolean state telling whether to use the TensorRT DLA. + public bool TensorRtDlaEnabled => PaddleNative.PD_ConfigTensorRtDlaEnabled(_ptr) != 0; + + /// Set the path of optimization cache directory. + public unsafe void SetOptimCacheDir(string path) + { + fixed (byte* cacheDirPtr = PaddleEncoding.GetBytes(path)) + { + PaddleNative.PD_ConfigSetOptimCacheDir(_ptr, (IntPtr)cacheDirPtr); + } + } + + /// Set min, max, opt shape for TensorRT Dynamic shape mode. + public unsafe void SetTrtDynamicShapeInfo(Dictionary shapeInfo) + { + using PtrFromStringArray shapeNames = new(shapeInfo.Keys.ToArray()); + + long[] shape = new long[shapeInfo.Count]; + for (int i = 0; i < shape.Length; ++i) + { + shape[i] = 4; + } + using PtrFromIntArray minShapePtr = new(shapeInfo.Select(x => x.Value.Min.ToArray()).ToArray()); + using PtrFromIntArray maxShapePtr = new(shapeInfo.Select(x => x.Value.Max.ToArray()).ToArray()); + using PtrFromIntArray optShapePtr = new(shapeInfo.Select(x => x.Value.Opt.ToArray()).ToArray()); + + fixed (long* shapePtr = shape) + { + PaddleNative.PD_ConfigSetTrtDynamicShapeInfo(_ptr, 10, shapeNames.Ptr, (IntPtr)shapePtr, + minShapePtr.Ptr, + maxShapePtr.Ptr, + optShapePtr.Ptr, 0); + } + } + + /// Get information of config. public unsafe string? Summary { get @@ -299,6 +366,7 @@ public unsafe string? Summary } } + /// A boolean state telling whether the thread local CUDA stream is enabled. public bool EnableGpuMultiStream { get => PaddleNative.PD_ConfigThreadLocalStreamEnabled(_ptr) != 0; @@ -315,13 +383,16 @@ public bool EnableGpuMultiStream } } + /// A boolean state telling whether the Config is valid. public bool Valid => PaddleNative.PD_ConfigIsValid(_ptr) != 0; + /// Turn on GPU. public void EnableUseGpu(int initialMemoryMB, int deviceId) { PaddleNative.PD_ConfigEnableUseGpu(_ptr, (ulong)initialMemoryMB, deviceId); } + /// Set the combined model with two specific pathes for program and parameters. public unsafe void SetModel(string programPath, string paramsPath) { if (programPath == null) throw new ArgumentNullException(nameof(programPath)); @@ -337,9 +408,13 @@ public unsafe void SetModel(string programPath, string paramsPath) } } + /// Get the program file path. public string? ProgramPath => PaddleNative.PD_ConfigGetProgFile(_ptr).ANSIToString(); + + /// Get the params file path. public string? ParamsPath => PaddleNative.PD_ConfigGetParamsFile(_ptr).ANSIToString(); + /// Specify the memory buffer of program and parameter. Used when model and params are loaded directly from memory. public unsafe void SetMemoryModel(byte[] programBuffer, byte[] paramsBuffer) { fixed (byte* pprogram = programBuffer) @@ -351,12 +426,14 @@ public unsafe void SetMemoryModel(byte[] programBuffer, byte[] paramsBuffer) } } + /// An int state telling how many threads are used in the CPU math library. public int CpuMathThreadCount { get => PaddleNative.PD_ConfigGetCpuMathLibraryNumThreads(_ptr); set => PaddleNative.PD_ConfigSetCpuMathLibraryNumThreads(_ptr, value); } + /// Create a new Predictor public PaddlePredictor CreatePredictor() { try @@ -369,6 +446,7 @@ public PaddlePredictor CreatePredictor() } } + /// Delete all passes that has a certain type 'pass'. public unsafe void DeletePass(string passName) { byte[] passNameBytes = PaddleEncoding.GetBytes(passName); @@ -378,6 +456,7 @@ public unsafe void DeletePass(string passName) } } + /// Destroy the paddle config public void Dispose() { if (_ptr != IntPtr.Zero) @@ -406,4 +485,62 @@ public class PaddleConfigDefaults public int GpuDeviceId { get; set; } = 0; public bool EnableGpuMultiStream { get; set; } = false; } + + file class PtrFromStringArray : IDisposable + { + readonly IntPtr[] internalArray; + readonly GCHandle[] handles; + readonly GCHandle mainHandle; + + public PtrFromStringArray(string[] data) + { + handles = new GCHandle[data.Length]; + internalArray = new IntPtr[data.Length]; + + for (int i = 0; i < data.Length; ++i) + { + byte[] byteArray = Encoding.UTF8.GetBytes(data[i] + '\0'); + handles[i] = GCHandle.Alloc(byteArray, GCHandleType.Pinned); + internalArray[i] = handles[i].AddrOfPinnedObject(); + } + + mainHandle = GCHandle.Alloc(internalArray, GCHandleType.Pinned); + } + + public IntPtr Ptr => mainHandle.AddrOfPinnedObject(); + + public void Dispose() + { + foreach (GCHandle handle in handles) handle.Free(); + mainHandle.Free(); + } + } + + file class PtrFromIntArray : IDisposable + { + readonly IntPtr[] internalArray; + readonly GCHandle[] handles; + readonly GCHandle mainHandle; + + public PtrFromIntArray(int[][] data) + { + handles = new GCHandle[data.Length]; + internalArray = new IntPtr[data.Length]; + mainHandle = GCHandle.Alloc(internalArray, GCHandleType.Pinned); + + for (int i = 0; i < data.Length; ++i) + { + handles[i] = GCHandle.Alloc(data[i], GCHandleType.Pinned); + internalArray[i] = handles[i].AddrOfPinnedObject(); + } + } + + public IntPtr Ptr => mainHandle.AddrOfPinnedObject(); + + public void Dispose() + { + foreach (GCHandle handle in handles) handle.Free(); + mainHandle.Free(); + } + } } diff --git a/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj b/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj index 15ecc32..b091423 100644 --- a/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj +++ b/src/Sdcb.PaddleInference/Sdcb.PaddleInference.csproj @@ -3,7 +3,7 @@ net6;netstandard2.0;net45 enable True - 10 + 11 Apache-2.0 https://github.com/sdcb/PaddleSharp https://github.com/sdcb/PaddleSharp.git diff --git a/src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs b/src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs new file mode 100644 index 0000000..c7a7654 --- /dev/null +++ b/src/Sdcb.PaddleInference/TensorRt/TrtDynamicShapeInfo.cs @@ -0,0 +1,4 @@ +namespace Sdcb.PaddleInference.TensorRt +{ + public record TrtShapeGroup(TrtShape Min, TrtShape Max, TrtShape Opt); +} diff --git a/src/Sdcb.PaddleInference/TensorRt/TrtShape.cs b/src/Sdcb.PaddleInference/TensorRt/TrtShape.cs new file mode 100644 index 0000000..656fbd3 --- /dev/null +++ b/src/Sdcb.PaddleInference/TensorRt/TrtShape.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Sdcb.PaddleInference.TensorRt +{ + public record struct TrtShape(int _0, int _1, int _2, int _3) + { + public int[] ToArray() => new int[] { _0, _1, _2, _3 }; + } +}