From 457af03b6f40edb2c889d15c4e0c9137a5a160a8 Mon Sep 17 00:00:00 2001 From: Robert Pickering Date: Mon, 20 Nov 2023 16:44:38 +0100 Subject: [PATCH] =?UTF-8?q?Revert=20"[ASM]=20New=20marshalling=20system=20?= =?UTF-8?q?for=20Waf.Run=20calls=20to=20improve=20spe=E2=80=A6=20(#4891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "[ASM] New marshalling system for Waf.Run calls to improve speed and reduce allocations (#4302)" This reverts commit 096f106a1705d852cd511f1b7853a2e2348c77a4. * fix schema test by syncing with master --- tracer/src/Datadog.Trace/AppSec/Security.cs | 6 +- .../src/Datadog.Trace/AppSec/Waf/Context.cs | 27 +- .../src/Datadog.Trace/AppSec/Waf/Encoder.cs | 560 ++++-------------- tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs | 2 +- .../Waf/Initialization/WafConfigurator.cs | 16 +- .../Waf/NativeBindings/WafLibraryInvoker.cs | 12 +- tracer/src/Datadog.Trace/AppSec/Waf/Obj.cs | 118 ++++ .../src/Datadog.Trace/AppSec/Waf/ObjType.cs | 24 + .../ReturnTypes.Managed/DiagnosticResult.cs | 5 +- .../DiagnosticResultUtils.cs | 7 +- .../Waf/ReturnTypes.Managed/InitResult.cs | 2 +- .../Waf/ReturnTypes.Managed/UpdateResult.cs | 11 +- tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs | 53 +- .../Datadog.Trace/Util/UnmanagedMemoryPool.cs | 159 ----- tracer/src/Datadog.Trace/Util/UnsafeHelper.cs | 56 -- .../EncoderUnitTests.cs | 47 +- .../FuzzEncoder.cs | 7 +- .../WafMemoryTests.cs | 22 +- .../WafTests.cs | 1 - .../WafUpdateTests.cs | 1 - 20 files changed, 381 insertions(+), 755 deletions(-) create mode 100644 tracer/src/Datadog.Trace/AppSec/Waf/Obj.cs create mode 100644 tracer/src/Datadog.Trace/AppSec/Waf/ObjType.cs delete mode 100644 tracer/src/Datadog.Trace/Util/UnmanagedMemoryPool.cs delete mode 100644 tracer/src/Datadog.Trace/Util/UnsafeHelper.cs diff --git a/tracer/src/Datadog.Trace/AppSec/Security.cs b/tracer/src/Datadog.Trace/AppSec/Security.cs index 5f4b8893ec7f..62252c0da3af 100644 --- a/tracer/src/Datadog.Trace/AppSec/Security.cs +++ b/tracer/src/Datadog.Trace/AppSec/Security.cs @@ -337,11 +337,7 @@ void SetHtmlResponseContent() } /// Frees resources - public void Dispose() - { - _waf?.Dispose(); - Encoder.Pool.Dispose(); - } + public void Dispose() => _waf?.Dispose(); internal void SetDebugEnabled(bool enabled) { diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs index 84b80d9f8728..7322eef32405 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs @@ -20,11 +20,12 @@ internal class Context : IContext // the context handle should be locked, it is not safe for concurrent access and two // waf events may be processed at the same time due to code being run asynchronously + private readonly object _sync = new object(); private readonly IntPtr _contextHandle; private readonly Waf _waf; - private readonly List _argCache; + private readonly List _argCache = new(); private readonly Stopwatch _stopwatch; private readonly WafLibraryInvoker _wafLibraryInvoker; @@ -38,7 +39,6 @@ private Context(IntPtr contextHandle, Waf waf, WafLibraryInvoker wafLibraryInvok _waf = waf; _wafLibraryInvoker = wafLibraryInvoker; _stopwatch = new Stopwatch(); - _argCache = new(64); } ~Context() => Dispose(false); @@ -78,21 +78,13 @@ private Context(IntPtr contextHandle, Waf waf, WafLibraryInvoker wafLibraryInvok // not restart cause it's the total runtime over runs, and we run several * during request _stopwatch.Start(); + using var pwArgs = Encoder.Encode(addresses, _wafLibraryInvoker, _argCache, applySafetyLimits: true); + var rawArgs = pwArgs.RawPtr; WafReturnCode code; - lock (_stopwatch) + lock (_sync) { - var pool = Encoder.Pool; - try - { - var pwArgs = Encoder.Encode(addresses, applySafetyLimits: true, argToFree: _argCache, pool: pool); - code = _waf.Run(_contextHandle, ref pwArgs, ref retNative, timeoutMicroSeconds); - } - finally - { - pool.Return(_argCache); - _argCache.Clear(); - } + code = _waf.Run(_contextHandle, rawArgs, ref retNative, timeoutMicroSeconds); } _stopwatch.Stop(); @@ -120,7 +112,12 @@ public void Dispose(bool disposing) _disposed = true; - lock (_stopwatch) + foreach (var arg in _argCache) + { + arg.Dispose(); + } + + lock (_sync) { _wafLibraryInvoker.ContextDestroy(_contextHandle); } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Encoder.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Encoder.cs index fb0dc3be6744..ae36f0432bc3 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Encoder.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Encoder.cs @@ -5,10 +5,8 @@ #nullable enable using System; -using System.Collections; using System.Collections.Generic; using System.Linq; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using Datadog.Trace.AppSec.Waf.NativeBindings; @@ -20,461 +18,166 @@ namespace Datadog.Trace.AppSec.Waf { internal static class Encoder { - private const int MaxBytesForMaxStringLength = (WafConstants.MaxStringLength * 4) + 1; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(Encoder)); private static readonly int ObjectStructSize = Marshal.SizeOf(typeof(DdwafObjectStruct)); - [ThreadStatic] - private static UnmanagedMemoryPool? _pool; - - internal static UnmanagedMemoryPool Pool + public static ObjType DecodeArgsType(DDWAF_OBJ_TYPE t) { - get + return t switch { - if (_pool is { IsDisposed: false }) - { - return _pool; - } - - var instance = new UnmanagedMemoryPool(MaxBytesForMaxStringLength, 1000); - _pool = instance; - return instance; - } + DDWAF_OBJ_TYPE.DDWAF_OBJ_INVALID => ObjType.Invalid, + DDWAF_OBJ_TYPE.DDWAF_OBJ_SIGNED => ObjType.SignedNumber, + DDWAF_OBJ_TYPE.DDWAF_OBJ_UNSIGNED => ObjType.UnsignedNumber, + DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING => ObjType.String, + DDWAF_OBJ_TYPE.DDWAF_OBJ_BOOL => ObjType.Bool, + DDWAF_OBJ_TYPE.DDWAF_OBJ_DOUBLE => ObjType.Double, + DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY => ObjType.Array, + DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP => ObjType.Map, + DDWAF_OBJ_TYPE.DDWAF_OBJ_NULL => ObjType.Null, + _ => throw new Exception($"Invalid DDWAF_INPUT_TYPE {t}") + }; } - public static EncodeResult Encode(TInstance? o, int remainingDepth = WafConstants.MaxContainerDepth, string? key = null, bool applySafetyLimits = true) - { - var lstPointers = new List(); - var pool = Pool; - return new EncodeResult(lstPointers, pool, Encode(o, lstPointers, remainingDepth, key, applySafetyLimits, pool: pool)); - } + private static string TruncateLongString(string s) => s.Length > WafConstants.MaxStringLength ? s.Substring(0, WafConstants.MaxStringLength) : s; + + public static Obj Encode(object o, WafLibraryInvoker wafLibraryInvoker, List? argCache = null, bool applySafetyLimits = true) => EncodeInternal(o, argCache, WafConstants.MaxContainerDepth, applySafetyLimits, wafLibraryInvoker); - public static unsafe DdwafObjectStruct Encode(TInstance? o, List argToFree, int remainingDepth = WafConstants.MaxContainerDepth, string? key = null, bool applySafetyLimits = true, UnmanagedMemoryPool? pool = null) + private static Obj EncodeUnknownType(object o, WafLibraryInvoker wafLibraryInvoker) { - pool ??= Pool; + Log.Warning("Couldn't encode object of unknown type {Type}, falling back to ToString", o.GetType()); - DdwafObjectStruct ProcessKeyValuePairs(IEnumerable> enumerableDic, int count, delegate*, string?> getKey, delegate*, object?> getValue) - where TKey : notnull - { - var ddWafObjectMap = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP }; - if (!string.IsNullOrEmpty(key)) - { - var convertToUtf8 = ConvertToUtf8(key!, false); - ddWafObjectMap.ParameterName = convertToUtf8.Item1; - ddWafObjectMap.ParameterNameLength = (ulong)key!.Length; - } + var s = o.ToString() ?? string.Empty; + + return CreateNativeString(s, applyLimits: true, wafLibraryInvoker); + } - if (applySafetyLimits) + private static Obj EncodeInternal(T o, List? argCache, int remainingDepth, bool applyLimits, WafLibraryInvoker wafLibraryInvoker) + { + var value = + o switch { - if (remainingDepth-- <= 0) - { - string GetItemsAsString() - { - var sb = StringBuilderCache.Acquire(StringBuilderCache.MaxBuilderSize); - foreach (var x in enumerableDic) - { - sb.Append($"{getKey(x)}, {getValue(x)}, "); - } - - if (sb.Length > 0) - { - sb.Remove(sb.Length - 2, 2); - } - - return StringBuilderCache.GetStringAndRelease(sb); - } - - Log.Warning("EncodeDictionary: object graph too deep, truncating nesting {Items}", GetItemsAsString()); - return ddWafObjectMap; - } - - if (count > WafConstants.MaxContainerSize) - { - Log.Warning("EncodeList: list too long, it will be truncated, MaxMapOrArrayLength {MaxMapOrArrayLength}", WafConstants.MaxContainerSize); - } - } + null => CreateNativeNull(wafLibraryInvoker), + string s => CreateNativeString(s, applyLimits, wafLibraryInvoker), + JValue jv => CreateNativeString(jv.Value?.ToString() ?? string.Empty, applyLimits, wafLibraryInvoker), + int i => CreateNativeLong(i, wafLibraryInvoker), + uint i => CreateNativeUlong(i, wafLibraryInvoker), + long i => CreateNativeLong(i, wafLibraryInvoker), + ulong i => CreateNativeUlong(i, wafLibraryInvoker), + float i => CreateNativeDouble(i, wafLibraryInvoker), + double i => CreateNativeDouble(i, wafLibraryInvoker), + decimal i => CreateNativeDouble((double)i, wafLibraryInvoker), + bool b => CreateNativeBool(b, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable>> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable>> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IEnumerable> objDict => EncodeDictionary(objDict, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + IList objs => EncodeList(objs, argCache, remainingDepth, applyLimits, wafLibraryInvoker), + _ => EncodeUnknownType(o, wafLibraryInvoker), + }; - var childrenCount = !applySafetyLimits || count < WafConstants.MaxContainerSize ? count : WafConstants.MaxContainerSize; - var childrenFromPool = ObjectStructSize * childrenCount < MaxBytesForMaxStringLength; - var childrenData = childrenFromPool ? pool.Rent() : Marshal.AllocCoTaskMem(ObjectStructSize * childrenCount); + argCache?.Add(value); - if (enumerableDic is IDictionary) - { - var typeKVP = typeof(KeyValuePair); - if (typeKVP == typeof(KeyValuePair)) - { - EnumerateItems(); - } - else if (typeKVP == typeof(KeyValuePair)) - { - EnumerateItems(); - } - else if (typeKVP == typeof(KeyValuePair)) - { - EnumerateItems(); - } - else if (typeKVP == typeof(KeyValuePair>)) - { - EnumerateItems>(); - } - else if (typeKVP == typeof(KeyValuePair)) - { - EnumerateItems(); - } - else - { - EnumerateItems(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - void EnumerateItems() - where TKeySource : notnull - { - var itemData = childrenData; - foreach (var originalKeyValue in (Dictionary)enumerableDic) - { - var keyValue = UnsafeHelper.As, KeyValuePair>(originalKeyValue); - var key = getKey(keyValue!); - if (string.IsNullOrEmpty(key)) - { - childrenCount--; - Log.Warning("EncodeDictionary: ignoring dictionary member with null name"); - continue; - } - - *(DdwafObjectStruct*)itemData = Encode(getValue(keyValue!), argToFree, applySafetyLimits: applySafetyLimits, key: key, remainingDepth: remainingDepth, pool: pool); - itemData += ObjectStructSize; - } - } - } - else - { - var itemData = childrenData; - foreach (var keyValue in enumerableDic) - { - var key = getKey(keyValue); - if (string.IsNullOrEmpty(key)) - { - childrenCount--; - Log.Warning("EncodeDictionary: ignoring dictionary member with null name"); - continue; - } - - *(DdwafObjectStruct*)itemData = Encode(getValue(keyValue), argToFree, applySafetyLimits: applySafetyLimits, key: key, remainingDepth: remainingDepth, pool: pool); - itemData += ObjectStructSize; - } - } + return value; + } - ddWafObjectMap.Array = childrenData; - ddWafObjectMap.NbEntries = (ulong)childrenCount; - argToFree.Add(childrenData); - return ddWafObjectMap; - } + private static Obj EncodeList(IEnumerable objEnumerator, List? argCache, int remainingDepth, bool applyLimits, WafLibraryInvoker wafLibraryInvoker) + { + var arrNat = wafLibraryInvoker.ObjectArray(); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - Tuple ConvertToUtf8(string s, bool applySafety) + if (applyLimits && remainingDepth-- <= 0) { - IntPtr unmanagedMemory; - int writtenBytes; - var length = s.Length; - if (applySafety || length <= WafConstants.MaxStringLength) - { - length = Math.Min(length, WafConstants.MaxStringLength); - unmanagedMemory = pool.Rent(); - fixed (char* chrPtr = s) - { - writtenBytes = Encoding.UTF8.GetBytes(chrPtr, length, (byte*)unmanagedMemory, MaxBytesForMaxStringLength); - } - } - else - { - var bytesCount = Encoding.UTF8.GetMaxByteCount(length) + 1; - unmanagedMemory = Marshal.AllocCoTaskMem(bytesCount); - fixed (char* chrPtr = s) - { - writtenBytes = Encoding.UTF8.GetBytes(chrPtr, length, (byte*)unmanagedMemory, bytesCount); - } - } - - Marshal.WriteByte(unmanagedMemory, writtenBytes, (byte)'\0'); - argToFree.Add(unmanagedMemory); - return new Tuple(unmanagedMemory, length); + Log.Warning("EncodeList: object graph too deep, truncating nesting {Items}", string.Join(", ", objEnumerator)); + return new Obj(arrNat); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - DdwafObjectStruct GetStringObject(string value) + var count = objEnumerator is IList objs ? objs.Count : objEnumerator.Count(); + if (applyLimits && count > WafConstants.MaxContainerSize) { - var convertToUtf8 = ConvertToUtf8(value, applySafetyLimits); - var ddWafObject = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING, Array = convertToUtf8.Item1, NbEntries = (ulong)convertToUtf8.Item2 }; - return ddWafObject; + Log.Warning("EncodeList: list too long, it will be truncated, count: {Count}, MaxMapOrArrayLength {MaxMapOrArrayLength}", count, WafConstants.MaxContainerSize); + objEnumerator = objEnumerator.Take(WafConstants.MaxContainerSize); } - DdwafObjectStruct ddwafObjectStruct; - - switch (o) + foreach (var o in objEnumerator) { - case string str: - { - ddwafObjectStruct = GetStringObject(str); - break; - } - - case JValue: - { - ddwafObjectStruct = GetStringObject(o?.ToString() ?? string.Empty); - break; - } - - case null: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_NULL }; - break; - } - - case ulong u: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_UNSIGNED, UintValue = u }; - break; - } - - case uint u: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_UNSIGNED, UintValue = u }; - break; - } - - case int i: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_SIGNED, IntValue = i }; - break; - } - - case long u: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_SIGNED, IntValue = u }; - break; - } - - case decimal d: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_DOUBLE, DoubleValue = (double)d }; - break; - } + var value = EncodeInternal(o, argCache, remainingDepth, applyLimits, wafLibraryInvoker); + wafLibraryInvoker.ObjectArrayAdd(arrNat, value.RawPtr); + } - case double d: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_DOUBLE, DoubleValue = d }; - break; - } + return new Obj(arrNat); + } - case float d: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_DOUBLE, DoubleValue = d }; - break; - } + private static Obj EncodeDictionary(IEnumerable> objDictEnumerator, List? argCache, int remainingDepth, bool applyLimits, WafLibraryInvoker wafLibraryInvoker) + { + var mapNat = wafLibraryInvoker.ObjectMap(); - case bool b: - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_BOOL, ByteValue = b ? (byte)1 : (byte)0 }; - break; + if (applyLimits && remainingDepth-- <= 0) + { + Log.Warning("EncodeDictionary: object graph too deep, truncating nesting {Items}", string.Join(", ", objDictEnumerator.Select(x => $"{x.Key}, {x.Value}"))); + return new Obj(mapNat); + } - case IEnumerable> objDict: - { - var collectionDict = objDict as ICollection> ?? objDict.ToList(); - var count = collectionDict.Count; - ddwafObjectStruct = ProcessKeyValuePairs(collectionDict, count, &GetKey1, &GetValue1); - static string GetKey1(KeyValuePair item) => item.Key; - static object GetValue1(KeyValuePair item) => item.Value; - break; - } + var count = objDictEnumerator is IDictionary objDict ? objDict.Count : objDictEnumerator.Count(); - case IEnumerable> objDict: - { - var collectionDict = objDict as ICollection> ?? objDict.ToList(); - var count = collectionDict.Count; - ddwafObjectStruct = ProcessKeyValuePairs(collectionDict, count, &GetKey1, &GetValue1); - static string GetKey1(KeyValuePair item) => item.Key; - static object GetValue1(KeyValuePair item) => item.Value; - break; - } + if (applyLimits && count > WafConstants.MaxContainerSize) + { + Log.Warning("EncodeDictionary: list too long, it will be truncated, count: {Count}, MaxMapOrArrayLength {MaxMapOrArrayLength}", count, WafConstants.MaxContainerSize); + objDictEnumerator = objDictEnumerator.Take(WafConstants.MaxContainerSize); + } - case IEnumerable> objDict: + foreach (var o in objDictEnumerator) + { + var name = o.Key; + if (name != null) { - var collectionDict = objDict as ICollection> ?? objDict.ToList(); - var count = collectionDict.Count; - ddwafObjectStruct = ProcessKeyValuePairs(collectionDict, count, &GetKey2, &GetValue2); - static string GetKey2(KeyValuePair item) => item.Key; - static object GetValue2(KeyValuePair item) => item.Value; - break; + var value = EncodeInternal(o.Value, argCache, remainingDepth, applyLimits, wafLibraryInvoker); + wafLibraryInvoker.ObjectMapAdd(mapNat, name, Convert.ToUInt64(name.Length), value.RawPtr); } - - case IEnumerable> objDict: + else { - var collectionDict = objDict as ICollection> ?? objDict.ToList(); - var count = collectionDict.Count; - ddwafObjectStruct = ProcessKeyValuePairs(collectionDict, count, &GetKey3, &GetValue3); - static string GetKey3(KeyValuePair item) => item.Key; - static object GetValue3(KeyValuePair item) => item.Value; - break; + Log.Warning("EncodeDictionary: ignoring dictionary member with null name"); } + } - case IEnumerable> objDict: - { - var collectionDict = objDict as ICollection> ?? objDict.ToList(); - var count = collectionDict.Count; - ddwafObjectStruct = ProcessKeyValuePairs(collectionDict, count, &GetKey4, &GetValue4); - static string GetKey4(KeyValuePair item) => item.Key; - static object GetValue4(KeyValuePair item) => item.Value; - break; - } + return new Obj(mapNat); + } - case IEnumerable>> objDict: - { - var collectionDict = objDict as ICollection>> ?? objDict.ToList(); - var count = collectionDict.Count; - ddwafObjectStruct = ProcessKeyValuePairs(collectionDict, count, &GetKey5, &GetValue5); - static string GetKey5(KeyValuePair> item) => item.Key; - static object GetValue5(KeyValuePair> item) => item.Value; - break; - } + private static Obj CreateNativeString(string s, bool applyLimits, WafLibraryInvoker wafLibraryInvoker) + { + var encodeString = + applyLimits + ? TruncateLongString(s) + : s; + return new Obj(wafLibraryInvoker.ObjectStringLength(encodeString, Convert.ToUInt64(encodeString.Length))); + } - case IEnumerable enumerable: - { - ddwafObjectStruct = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_ARRAY }; - - if (applySafetyLimits && remainingDepth-- <= 0) - { - Log.Warning("EncodeList: object graph too deep, truncating nesting {Items}", string.Join(", ", enumerable)); - break; - } - - if (enumerable is IList { Count: var count } listInstance) - { - if (applySafetyLimits && count > WafConstants.MaxContainerSize) - { - Log.Warning("EncodeList: list too long, it will be truncated, MaxMapOrArrayLength {MaxMapOrArrayLength}", WafConstants.MaxContainerSize); - } - - var childrenCount = !applySafetyLimits || count < WafConstants.MaxContainerSize ? count : WafConstants.MaxContainerSize; - var childrenFromPool = ObjectStructSize * childrenCount < MaxBytesForMaxStringLength; - var childrenData = childrenFromPool ? pool.Rent() : Marshal.AllocCoTaskMem(ObjectStructSize * childrenCount); - - // Avoid boxing of known values types from the switch above - switch (listInstance) - { - case IList boolCollection: - EnumerateAndEncode(boolCollection); - break; - case IList intCollection: - EnumerateAndEncode(intCollection); - break; - case IList intCollection: - EnumerateAndEncode(intCollection); - break; - case IList intCollection: - EnumerateAndEncode(intCollection); - break; - case IList intCollection: - EnumerateAndEncode(intCollection); - break; - case IList uintCollection: - EnumerateAndEncode(uintCollection); - break; - case IList longCollection: - EnumerateAndEncode(longCollection); - break; - case IList ulongCollection: - EnumerateAndEncode(ulongCollection); - break; - default: - EnumerateAndEncodeIList(listInstance); - break; - } - - ddwafObjectStruct.Array = childrenData; - ddwafObjectStruct.NbEntries = (ulong)childrenCount; - argToFree.Add(childrenData); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - void EnumerateAndEncode(IList lstInstance) - { - var itemData = childrenData; - for (var idx = 0; idx < count; idx++) - { - *(DdwafObjectStruct*)itemData = Encode(lstInstance[idx], argToFree, applySafetyLimits: applySafetyLimits, remainingDepth: remainingDepth, pool: pool); - itemData += ObjectStructSize; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - void EnumerateAndEncodeIList(IList lstInstance) - { - var itemData = childrenData; - for (var idx = 0; idx < count; idx++) - { - *(DdwafObjectStruct*)itemData = Encode(lstInstance[idx], argToFree, applySafetyLimits: applySafetyLimits, remainingDepth: remainingDepth, pool: pool); - itemData += ObjectStructSize; - } - } - } - else - { - var childrenCount = 0; - // Let's enumerate first. - foreach (var val in enumerable) - { - childrenCount++; - if (applySafetyLimits && childrenCount == WafConstants.MaxContainerSize) - { - Log.Warning("EncodeList: list too long, it will be truncated, MaxMapOrArrayLength {MaxMapOrArrayLength}", WafConstants.MaxContainerSize); - break; - } - } - - if (childrenCount > 0) - { - var childrenFromPool = ObjectStructSize * childrenCount < MaxBytesForMaxStringLength; - var childrenData = childrenFromPool ? pool.Rent() : Marshal.AllocCoTaskMem(ObjectStructSize * childrenCount); - var itemData = childrenData; - var idx = 0; - foreach (var val in enumerable) - { - if (idx > childrenCount) - { - break; - } - - *(DdwafObjectStruct*)itemData = Encode(val, argToFree, applySafetyLimits: applySafetyLimits, remainingDepth: remainingDepth, pool: pool); - itemData += ObjectStructSize; - idx++; - } - - ddwafObjectStruct.Array = childrenData; - ddwafObjectStruct.NbEntries = (ulong)childrenCount; - argToFree.Add(childrenData); - } - } - - break; - } + private static Obj CreateNativeBool(bool b, WafLibraryInvoker wafLibraryInvoker) => new(wafLibraryInvoker.ObjectBool(b)); - default: - if (Log.IsEnabled(Vendors.Serilog.Events.LogEventLevel.Debug)) - { - Log.Debug("Couldn't encode object of unknown type {Type}, falling back to ToString", o.GetType()); - } + private static Obj CreateNativeLong(long value, WafLibraryInvoker wafLibraryInvoker) => new(wafLibraryInvoker.ObjectLong(value)); - ddwafObjectStruct = GetStringObject(string.Empty); - break; - } + private static Obj CreateNativeNull(WafLibraryInvoker wafLibraryInvoker) => new(wafLibraryInvoker.ObjectNull()); - if (!string.IsNullOrEmpty(key)) - { - ddwafObjectStruct.ParameterName = ConvertToUtf8(key!, false).Item1; - ddwafObjectStruct.ParameterNameLength = (ulong)key!.Length; - } + private static Obj CreateNativeUlong(ulong value, WafLibraryInvoker wafLibraryInvoker) => new(wafLibraryInvoker.ObjectUlong(value)); - return ddwafObjectStruct; - } + private static Obj CreateNativeDouble(double value, WafLibraryInvoker wafLibraryInvoker) => new(wafLibraryInvoker.ObjectDouble(value)); public static string FormatArgs(object o) { @@ -495,14 +198,12 @@ private static void FormatArgsInternal(object o, StringBuilder sb) uint i => sb.Append(i), ulong i => sb.Append(i), double i => sb.Append(i), - bool i => sb.Append(i), IEnumerable> objDict => FormatDictionary(objDict.Select(x => new KeyValuePair(x.Key, x.Value)), sb), IEnumerable> objDict => FormatDictionary(objDict.Select(x => new KeyValuePair(x.Key, x.Value)), sb), IEnumerable>> objDict => FormatDictionary(objDict.Select(x => new KeyValuePair(x.Key, x.Value)), sb), // dont remove IEnumerable>, it is used for logging cookies which are this type in debug mode IEnumerable> objDict => FormatDictionary(objDict.Select(x => new KeyValuePair(x.Key, x.Value)), sb), IEnumerable> objDict => FormatDictionary(objDict, sb), - IEnumerable> objDict => FormatDictionary(objDict.Select(x => new KeyValuePair(x.Key, x.Value)), sb), IList objs => FormatList(objs, sb), IList objs => FormatList(objs, sb), // this becomes ugly but this should change once PR improving marshalling of the waf is merged @@ -512,7 +213,6 @@ private static void FormatArgsInternal(object o, StringBuilder sb) IList objs => FormatList(objs, sb), IList objs => FormatList(objs, sb), IList objs => FormatList(objs, sb), - IList objs => FormatList(objs, sb), IList objs => FormatList(objs, sb), IList objs => FormatList(objs, sb), _ => sb.Append($"Error: couldn't format type: {o?.GetType()}") @@ -577,25 +277,5 @@ private static StringBuilder FormatList(IEnumerable objs, StringBuilder sb sb.Append(" ]"); return sb; } - - public readonly ref struct EncodeResult - { - private readonly List _pointers; - private readonly UnmanagedMemoryPool _pool; - public readonly DdwafObjectStruct Result; - - internal EncodeResult(List pointers, UnmanagedMemoryPool pool, DdwafObjectStruct result) - { - _pointers = pointers; - _pool = pool; - Result = result; - } - - public void Dispose() - { - _pool.Return(_pointers); - _pointers.Clear(); - } - } } } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs b/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs index 2a86d05029c0..0ca5c908af7d 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs @@ -19,7 +19,7 @@ internal interface IWaf : IDisposable public IContext CreateContext(); - internal WafReturnCode Run(IntPtr contextHandle, ref DdwafObjectStruct args, ref DdwafResultStruct retNative, ulong timeoutMicroSeconds); + internal WafReturnCode Run(IntPtr contextHandle, IntPtr rawArgs, ref DdwafResultStruct retNative, ulong timeoutMicroSeconds); UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus); } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs index 8d3e637310d7..9810fa6c779a 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs @@ -155,7 +155,7 @@ private static void LogRuleDetailsIfDebugEnabled(JToken root) return root; } - internal InitResult ConfigureAndDispose(DdwafObjectStruct? rulesObj, string? rulesFile, string obfuscationParameterKeyRegex, string obfuscationParameterValueRegex) + internal InitResult ConfigureAndDispose(Obj? rulesObj, string? rulesFile, List argsToDispose, string obfuscationParameterKeyRegex, string obfuscationParameterValueRegex) { if (rulesObj == null) { @@ -163,9 +163,9 @@ internal InitResult ConfigureAndDispose(DdwafObjectStruct? rulesObj, string? rul return InitResult.FromUnusableRuleFile(); } + Obj? diagnostics = null; var keyRegex = IntPtr.Zero; var valueRegex = IntPtr.Zero; - var diagnostics = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP }; try { @@ -174,10 +174,10 @@ internal InitResult ConfigureAndDispose(DdwafObjectStruct? rulesObj, string? rul valueRegex = Marshal.StringToHGlobalAnsi(obfuscationParameterValueRegex); args.KeyRegex = keyRegex; args.ValueRegex = valueRegex; - args.FreeWafFunction = IntPtr.Zero; + args.FreeWafFunction = _wafLibraryInvoker.ObjectFreeFuncPtr; - var rules = rulesObj.Value; - var wafHandle = _wafLibraryInvoker.Init(ref rules, ref args, ref diagnostics); + diagnostics = new Obj(_wafLibraryInvoker.ObjectMap()); + var wafHandle = _wafLibraryInvoker.Init(rulesObj.RawPtr, ref args, diagnostics.RawPtr); if (wafHandle == IntPtr.Zero) { Log.Warning("DDAS-0005-00: WAF initialization failed."); @@ -222,9 +222,11 @@ internal InitResult ConfigureAndDispose(DdwafObjectStruct? rulesObj, string? rul Marshal.FreeHGlobal(valueRegex); } - if (diagnostics.Array != IntPtr.Zero) + diagnostics?.Dispose(_wafLibraryInvoker); + rulesObj.Dispose(_wafLibraryInvoker); + foreach (var arg in argsToDispose) { - _wafLibraryInvoker.ObjectFreePtr(ref diagnostics.Array); + arg.Dispose(); } } } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/NativeBindings/WafLibraryInvoker.cs b/tracer/src/Datadog.Trace/AppSec/Waf/NativeBindings/WafLibraryInvoker.cs index 40a7f0810724..18ed376e252c 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/NativeBindings/WafLibraryInvoker.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/NativeBindings/WafLibraryInvoker.cs @@ -84,13 +84,13 @@ private WafLibraryInvoker(IntPtr libraryHandle) private delegate void FreeResultDelegate(ref DdwafResultStruct output); - private delegate IntPtr InitDelegate(ref DdwafObjectStruct wafRule, ref DdwafConfigStruct config, ref DdwafObjectStruct diagnostics); + private delegate IntPtr InitDelegate(IntPtr wafRule, ref DdwafConfigStruct config, IntPtr diagnostics); - private delegate IntPtr UpdateDelegate(IntPtr oldWafHandle, ref DdwafObjectStruct wafRule, ref DdwafObjectStruct diagnostics); + private delegate IntPtr UpdateDelegate(IntPtr oldWafHandle, IntPtr wafRule, IntPtr diagnostics); private delegate IntPtr InitContextDelegate(IntPtr wafHandle); - private delegate WafReturnCode RunDelegate(IntPtr context, ref DdwafObjectStruct newArgs, ref DdwafResultStruct result, ulong timeLeftInUs); + private delegate WafReturnCode RunDelegate(IntPtr context, IntPtr newArgs, ref DdwafResultStruct result, ulong timeLeftInUs); private delegate void DestroyDelegate(IntPtr handle); @@ -204,7 +204,7 @@ internal string GetVersion() return _version; } - internal IntPtr Init(ref DdwafObjectStruct wafRule, ref DdwafConfigStruct config, ref DdwafObjectStruct diagnostics) => _initField(ref wafRule, ref config, ref diagnostics); + internal IntPtr Init(IntPtr wafRule, ref DdwafConfigStruct config, IntPtr diagnostics) => _initField(wafRule, ref config, diagnostics); /// /// Only give a non null ruleSetInfo when updating rules. When updating rules overrides, rules datas, the ruleSetInfo will return no error and no diagnostics, even if there are, it's misleading, so give null in this case. @@ -213,11 +213,11 @@ internal string GetVersion() /// a pointer to the new waf data (rules or overrides or other) /// errors and diagnostics of the update, only for valid for new rules /// the new waf handle, if error, will be a nullptr - internal IntPtr Update(IntPtr oldWafHandle, ref DdwafObjectStruct wafData, ref DdwafObjectStruct diagnostics) => _updateField(oldWafHandle, ref wafData, ref diagnostics); + internal IntPtr Update(IntPtr oldWafHandle, IntPtr wafData, IntPtr diagnostics) => _updateField(oldWafHandle, wafData, diagnostics); internal IntPtr InitContext(IntPtr powerwafHandle) => _initContextField(powerwafHandle); - internal WafReturnCode Run(IntPtr context, ref DdwafObjectStruct newArgs, ref DdwafResultStruct result, ulong timeLeftInUs) => _runField(context, ref newArgs, ref result, timeLeftInUs); + internal WafReturnCode Run(IntPtr context, IntPtr newArgs, ref DdwafResultStruct result, ulong timeLeftInUs) => _runField(context, newArgs, ref result, timeLeftInUs); internal void Destroy(IntPtr wafHandle) => _destroyField(wafHandle); diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Obj.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Obj.cs new file mode 100644 index 000000000000..3d4edb5e56e7 --- /dev/null +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Obj.cs @@ -0,0 +1,118 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Runtime.InteropServices; +using Datadog.Trace.AppSec.Waf.NativeBindings; + +namespace Datadog.Trace.AppSec.Waf +{ + // NOTE: this is referred to as ddwaf_object in the C++ code, we call it Obj to avoid a naming clash + internal class Obj : IDisposable + { + private IntPtr ptr; + private DdwafObjectStruct innerObj; + private bool innerObjInitialized; + private bool disposed; + + public Obj(IntPtr ptr) => this.ptr = ptr; + + ~Obj() + { + Dispose(false); + } + + public ObjType ArgsType + { + get + { + Initialize(); + return Encoder.DecodeArgsType(innerObj.Type); + } + } + + public long IntValue + { + get + { + Initialize(); + return innerObj.IntValue; + } + } + + public ulong UintValue + { + get + { + Initialize(); + return innerObj.UintValue; + } + } + + public nint InnerPtr + { + get + { + Initialize(); + return innerObj.Array; + } + } + + public DdwafObjectStruct InnerStruct + { + get + { + Initialize(); + return innerObj; + } + } + + public IntPtr RawPtr => ptr; + + public void Dispose(WafLibraryInvoker libraryInvoker) + { + if (libraryInvoker != null) + { + var rawPtr = ptr; + libraryInvoker.ObjectFreePtr(ref rawPtr); + Dispose(); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (disposed) + { + ptr = IntPtr.Zero; + return; + } + + disposed = true; + + if (ptr != IntPtr.Zero) + { + Marshal.FreeHGlobal(ptr); + ptr = IntPtr.Zero; + } + } + + private void Initialize() + { + if (innerObjInitialized) + { + return; + } + + innerObjInitialized = true; + innerObj = (DdwafObjectStruct)Marshal.PtrToStructure(ptr, typeof(DdwafObjectStruct)); + } + } +} diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ObjType.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ObjType.cs new file mode 100644 index 000000000000..0f94006addb0 --- /dev/null +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ObjType.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Datadog.Trace.AppSec.Waf +{ + internal enum ObjType + { + Invalid = 0, + SignedNumber = 1 << 0, + UnsignedNumber = 1 << 1, + String = 1 << 2, + Array = 1 << 3, + Map = 1 << 4, + Bool = 1 << 5, + Double = 1 << 6, + Null = 1 << 7 + } +} diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResult.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResult.cs index bfeda9ad52d2..f6fdd23d2fcb 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResult.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResult.cs @@ -6,7 +6,6 @@ #nullable enable using System; using System.Collections.Generic; -using Datadog.Trace.AppSec.Waf.NativeBindings; namespace Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; @@ -19,9 +18,9 @@ internal class DiagnosticResult private readonly Lazy _rulesData; private readonly Lazy _rulesOverride; - public DiagnosticResult(DdwafObjectStruct diagObject) + public DiagnosticResult(Obj diagObject) { - _diagnosticsData = diagObject.DecodeMap(); + _diagnosticsData = diagObject.InnerStruct.DecodeMap(); _customRules = MakeLazy("custom_rules"); _exclusions = MakeLazy("exclusions"); _rules = MakeLazy("rules"); diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResultUtils.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResultUtils.cs index c055a6e2b8f2..36a1f1be762a 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResultUtils.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/DiagnosticResultUtils.cs @@ -6,7 +6,6 @@ #nullable enable using System; using System.Collections.Generic; -using Datadog.Trace.AppSec.Waf.NativeBindings; using Datadog.Trace.Vendors.Serilog; namespace Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; @@ -15,15 +14,15 @@ namespace Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; internal static class DiagnosticResultUtils { - internal static ReportedDiagnostics ExtractReportedDiagnostics(DdwafObjectStruct diagObject, bool noRuleDiagnoticsIsError) + internal static ReportedDiagnostics ExtractReportedDiagnostics(Obj diagObject, bool noRuleDiagnoticsIsError) { ushort failedCount = 0; ushort loadedCount = 0; - var rulesetVersion = string.Empty; + string rulesetVersion = string.Empty; IReadOnlyDictionary? errors = null; try { - if (diagObject.Type == DDWAF_OBJ_TYPE.DDWAF_OBJ_INVALID) + if (diagObject.ArgsType == ObjType.Invalid) { errors = new Dictionary { { "diagnostics-error", "Waf didn't provide a valid diagnostics object at initialization, most likely due to an older waf version < 1.11.0" } }; return new ReportedDiagnostics { FailedCount = failedCount, LoadedCount = loadedCount, RulesetVersion = rulesetVersion, Errors = errors }; diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs index 493bb462400c..20a9aa7b01f6 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs @@ -72,7 +72,7 @@ private InitResult(ushort failedToLoadRules, ushort loadedRules, string ruleFile internal static InitResult FromIncompatibleWaf() => new(0, 0, string.Empty, new Dictionary(), incompatibleWaf: true); - internal static InitResult From(DdwafObjectStruct diagObject, IntPtr? wafHandle, WafLibraryInvoker? wafLibraryInvoker) + internal static InitResult From(Obj diagObject, IntPtr? wafHandle, WafLibraryInvoker? wafLibraryInvoker) { var reportedDiag = DiagnosticResultUtils.ExtractReportedDiagnostics(diagObject, true); diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/UpdateResult.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/UpdateResult.cs index af78d9259d12..63b1c672f228 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/UpdateResult.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/UpdateResult.cs @@ -6,25 +6,24 @@ #nullable enable using System; using System.Collections.Generic; -using Datadog.Trace.AppSec.Waf.NativeBindings; using Datadog.Trace.Vendors.Newtonsoft.Json; namespace Datadog.Trace.AppSec.Waf.ReturnTypes.Managed { internal class UpdateResult { - internal UpdateResult(DdwafObjectStruct? diagObject, bool success, bool unusableRules = false) + internal UpdateResult(Obj? diagObject, bool success, bool unusableRules = false) { if (diagObject != null) { - var reportedDiag = DiagnosticResultUtils.ExtractReportedDiagnostics(diagObject.Value, false); + var reportedDiag = DiagnosticResultUtils.ExtractReportedDiagnostics(diagObject, false); FailedToLoadRules = reportedDiag.FailedCount; LoadedRules = reportedDiag.LoadedCount; Errors = reportedDiag.Errors; RuleFileVersion = reportedDiag.RulesetVersion; - if (Errors is { Count: > 0 }) + if (Errors != null && Errors.Count > 0) { HasErrors = true; ErrorMessage = JsonConvert.SerializeObject(Errors); @@ -54,8 +53,8 @@ internal UpdateResult(DdwafObjectStruct? diagObject, bool success, bool unusable internal string? RuleFileVersion { get; } - public static UpdateResult FromUnusableRules() => new(null, false, true); + public static UpdateResult FromUnusableRules() => new UpdateResult(null, false, true); - public static UpdateResult FromFailed() => new(null, false); + public static UpdateResult FromFailed() => new UpdateResult(null, false); } } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs index d54d41799364..1d56fa3494fe 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs @@ -63,7 +63,6 @@ internal static InitResult Create(WafLibraryInvoker wafLibraryInvoker, string ob // set the log level and setup the logger wafLibraryInvoker.SetupLogging(GlobalSettings.Instance.DebugEnabledInternal); - var jtokenRoot = rulesFromRcm ?? WafConfigurator.DeserializeEmbeddedOrStaticRules(embeddedRulesetPath)!; if (setupWafSchemaExtraction) { @@ -71,22 +70,24 @@ internal static InitResult Create(WafLibraryInvoker wafLibraryInvoker, string ob jtokenRoot.Children().Last().AddAfterSelf(schemaConfig!.Children()); } - var configObj = Encoder.Encode(jtokenRoot, applySafetyLimits: false); + var argCache = new List(); + var configObj = Encoder.Encode(jtokenRoot, wafLibraryInvoker, argCache, applySafetyLimits: false); - var initResult = wafConfigurator.ConfigureAndDispose(configObj.Result, rulesFromRcm != null ? embeddedRulesetPath : "RemoteConfig", obfuscationParameterKeyRegex, obfuscationParameterValueRegex); + var ruleSetFile = rulesFromRcm != null ? embeddedRulesetPath : "RemoteConfig"; + var initResult = wafConfigurator.ConfigureAndDispose(configObj, ruleSetFile, argCache, obfuscationParameterKeyRegex, obfuscationParameterValueRegex); initResult.EmbeddedRules = jtokenRoot; + return initResult; } - private UpdateResult UpdateWafAndDisposeItems(DdwafObjectStruct updateData) + private UpdateResult UpdateWafAndDisposeItems(Obj updateData, IEnumerable argsToDispose) { - UpdateResult? res = null; - DdwafObjectStruct? diagnostics = null; + UpdateResult res; + Obj? diagnostics = null; try { - diagnostics = new DdwafObjectStruct { Type = DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP }; - var diagnosticsValue = diagnostics.Value; - var newHandle = _wafLibraryInvoker.Update(_wafHandle, ref updateData, ref diagnosticsValue); + diagnostics = new Obj(_wafLibraryInvoker.ObjectMap()); + var newHandle = _wafLibraryInvoker.Update(_wafHandle, updateData.RawPtr, diagnostics.RawPtr); if (newHandle != IntPtr.Zero) { var oldHandle = _wafHandle; @@ -95,7 +96,9 @@ private UpdateResult UpdateWafAndDisposeItems(DdwafObjectStruct updateData) _wafHandle = newHandle; _wafLocker.ExitWriteLock(); _wafLibraryInvoker.Destroy(oldHandle); - return new(diagnosticsValue, true); + res = new UpdateResult(diagnostics, true); + DisposeItems(updateData, argsToDispose, diagnostics); + return res; } _wafLibraryInvoker.Destroy(newHandle); @@ -105,20 +108,23 @@ private UpdateResult UpdateWafAndDisposeItems(DdwafObjectStruct updateData) { Log.Error(e, "An exception occurred while trying to update waf with new data"); } - finally - { - res ??= new(diagnostics, false); - - if (diagnostics?.Array != IntPtr.Zero) - { - var diagValue = diagnostics!.Value; - _wafLibraryInvoker.ObjectFreePtr(ref diagValue.Array); - } - } + res = new UpdateResult(diagnostics, false); + DisposeItems(updateData, argsToDispose, diagnostics); return res; } + private void DisposeItems(Obj updateData, IEnumerable argsToDispose, Obj? diagnostics) + { + diagnostics?.Dispose(_wafLibraryInvoker); + updateData.Dispose(_wafLibraryInvoker); + + foreach (var arg in argsToDispose) + { + arg.Dispose(); + } + } + public UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus) { var dic = configurationStatus.BuildDictionaryForWafAccordingToIncomingUpdate(); @@ -161,10 +167,11 @@ public UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configu public UpdateResult Update(IDictionary arguments) { + var argsToDispose = new List(); UpdateResult updated; try { - using var encodedArgs = Encoder.Encode(arguments, applySafetyLimits: false); + var encodedArgs = Encoder.Encode(arguments, _wafLibraryInvoker, argsToDispose, false); // only if rules are provided will the waf give metrics if (arguments.ContainsKey("rules")) @@ -172,7 +179,7 @@ public UpdateResult Update(IDictionary arguments) TelemetryFactory.Metrics.RecordCountWafUpdates(); } - updated = UpdateWafAndDisposeItems(encodedArgs.Result); + updated = UpdateWafAndDisposeItems(encodedArgs, argsToDispose); } catch { @@ -183,7 +190,7 @@ public UpdateResult Update(IDictionary arguments) } // Doesn't require a non disposed waf handle, but as the WAF instance needs to be valid for the lifetime of the context, if waf is disposed, don't run (unpredictable) - public WafReturnCode Run(IntPtr contextHandle, ref DdwafObjectStruct args, ref DdwafResultStruct retNative, ulong timeoutMicroSeconds) => _wafLibraryInvoker.Run(contextHandle, ref args, ref retNative, timeoutMicroSeconds); + public WafReturnCode Run(IntPtr contextHandle, IntPtr rawArgs, ref DdwafResultStruct retNative, ulong timeoutMicroSeconds) => _wafLibraryInvoker.Run(contextHandle, rawArgs, ref retNative, timeoutMicroSeconds); internal static List MergeRuleData(IEnumerable res) { diff --git a/tracer/src/Datadog.Trace/Util/UnmanagedMemoryPool.cs b/tracer/src/Datadog.Trace/Util/UnmanagedMemoryPool.cs deleted file mode 100644 index ef1cea756ff6..000000000000 --- a/tracer/src/Datadog.Trace/Util/UnmanagedMemoryPool.cs +++ /dev/null @@ -1,159 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace Datadog.Trace.Util; - -/// -/// Beware that this type is not thread safe and should be used with [ThreadStatic] -/// -internal unsafe class UnmanagedMemoryPool : IDisposable -{ - private readonly IntPtr* _items; - private readonly int _length; - private readonly int _blockSize; - private int _initialSearchIndex; - private bool _isDisposed; - - public UnmanagedMemoryPool(int blockSize, int poolSize) - { - _blockSize = blockSize; - _items = (IntPtr*)Marshal.AllocCoTaskMem(poolSize * sizeof(IntPtr)); - _length = poolSize; - _initialSearchIndex = 0; - - for (var i = 0; i < _length; i++) - { - _items[i] = IntPtr.Zero; - } - } - - ~UnmanagedMemoryPool() - { - Dispose(); - } - - public bool IsDisposed => _isDisposed; - - /// - /// Beware that this method is not thread safe, and needs to be used with [ThreadStatic] in case of multiple thread scenarios - /// - /// Pointer to a memory block of size specified in the constructor - public IntPtr Rent() - { - if (IsDisposed) - { - ThrowObjectDisposedException(); - } - - for (var i = _initialSearchIndex; i < _length; i++) - { - var inst = _items[i]; - if (inst != IntPtr.Zero) - { - _initialSearchIndex = i + 1; - _items[i] = IntPtr.Zero; - return inst; - } - } - - return RentSlow(); - } - - private IntPtr RentSlow() - { - return Marshal.AllocCoTaskMem(_blockSize); - } - - public void Return(IntPtr block) - { - if (IsDisposed) - { - ThrowObjectDisposedException(); - } - - for (var i = 0; i < _length; i++) - { - if (_items[i] == IntPtr.Zero) - { - _items[i] = block; - _initialSearchIndex = 0; - return; - } - } - - ReturnSlow(block); - } - - public void Return(IList blocks) - { - if (IsDisposed) - { - ThrowObjectDisposedException(); - } - - if (blocks.Count == 0) - { - return; - } - - var blockIndex = 0; - for (var i = 0; i < _length; i++) - { - if (_items[i] == IntPtr.Zero) - { - _items[i] = blocks[blockIndex++]; - if (blockIndex == blocks.Count) - { - _initialSearchIndex = 0; - return; - } - } - } - - for (var i = blockIndex; i < blocks.Count; i++) - { - ReturnSlow(blocks[i]); - } - } - - private void ReturnSlow(IntPtr block) - { - Marshal.FreeCoTaskMem(block); - } - - public void Dispose() - { - if (IsDisposed) - { - return; - } - - for (var i = 0; i < _length; i++) - { - if (_items[i] != IntPtr.Zero) - { - Marshal.FreeCoTaskMem(_items[i]); - _items[i] = IntPtr.Zero; - } - } - - Marshal.FreeCoTaskMem((IntPtr)_items); - - _isDisposed = true; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private void ThrowObjectDisposedException() - { - throw new ObjectDisposedException("UnmanagedMemoryPool"); - } -} diff --git a/tracer/src/Datadog.Trace/Util/UnsafeHelper.cs b/tracer/src/Datadog.Trace/Util/UnsafeHelper.cs deleted file mode 100644 index 6d6ae73c8981..000000000000 --- a/tracer/src/Datadog.Trace/Util/UnsafeHelper.cs +++ /dev/null @@ -1,56 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#nullable enable - -using System.Reflection.Emit; -using System.Runtime.CompilerServices; - -namespace Datadog.Trace.Util; - -internal static class UnsafeHelper -{ -#if NETCOREAPP3_1_OR_GREATER - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static TTo As(TFrom value) - { - return Unsafe.As(ref value); - } -#else - private static readonly object Instance = new(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static TTo As(TFrom value) - { - return Converter.Convert(value); - } - - private static class Converter - { - private static readonly ConvertDelegate ConverterInstance; - - static Converter() - { - var dMethod = new DynamicMethod($"Converter<{typeof(TFrom).Name},{typeof(TTo).Name}>", typeof(TTo), new[] { typeof(object), typeof(TFrom) }, typeof(ConvertDelegate).Module, true); - var il = dMethod.GetILGenerator(); - il.Emit(OpCodes.Ldarg_1); - il.Emit(OpCodes.Ret); - ConverterInstance = (ConvertDelegate)dMethod.CreateDelegate(typeof(ConvertDelegate), Instance); - } - - private delegate TTo ConvertDelegate(TFrom value); - - public static TTo Convert(ref TFrom value) - { - return ConverterInstance(value); - } - - public static TTo Convert(TFrom value) - { - return ConverterInstance(value); - } - } -#endif -} diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/EncoderUnitTests.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/EncoderUnitTests.cs index 26884336ed03..a1e403b6fa5b 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/EncoderUnitTests.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/EncoderUnitTests.cs @@ -20,13 +20,16 @@ public class EncoderUnitTests : WafLibraryRequiredTest [InlineData(WafConstants.MaxStringLength + 1, WafConstants.MaxStringLength)] public void TestStringLength(int length, int expectedLength) { + var l = new List(); var target = new string('c', length); - using var intermediate = Encoder.Encode(target, applySafetyLimits: true); - var result = intermediate.Result.Decode() as string; + using var intermediate = Encoder.Encode(target, WafLibraryInvoker, l, applySafetyLimits: true); + var result = intermediate.InnerStruct.Decode() as string; Assert.NotNull(result); Assert.Equal(expectedLength, result.Length); + + Dispose(l); } [SkippableTheory] @@ -35,13 +38,17 @@ public void TestStringLength(int length, int expectedLength) [InlineData(WafConstants.MaxContainerSize + 1, WafConstants.MaxContainerSize)] public void TestArrayLength(int length, int expectedLength) { + var l = new List(); + var target = Enumerable.Repeat((object)"test", length).ToList(); - using var intermediate = Encoder.Encode(target, applySafetyLimits: true); - var result = intermediate.Result.Decode() as List; + using var intermediate = Encoder.Encode(target, WafLibraryInvoker, l, applySafetyLimits: true); + var result = intermediate.InnerStruct.Decode() as List; Assert.NotNull(result); Assert.Equal(expectedLength, result.Count); + + Dispose(l); } [SkippableTheory] @@ -50,13 +57,17 @@ public void TestArrayLength(int length, int expectedLength) [InlineData(WafConstants.MaxContainerSize + 1, WafConstants.MaxContainerSize)] public void TestMapLength(int length, int expectedLength) { + var l = new List(); + var target = Enumerable.Range(0, length).ToDictionary(x => x.ToString(), _ => (object)"test"); - using var intermediate = Encoder.Encode(target, applySafetyLimits: true); - var result = intermediate.Result.Decode() as Dictionary; + using var intermediate = Encoder.Encode(target, WafLibraryInvoker, l, applySafetyLimits: true); + var result = intermediate.InnerStruct.Decode() as Dictionary; Assert.NotNull(result); Assert.Equal(expectedLength, result.Count); + + Dispose(l); } [SkippableTheory] @@ -65,13 +76,17 @@ public void TestMapLength(int length, int expectedLength) [InlineData(WafConstants.MaxContainerDepth + 1, WafConstants.MaxContainerDepth)] public void TestNestedListDepth(int length, int expectedLength) { + var l = new List(); + var target = MakeNestedList(length); - using var intermediate = Encoder.Encode(target, applySafetyLimits: true); - var result = intermediate.Result.Decode() as List; + using var intermediate = Encoder.Encode(target, WafLibraryInvoker, l, applySafetyLimits: true); + var result = intermediate.InnerStruct.Decode() as List; Assert.NotNull(result); Assert.Equal(expectedLength, CountNestedListDepth(result)); + + Dispose(l); } [SkippableTheory] @@ -80,13 +95,25 @@ public void TestNestedListDepth(int length, int expectedLength) [InlineData(WafConstants.MaxContainerDepth + 1, WafConstants.MaxContainerDepth)] public void TestMapListDepth(int length, int expectedLength) { + var l = new List(); + var target = MakeNestedMap(length); - using var intermediate = Encoder.Encode(target, applySafetyLimits: true); - var result = intermediate.Result.Decode() as Dictionary; + using var intermediate = Encoder.Encode(target, WafLibraryInvoker, l, applySafetyLimits: true); + var result = intermediate.InnerStruct.Decode() as Dictionary; Assert.NotNull(result); Assert.Equal(expectedLength, CountNestedMapDepth(result)); + + Dispose(l); + } + + private static void Dispose(List l) + { + foreach (var obj in l) + { + obj.Dispose(); + } } private static List MakeNestedList(int nestingDepth) diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/FuzzEncoder.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/FuzzEncoder.cs index f8b38d385400..9be83ae07463 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/FuzzEncoder.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/FuzzEncoder.cs @@ -48,10 +48,13 @@ public void LetsFuzz() using var jsonReader = new JsonTextReader(streamReader); var root = JToken.ReadFrom(jsonReader); - using var result = Encoder.Encode(root, applySafetyLimits: true); + var l = new List(); + using var result = Encoder.Encode(root, WafLibraryInvoker!, l, applySafetyLimits: true); // check the object is valid - Assert.NotEqual(DDWAF_OBJ_TYPE.DDWAF_OBJ_INVALID, result.Result.Type); + Assert.NotEqual(ObjType.Invalid, result.ArgsType); + + l.ForEach(x => x.Dispose()); } catch (Exception ex) { diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/WafMemoryTests.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/WafMemoryTests.cs index e763ed02b6df..cb4aa3db0f6d 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/WafMemoryTests.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/WafMemoryTests.cs @@ -11,12 +11,14 @@ using Datadog.Trace.AppSec.Rcm; using Datadog.Trace.AppSec.Rcm.Models.Asm; using Datadog.Trace.AppSec.Waf; +using Datadog.Trace.AppSec.Waf.NativeBindings; using Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; using Datadog.Trace.Security.Unit.Tests.Utils; using Datadog.Trace.TestHelpers; using Datadog.Trace.Vendors.Newtonsoft.Json; using FluentAssertions; using Xunit; +using YamlDotNet.Core.Tokens; namespace Datadog.Trace.Security.Unit.Tests { @@ -41,8 +43,7 @@ public void InitMemoryLeakCheck() Execute(AddressesConstants.RequestBody, "/.adsensepostnottherenonobook", "security_scanner", "crs-913-120"); } - var current = GetMemory(true); - + var current = GetMemory(); current.Should().BeLessThanOrEqualTo(baseline + OverheadMargin); } @@ -66,7 +67,7 @@ public void RunMemoryLeakCheck() resultData.Rule.Id.Should().Be("crs-913-120"); } - var current = GetMemory(true); + var current = GetMemory(); current.Should().BeLessThanOrEqualTo(baseline + OverheadMargin); } @@ -80,7 +81,7 @@ public void UpdateMemoryLeakCheck() bool enabled = false; - for (int x = 0; x < 200; x++) + for (int x = 0; x < 1000; x++) { var ruleOverrides = new List(); var ruleOverride = new RuleOverride { Enabled = enabled, Id = "crs-913-120" }; @@ -96,21 +97,12 @@ public void UpdateMemoryLeakCheck() enabled = !enabled; } - var current = GetMemory(true); + var current = GetMemory(); current.Should().BeLessThanOrEqualTo(baseline + OverheadMargin); } - private long GetMemory(bool disposePool = false) + private long GetMemory() { - if (disposePool) - { - // Size of the unmanaged pool is already MaxBytesForMaxStringLength = (WafConstants.MaxStringLength * 4) + 1 * BlockSize = 1000 - // > ((4096 * 4) + 1) * 1000 - // > 16.385.000 - // so give more margin for execution - Encoder.Pool.Dispose(); - } - GC.Collect(); GC.WaitForPendingFinalizers(); GC.WaitForFullGCComplete(); diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/WafTests.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/WafTests.cs index d57e6a9de825..4b0e7c62d1c7 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/WafTests.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/WafTests.cs @@ -11,7 +11,6 @@ using Datadog.Trace.AppSec.Waf; using Datadog.Trace.AppSec.Waf.NativeBindings; using Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; -using Datadog.Trace.Configuration; using Datadog.Trace.Security.Unit.Tests.Utils; using Datadog.Trace.TestHelpers.FluentAssertionsExtensions.Json; using Datadog.Trace.Vendors.Newtonsoft.Json; diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/WafUpdateTests.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/WafUpdateTests.cs index a3d2ca727806..9a864b1d97e8 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/WafUpdateTests.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/WafUpdateTests.cs @@ -13,7 +13,6 @@ using Datadog.Trace.AppSec.Waf; using Datadog.Trace.AppSec.Waf.Initialization; using Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; -using Datadog.Trace.Configuration; using Datadog.Trace.Security.Unit.Tests.Utils; using Datadog.Trace.Vendors.Newtonsoft.Json; using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;