From f93ed0a50650adad9a5c1aeb64afeaf3a8325b91 Mon Sep 17 00:00:00 2001 From: NachoEchevarria <53266532+NachoEchevarria@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:42:03 +0200 Subject: [PATCH] [ASM] Suspicious attacker blocking (#6057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary of changes This PR adds the required code to support the suspicious attack functionality. It has added the required code to read the exclusion data from the RC. It also has modified the existing code regarding RC actions. Previously, if a configuration was received with an action, the configuration was stored and later sent to the WAF. If new values with an empty action array would come later, the previous action configurations would be deleted, but if a new array would come with a new action different than the previous one, we would report them both to the WAF. This behavior was making the suspicious attacker system tests fail because we would keep RC changes from previous tests. This change seems to be aligned with the behavior of other libraries. The file [AspNetBase.cs](https://github.com/DataDog/dd-trace-dotnet/pull/6057/files#diff-0faff2451113067d7669566ba9199908b720a3764914b00d6f33d4b376098d74) has been updated. Now, tests have more control over the used headers by allowing them to remove headers or replace previous values with new ones. ## Reason for change ## Implementation details ## Test coverage ## Other details --- .../AppSec/Rcm/AsmDataProduct.cs | 24 ++- .../AppSec/Rcm/ConfigurationStatus.cs | 9 ++ .../AppSec/Rcm/Models/AsmData/Payload.cs | 5 +- .../src/Datadog.Trace/AppSec/Waf/Context.cs | 3 +- tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs | 7 + .../AspNetBase.cs | 10 +- .../Rcm/AspNetCore5AsmAttackerBlocking.cs | 138 ++++++++++++++++++ 7 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmAttackerBlocking.cs diff --git a/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDataProduct.cs b/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDataProduct.cs index 66870e1bd7d9..20d737e8e4f6 100644 --- a/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDataProduct.cs +++ b/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDataProduct.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -18,26 +18,40 @@ public void ProcessUpdates(ConfigurationStatus configurationStatus, List(); - var rulesData = asmDataConfig.TypedFile!.RulesData; + var rulesData = asmDataConfig.TypedFile?.RulesData; if (rulesData != null) { configurationStatus.RulesDataByFile[rawFile.Path.Path] = rulesData; configurationStatus.IncomingUpdateState.WafKeysToApply.Add(ConfigurationStatus.WafRulesDataKey); } + + var exclusionsData = asmDataConfig.TypedFile?.ExclusionsData; + if (exclusionsData != null) + { + configurationStatus.ExclusionsDataByFile[rawFile.Path.Path] = exclusionsData; + configurationStatus.IncomingUpdateState.WafKeysToApply.Add(ConfigurationStatus.WafExclusionsDataKey); + } } } public void ProcessRemovals(ConfigurationStatus configurationStatus, List removedConfigsForThisProduct) { - var removedData = false; + var removedRulesData = false; + var removedExclusionsData = false; foreach (var configurationPath in removedConfigsForThisProduct) { - removedData |= configurationStatus.RulesDataByFile.Remove(configurationPath.Path); + removedRulesData |= configurationStatus.RulesDataByFile.Remove(configurationPath.Path); + removedExclusionsData |= configurationStatus.ExclusionsDataByFile.Remove(configurationPath.Path); } - if (removedData) + if (removedRulesData) { configurationStatus.IncomingUpdateState.WafKeysToApply.Add(ConfigurationStatus.WafRulesDataKey); } + + if (removedExclusionsData) + { + configurationStatus.IncomingUpdateState.WafKeysToApply.Add(ConfigurationStatus.WafExclusionsDataKey); + } } } diff --git a/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs b/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs index 78cd40181b5e..2e4ce01d72f2 100644 --- a/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs +++ b/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs @@ -35,6 +35,7 @@ internal record ConfigurationStatus internal const string WafRulesOverridesKey = "rules_override"; internal const string WafExclusionsKey = "exclusions"; internal const string WafRulesDataKey = "rules_data"; + internal const string WafExclusionsDataKey = "exclusion_data"; internal const string WafCustomRulesKey = "custom_rules"; internal const string WafActionsKey = "actions"; private readonly IAsmConfigUpdater _asmFeatureProduct = new AsmFeaturesProduct(); @@ -57,6 +58,8 @@ internal record ConfigurationStatus internal Dictionary RulesDataByFile { get; } = new(); + internal Dictionary ExclusionsDataByFile { get; } = new(); + internal Dictionary ExclusionsByFile { get; } = new(); internal Dictionary RulesByFile { get; } = new(); @@ -123,6 +126,12 @@ internal Dictionary BuildDictionaryForWafAccordingToIncomingUpda dictionary.Add(WafRulesDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray()); } + if (IncomingUpdateState.WafKeysToApply.Contains(WafExclusionsDataKey)) + { + var rulesData = MergeRuleData(ExclusionsDataByFile.SelectMany(x => x.Value)); + dictionary.Add(WafExclusionsDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray()); + } + if (IncomingUpdateState.WafKeysToApply.Contains(WafActionsKey)) { var actions = ActionsByFile.SelectMany(x => x.Value).ToList(); diff --git a/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmData/Payload.cs b/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmData/Payload.cs index 21c8aabf0c72..79b738d87a1b 100644 --- a/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmData/Payload.cs +++ b/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmData/Payload.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -11,4 +11,7 @@ internal class Payload { [JsonProperty("rules_data")] public RuleData[]? RulesData { get; set; } + + [JsonProperty("exclusion_data")] + public RuleData[]? ExclusionsData { get; set; } } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs index 2a72d96b891d..ad6c93eb12ee 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Context.cs @@ -140,8 +140,9 @@ private Context(IntPtr contextHandle, Waf waf, WafLibraryInvoker wafLibraryInvok if (Log.IsEnabled(LogEventLevel.Debug)) { Log.Debug( - "DDAS-0011-00: AppSec In-App WAF returned: {ReturnCode} {Data}", + "DDAS-0011-00: AppSec In-App WAF returned: {ReturnCode} {BlockInfo} {Data}", result.ReturnCode, + result.BlockInfo, result.Data); } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs index 45ba8519b81f..1f0be2ffb0f6 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs @@ -18,6 +18,8 @@ using Datadog.Trace.ExtensionMethods; using Datadog.Trace.Logging; using Datadog.Trace.Telemetry; +using Datadog.Trace.Vendors.Newtonsoft.Json; +using Datadog.Trace.Vendors.Serilog.Events; namespace Datadog.Trace.AppSec.Waf { @@ -218,6 +220,11 @@ private UpdateResult Update(IDictionary arguments) UpdateResult updated; try { + if (Log.IsEnabled(LogEventLevel.Debug)) + { + Log.Debug("Updating WAF with new configuration: {Arguments}", JsonConvert.SerializeObject(arguments)); + } + var encodedArgs = _encoder.Encode(arguments, applySafetyLimits: false); updated = UpdateWafAndDispose(encodedArgs); diff --git a/tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetBase.cs b/tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetBase.cs index 7237367bece8..5d2112756a28 100644 --- a/tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetBase.cs +++ b/tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetBase.cs @@ -372,7 +372,15 @@ protected async Task TestRateLimiter(bool enableSecurity, string url, MockTracer { foreach (var header in headers) { - _httpClient.DefaultRequestHeaders.Add(header.Key, header.Value); + if (_httpClient.DefaultRequestHeaders.Contains(header.Key)) + { + _httpClient.DefaultRequestHeaders.Remove(header.Key); + } + + if (header.Value is not null) + { + _httpClient.DefaultRequestHeaders.Add(header.Key, header.Value); + } } } diff --git a/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmAttackerBlocking.cs b/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmAttackerBlocking.cs new file mode 100644 index 000000000000..d7e820a7c089 --- /dev/null +++ b/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmAttackerBlocking.cs @@ -0,0 +1,138 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NETCOREAPP3_0_OR_GREATER + +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading.Tasks; +using Datadog.Trace.AppSec; +using Datadog.Trace.AppSec.Rcm.Models.AsmFeatures; +using Datadog.Trace.Configuration; +using Datadog.Trace.TestHelpers; +using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; +using Action = Datadog.Trace.AppSec.Rcm.Models.Asm.Action; + +namespace Datadog.Trace.Security.IntegrationTests.Rcm; + +public class AspNetCore5AsmAttackerBlocking : RcmBase +{ + private const string AsmProduct = "ASM"; + + public AspNetCore5AsmAttackerBlocking(AspNetCoreTestFixture fixture, ITestOutputHelper outputHelper) + : base(fixture, outputHelper, enableSecurity: true, testName: nameof(AspNetCore5AsmAttackerBlocking)) + { + SetEnvironmentVariable(ConfigurationKeys.DebugEnabled, "0"); + SetEnvironmentVariable("DD_APPSEC_WAF_DEBUG", "0"); + } + + [Fact] + [Trait("RunOnWindows", "True")] + public async Task TestSuspiciousAttackerBlocking() + { + List> headersAttacker = new() + { + new KeyValuePair("http.client_ip", "34.65.27.85"), + new KeyValuePair("X-Real-Ip", "34.65.27.85"), + new KeyValuePair("accept-encoding", "identity"), + new KeyValuePair("x-forwarded-for", null), + }; + + List> headersRegular = new() + { + new KeyValuePair("X-Real-Ip", null), + new KeyValuePair("accept-encoding", "identity"), + new KeyValuePair("x-forwarded-for", null), + }; + + var headersAttackerArachni = new List>(headersAttacker) + { + new KeyValuePair("User-Agent", "Arachni/v1"), + }; + + var headersRegularArachni = new List>(headersRegular) + { + new KeyValuePair("User-Agent", "Arachni/v1"), + }; + + var headersAttackerScanner = new List>(headersAttacker) + { + new KeyValuePair("User-Agent", "dd-test-scanner-log-block"), + }; + + var headersRegularScanner = new List>(headersRegular) + { + new KeyValuePair("User-Agent", "dd-test-scanner-log-block"), + }; + + string url = "/Health"; + IncludeAllHttpSpans = true; + await TryStartApp(); + var agent = Fixture.Agent; + var result = SubmitRequest(url, null, null, headers: headersAttackerScanner); + result.Result.StatusCode.Should().Be(HttpStatusCode.Forbidden); + + var configurationInitial = new[] + { + ((object)new AppSec.Rcm.Models.Asm.Payload + { + Actions = new[] + { + new Action { Id = "block", Type = BlockingAction.BlockRequestType, Parameters = new AppSec.Rcm.Models.Asm.Parameter { StatusCode = 403, Type = "json" } } + }, + }, + AsmProduct, + nameof(TestSuspiciousAttackerBlocking)), + ((object)new AsmFeatures + { + Asm = new AsmFeature { Enabled = true }, + }, + "ASM_FEATURES", + nameof(TestSuspiciousAttackerBlocking)) + }; + + await agent.SetupRcmAndWait(Output, configurationInitial); + result = SubmitRequest(url, null, null, headers: headersAttackerScanner); + + var exclusions = "[{\"id\": \"exc-000-001\",\"on_match\": \"block_custom\",\"conditions\": [{\"operator\": \"ip_match\",\"parameters\": {\"data\": \"suspicious_ips_data_id\", \"inputs\": [{\"address\": \"http.client_ip\"}]}}]}]"; + var configuration = new[] + { + (new AppSec.Rcm.Models.Asm.Payload + { + Actions = new[] + { + new Action { Id = "block_custom", Type = BlockingAction.BlockRequestType, Parameters = new AppSec.Rcm.Models.Asm.Parameter { StatusCode = 405, Type = "auto" } } + }, + Exclusions = (JArray)JToken.Parse(exclusions) + }, + AsmProduct, + nameof(TestSuspiciousAttackerBlocking)), + ((object)new AppSec.Rcm.Models.AsmData.Payload + { + ExclusionsData = new[] + { + new AppSec.Rcm.Models.AsmData.RuleData { Id = "suspicious_ips_data_id", Type = "ip_with_expiration", Data = new AppSec.Rcm.Models.AsmData.Data[] { new() { Value = "34.65.27.85" } } } + }, + }, + "ASM_DATA", + nameof(TestSuspiciousAttackerBlocking)), + }; + + await agent.SetupRcmAndWait(Output, configuration); + result = SubmitRequest(url + "?a=3", null, null, headers: headersAttackerScanner); + result.Result.StatusCode.Should().Be(HttpStatusCode.MethodNotAllowed); + result = SubmitRequest(url + "?a=4", null, null, headers: headersRegularScanner); + result.Result.StatusCode.Should().Be(HttpStatusCode.Forbidden); + result = SubmitRequest(url + "?a=5", null, null, headers: headersAttackerArachni); + result.Result.StatusCode.Should().Be(HttpStatusCode.MethodNotAllowed); + result = SubmitRequest(url + "?a=6", null, null, headers: headersRegularArachni); + result.Result.StatusCode.Should().Be(HttpStatusCode.OK); + } +} +#endif