From 1968b9f834e2b17fe57e55547d002a2813760a21 Mon Sep 17 00:00:00 2001 From: Juliana Fajardini Date: Tue, 11 Apr 2023 15:36:48 -0300 Subject: [PATCH] decode/flow: add exception policy stats counters We will register stats counters for all policies, even though for now Suri only uses one possible configuration policy at a time. The idea is that this could change in the near future, so we want to have this ready. Task #5816 (cherry picked from commit 657419b53e5e2d79f45efc2e1f87aabdc3b2cbee) --- etc/schema.json | 34 ++++++++++++++++++++++++++++ src/counters.c | 2 +- src/decode.c | 37 +++++++++++++++++++++++++++++++ src/decode.h | 2 ++ src/flow-hash.c | 32 +++++++++++++++++++++----- src/flow.c | 7 +++++- src/flow.h | 4 +++- src/output-json-stats.c | 2 +- src/util-exception-policy-types.h | 2 +- src/util-exception-policy.c | 16 +++++++++++++ src/util-exception-policy.h | 3 +++ 11 files changed, 131 insertions(+), 10 deletions(-) diff --git a/etc/schema.json b/etc/schema.json index e01c6828c3c8..86e8b92aea18 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -4913,6 +4913,11 @@ "memcap": { "type": "integer" }, + "memcap_exception_policy": { + "description": + "How many times flow memcap exception policy was applied, and which one", + "$ref": "#/$defs/exceptionPolicy" + }, "memuse": { "type": "integer" }, @@ -5540,6 +5545,35 @@ ] } } + }, + "exceptionPolicy": { + "type": "object", + "properties": { + "drop_flow": { + "type": "integer", + "minimum": 0 + }, + "drop_packet": { + "type": "integer", + "minimum": 0 + }, + "pass_flow": { + "type": "integer", + "minimum": 0 + }, + "pass_packet": { + "type": "integer", + "minimum": 0 + }, + "bypass": { + "type": "integer", + "minimum": 0 + }, + "reject": { + "type": "integer", + "minimum": 0 + } + } } } } diff --git a/src/counters.c b/src/counters.c index e3ed5244cdfe..216a0bb252d6 100644 --- a/src/counters.c +++ b/src/counters.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2021 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free diff --git a/src/decode.c b/src/decode.c index 5cdeeead6b96..ce18bd3bedb3 100644 --- a/src/decode.c +++ b/src/decode.c @@ -76,6 +76,32 @@ extern bool stats_stream_events; uint8_t decoder_max_layers = PKT_DEFAULT_MAX_DECODED_LAYERS; uint16_t packet_alert_max = PACKET_ALERT_MAX; +/* Settings order as in the enum */ +// clang-format off +ExceptionPolicyStatsSetts flow_memcap_eps_stats = { + .valid_settings_ids = { + /* EXCEPTION_POLICY_NOT_SET */ false, + /* EXCEPTION_POLICY_AUTO */ false, + /* EXCEPTION_POLICY_PASS_PACKET */ true, + /* EXCEPTION_POLICY_PASS_FLOW */ false, + /* EXCEPTION_POLICY_BYPASS_FLOW */ true, + /* EXCEPTION_POLICY_DROP_PACKET */ false, + /* EXCEPTION_POLICY_DROP_FLOW */ false, + /* EXCEPTION_POLICY_REJECT */ true, + }, + .valid_settings_ips = { + /* EXCEPTION_POLICY_NOT_SET */ false, + /* EXCEPTION_POLICY_AUTO */ false, + /* EXCEPTION_POLICY_PASS_PACKET */ true, + /* EXCEPTION_POLICY_PASS_FLOW */ false, + /* EXCEPTION_POLICY_BYPASS_FLOW */ true, + /* EXCEPTION_POLICY_DROP_PACKET */ true, + /* EXCEPTION_POLICY_DROP_FLOW */ false, + /* EXCEPTION_POLICY_REJECT */ true, + }, +}; +// clang-format on + /** * \brief Initialize PacketAlerts with dynamic alerts array size * @@ -522,6 +548,14 @@ void DecodeUnregisterCounters(void) SCMutexUnlock(&g_counter_table_mutex); } +static bool IsFlowMemcapExceptionPolicyStatsValid(enum ExceptionPolicy policy) +{ + if (EngineModeIsIPS()) { + return flow_memcap_eps_stats.valid_settings_ips[policy]; + } + return flow_memcap_eps_stats.valid_settings_ids[policy]; +} + void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv) { /* register counters */ @@ -569,6 +603,9 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv) dtv->counter_erspan = StatsRegisterMaxCounter("decoder.erspan", tv); dtv->counter_nsh = StatsRegisterMaxCounter("decoder.nsh", tv); dtv->counter_flow_memcap = StatsRegisterCounter("flow.memcap", tv); + ExceptionPolicySetStatsCounters(tv, &dtv->counter_flow_memcap_eps, &flow_memcap_eps_stats, + FlowGetMemcapExceptionPolicy(), "flow.memcap_exception_policy.", + IsFlowMemcapExceptionPolicyStatsValid); dtv->counter_tcp_active_sessions = StatsRegisterCounter("tcp.active_sessions", tv); dtv->counter_flow_total = StatsRegisterCounter("flow.total", tv); diff --git a/src/decode.h b/src/decode.h index dedfbb09efd0..0b4527f5aefc 100644 --- a/src/decode.h +++ b/src/decode.h @@ -32,6 +32,7 @@ #include "threadvars.h" #include "util-debug.h" #include "decode-events.h" +#include "util-exception-policy-types.h" #ifdef PROFILING #include "flow-worker.h" #include "app-layer-protos.h" @@ -723,6 +724,7 @@ typedef struct DecodeThreadVars_ uint16_t counter_defrag_max_hit; uint16_t counter_flow_memcap; + ExceptionPolicyCounters counter_flow_memcap_eps; uint16_t counter_tcp_active_sessions; uint16_t counter_flow_total; diff --git a/src/flow-hash.c b/src/flow-hash.c index 3b221e2ffffc..57e3f9915817 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2023 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -641,9 +641,24 @@ static inline Flow *FlowSpareSync(ThreadVars *tv, FlowLookupStruct *fls, return f; } -static inline void NoFlowHandleIPS(Packet *p) +static void FlowExceptionPolicyStatsIncr( + ThreadVars *tv, FlowLookupStruct *fls, enum ExceptionPolicy policy) +{ +#ifdef UNITTESTS + if (tv == NULL) { + return; + } +#endif + uint16_t id = fls->dtv->counter_flow_memcap_eps.eps_id[policy]; + if (likely(id > 0)) { + StatsIncr(tv, id); + } +} + +static inline void NoFlowHandleIPS(ThreadVars *tv, FlowLookupStruct *fls, Packet *p) { ExceptionPolicyApply(p, flow_config.memcap_policy, PKT_DROP_REASON_FLOW_MEMCAP); + FlowExceptionPolicyStatsIncr(tv, fls, flow_config.memcap_policy); } /** @@ -663,7 +678,7 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p) const bool emerg = ((SC_ATOMIC_GET(flow_flags) & FLOW_EMERGENCY) != 0); #ifdef DEBUG if (g_eps_flow_memcap != UINT64_MAX && g_eps_flow_memcap == p->pcap_cnt) { - NoFlowHandleIPS(p); + NoFlowHandleIPS(tv, fls, p); StatsIncr(tv, fls->dtv->counter_flow_memcap); return NULL; } @@ -689,7 +704,14 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p) f = FlowGetUsedFlow(tv, fls->dtv, p->ts); if (f == NULL) { - NoFlowHandleIPS(p); + NoFlowHandleIPS(tv, fls, p); +#ifdef UNITTESTS + if (tv != NULL && fls->dtv != NULL) { +#endif + StatsIncr(tv, fls->dtv->counter_flow_memcap); +#ifdef UNITTESTS + } +#endif return NULL; } #ifdef UNITTESTS @@ -714,7 +736,7 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p) #ifdef UNITTESTS } #endif - NoFlowHandleIPS(p); + NoFlowHandleIPS(tv, fls, p); return NULL; } diff --git a/src/flow.c b/src/flow.c index b61823efd0ee..b833cb1376fc 100644 --- a/src/flow.c +++ b/src/flow.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -143,6 +143,11 @@ uint64_t FlowGetMemuse(void) return memusecopy; } +enum ExceptionPolicy FlowGetMemcapExceptionPolicy(void) +{ + return flow_config.memcap_policy; +} + void FlowCleanupAppLayer(Flow *f) { if (f == NULL || f->proto == 0) diff --git a/src/flow.h b/src/flow.h index 9866b568560b..45c624a34cd1 100644 --- a/src/flow.h +++ b/src/flow.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -30,6 +30,7 @@ typedef struct FlowStorageId FlowStorageId; #include "decode.h" #include "util-time.h" #include "util-exception-policy.h" +#include "util-exception-policy-types.h" #include "util-var.h" #include "util-optimize.h" #include "app-layer-protos.h" @@ -572,6 +573,7 @@ void FlowUpdateState(Flow *f, enum FlowState s); int FlowSetMemcap(uint64_t size); uint64_t FlowGetMemcap(void); uint64_t FlowGetMemuse(void); +enum ExceptionPolicy FlowGetMemcapExceptionPolicy(void); FlowStorageId GetFlowBypassInfoID(void); void RegisterFlowBypassInfo(void); diff --git a/src/output-json-stats.c b/src/output-json-stats.c index 33f98afa2dda..233f158fb339 100644 --- a/src/output-json-stats.c +++ b/src/output-json-stats.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2014-2020 Open Information Security Foundation +/* Copyright (C) 2014-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free diff --git a/src/util-exception-policy-types.h b/src/util-exception-policy-types.h index 04e6d3bca4e4..b5295d19305b 100644 --- a/src/util-exception-policy-types.h +++ b/src/util-exception-policy-types.h @@ -38,7 +38,7 @@ enum ExceptionPolicy { /* Max length = possible exception policy scenarios + counter names * + exception policy type. E.g.: * "tcp.reassembly_exception_policy.drop_packet" + 1 */ -#define EXCEPTION_POLICY_COUNTER_MAX_LEN 44 +#define EXCEPTION_POLICY_COUNTER_MAX_LEN 45 typedef struct ExceptionPolicyCounters_ { /* Follows enum order */ diff --git a/src/util-exception-policy.c b/src/util-exception-policy.c index 287ec64c490a..d3b7b0bd9456 100644 --- a/src/util-exception-policy.c +++ b/src/util-exception-policy.c @@ -294,6 +294,22 @@ enum ExceptionPolicy ExceptionPolicyMidstreamParse(bool midstream_enabled) return policy; } +void ExceptionPolicySetStatsCounters(ThreadVars *tv, ExceptionPolicyCounters *counter, + ExceptionPolicyStatsSetts *setting, enum ExceptionPolicy conf_policy, + const char *default_str, bool (*isExceptionPolicyValid)(enum ExceptionPolicy)) +{ + if (conf_policy != EXCEPTION_POLICY_NOT_SET) { + /* set-up policy counters */ + for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) { + if (isExceptionPolicyValid(i)) { + snprintf(setting->eps_name[i], sizeof(setting->eps_name[i]), "%s%s", default_str, + ExceptionPolicyEnumToString(i, true)); + counter->eps_id[i] = StatsRegisterCounter(setting->eps_name[i], tv); + } + } + } +} + #ifndef DEBUG int ExceptionSimulationCommandLineParser(const char *name, const char *arg) diff --git a/src/util-exception-policy.h b/src/util-exception-policy.h index f4a152a7e29b..0c0be9fce1ab 100644 --- a/src/util-exception-policy.h +++ b/src/util-exception-policy.h @@ -31,6 +31,9 @@ void ExceptionPolicyApply( Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason); enum ExceptionPolicy ExceptionPolicyParse(const char *option, const bool support_flow); enum ExceptionPolicy ExceptionPolicyMidstreamParse(bool midstream_enabled); +void ExceptionPolicySetStatsCounters(ThreadVars *tv, ExceptionPolicyCounters *counter, + ExceptionPolicyStatsSetts *setting, enum ExceptionPolicy conf_policy, + const char *default_str, bool (*isExceptionPolicyValid)(enum ExceptionPolicy)); extern enum ExceptionPolicy g_eps_master_switch; #ifdef DEBUG