decode/flow: add exception policy stats counters

We will register stats counters for all policies, even though for now Suri only uses one possible configuration policy at a time. The idea is that this could change in the near future, so we want to have this ready. Task OISF#5816
jlucovsky · Apr 11, 2024 · 657419b · 657419b
1 parent c2c8cdb
commit 657419b
Show file tree

Hide file tree

Showing 11 changed files with 131 additions and 10 deletions.
diff --git a/etc/schema.json b/etc/schema.json
@@ -5043,6 +5043,11 @@
                         "memcap": {
                             "type": "integer"
                         },
+                        "memcap_exception_policy": {
+                            "description":
+                                    "How many times flow memcap exception policy was applied, and which one",
+                            "$ref": "#/$defs/exceptionPolicy"
+                        },
                         "memuse": {
                             "type": "integer"
                         },
@@ -5723,6 +5728,35 @@
                     ]
                 }
             }
+        },
+        "exceptionPolicy": {
+            "type": "object",
+            "properties": {
+                "drop_flow": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "drop_packet": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "pass_flow": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "pass_packet": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "bypass": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "reject": {
+                    "type": "integer",
+                    "minimum": 0
+                }
+            }
         }
     }
 }
diff --git a/src/counters.c b/src/counters.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2021 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free

diff --git a/src/decode.c b/src/decode.c
@@ -76,6 +76,32 @@ extern bool stats_stream_events;
 uint8_t decoder_max_layers = PKT_DEFAULT_MAX_DECODED_LAYERS;
 uint16_t packet_alert_max = PACKET_ALERT_MAX;
 
+/* Settings order as in the enum */
+// clang-format off
+ExceptionPolicyStatsSetts flow_memcap_eps_stats = {
+    .valid_settings_ids = {
+    /* EXCEPTION_POLICY_NOT_SET */      false,
+    /* EXCEPTION_POLICY_AUTO */         false,
+    /* EXCEPTION_POLICY_PASS_PACKET */  true,
+    /* EXCEPTION_POLICY_PASS_FLOW */    false,
+    /* EXCEPTION_POLICY_BYPASS_FLOW */  true,
+    /* EXCEPTION_POLICY_DROP_PACKET */  false,
+    /* EXCEPTION_POLICY_DROP_FLOW */    false,
+    /* EXCEPTION_POLICY_REJECT */       true,
+    },
+    .valid_settings_ips = {
+    /* EXCEPTION_POLICY_NOT_SET */      false,
+    /* EXCEPTION_POLICY_AUTO */         false,
+    /* EXCEPTION_POLICY_PASS_PACKET */  true,
+    /* EXCEPTION_POLICY_PASS_FLOW */    false,
+    /* EXCEPTION_POLICY_BYPASS_FLOW */  true,
+    /* EXCEPTION_POLICY_DROP_PACKET */  true,
+    /* EXCEPTION_POLICY_DROP_FLOW */    false,
+    /* EXCEPTION_POLICY_REJECT */       true,
+    },
+};
+// clang-format on
+
 /**
  * \brief Initialize PacketAlerts with dynamic alerts array size
  *
@@ -525,6 +551,14 @@ void DecodeUnregisterCounters(void)
     SCMutexUnlock(&g_counter_table_mutex);
 }
 
+static bool IsFlowMemcapExceptionPolicyStatsValid(enum ExceptionPolicy policy)
+{
+    if (EngineModeIsIPS()) {
+        return flow_memcap_eps_stats.valid_settings_ips[policy];
+    }
+    return flow_memcap_eps_stats.valid_settings_ids[policy];
+}
+
 void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
 {
     /* register counters */
@@ -572,6 +606,9 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
     dtv->counter_erspan = StatsRegisterMaxCounter("decoder.erspan", tv);
     dtv->counter_nsh = StatsRegisterMaxCounter("decoder.nsh", tv);
     dtv->counter_flow_memcap = StatsRegisterCounter("flow.memcap", tv);
+    ExceptionPolicySetStatsCounters(tv, &dtv->counter_flow_memcap_eps, &flow_memcap_eps_stats,
+            FlowGetMemcapExceptionPolicy(), "flow.memcap_exception_policy.",
+            IsFlowMemcapExceptionPolicyStatsValid);
 
     dtv->counter_tcp_active_sessions = StatsRegisterCounter("tcp.active_sessions", tv);
     dtv->counter_flow_total = StatsRegisterCounter("flow.total", tv);

diff --git a/src/decode.h b/src/decode.h
@@ -32,6 +32,7 @@
 #include "threadvars.h"
 #include "util-debug.h"
 #include "decode-events.h"
+#include "util-exception-policy-types.h"
 #ifdef PROFILING
 #include "flow-worker.h"
 #include "app-layer-protos.h"
@@ -740,6 +741,7 @@ typedef struct DecodeThreadVars_
     uint16_t counter_defrag_max_hit;
 
     uint16_t counter_flow_memcap;
+    ExceptionPolicyCounters counter_flow_memcap_eps;
 
     uint16_t counter_tcp_active_sessions;
     uint16_t counter_flow_total;

diff --git a/src/flow-hash.c b/src/flow-hash.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2023 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -641,9 +641,24 @@ static inline Flow *FlowSpareSync(ThreadVars *tv, FlowLookupStruct *fls,
     return f;
 }
 
-static inline void NoFlowHandleIPS(Packet *p)
+static void FlowExceptionPolicyStatsIncr(
+        ThreadVars *tv, FlowLookupStruct *fls, enum ExceptionPolicy policy)
+{
+#ifdef UNITTESTS
+    if (tv == NULL) {
+        return;
+    }
+#endif
+    uint16_t id = fls->dtv->counter_flow_memcap_eps.eps_id[policy];
+    if (likely(id > 0)) {
+        StatsIncr(tv, id);
+    }
+}
+
+static inline void NoFlowHandleIPS(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
 {
     ExceptionPolicyApply(p, flow_config.memcap_policy, PKT_DROP_REASON_FLOW_MEMCAP);
+    FlowExceptionPolicyStatsIncr(tv, fls, flow_config.memcap_policy);
 }
 
 /**
@@ -663,7 +678,7 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
     const bool emerg = ((SC_ATOMIC_GET(flow_flags) & FLOW_EMERGENCY) != 0);
 #ifdef DEBUG
     if (g_eps_flow_memcap != UINT64_MAX && g_eps_flow_memcap == p->pcap_cnt) {
-        NoFlowHandleIPS(p);
+        NoFlowHandleIPS(tv, fls, p);
         StatsIncr(tv, fls->dtv->counter_flow_memcap);
         return NULL;
     }
@@ -689,7 +704,14 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
 
             f = FlowGetUsedFlow(tv, fls->dtv, p->ts);
             if (f == NULL) {
-                NoFlowHandleIPS(p);
+                NoFlowHandleIPS(tv, fls, p);
+#ifdef UNITTESTS
+                if (tv != NULL && fls->dtv != NULL) {
+#endif
+                    StatsIncr(tv, fls->dtv->counter_flow_memcap);
+#ifdef UNITTESTS
+                }
+#endif
                 return NULL;
             }
 #ifdef UNITTESTS
@@ -714,7 +736,7 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
 #ifdef UNITTESTS
             }
 #endif
-            NoFlowHandleIPS(p);
+            NoFlowHandleIPS(tv, fls, p);
             return NULL;
         }
 

diff --git a/src/flow.c b/src/flow.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -143,6 +143,11 @@ uint64_t FlowGetMemuse(void)
     return memusecopy;
 }
 
+enum ExceptionPolicy FlowGetMemcapExceptionPolicy(void)
+{
+    return flow_config.memcap_policy;
+}
+
 void FlowCleanupAppLayer(Flow *f)
 {
     if (f == NULL || f->proto == 0)

diff --git a/src/flow.h b/src/flow.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -30,6 +30,7 @@ typedef struct FlowStorageId FlowStorageId;
 #include "decode.h"
 #include "util-time.h"
 #include "util-exception-policy.h"
+#include "util-exception-policy-types.h"
 #include "util-var.h"
 #include "util-optimize.h"
 #include "app-layer-protos.h"
@@ -576,6 +577,7 @@ void FlowUpdateState(Flow *f, enum FlowState s);
 int FlowSetMemcap(uint64_t size);
 uint64_t FlowGetMemcap(void);
 uint64_t FlowGetMemuse(void);
+enum ExceptionPolicy FlowGetMemcapExceptionPolicy(void);
 
 FlowStorageId GetFlowBypassInfoID(void);
 void RegisterFlowBypassInfo(void);

diff --git a/src/output-json-stats.c b/src/output-json-stats.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2014-2020 Open Information Security Foundation
+/* Copyright (C) 2014-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free

diff --git a/src/util-exception-policy-types.h b/src/util-exception-policy-types.h
@@ -38,7 +38,7 @@ enum ExceptionPolicy {
 /* Max length = possible exception policy scenarios + counter names
  * + exception policy type. E.g.:
  * "tcp.reassembly_exception_policy.drop_packet" + 1 */
-#define EXCEPTION_POLICY_COUNTER_MAX_LEN 44
+#define EXCEPTION_POLICY_COUNTER_MAX_LEN 45
 
 typedef struct ExceptionPolicyCounters_ {
     /* Follows enum order */

diff --git a/src/util-exception-policy.c b/src/util-exception-policy.c
@@ -295,6 +295,22 @@ enum ExceptionPolicy ExceptionPolicyMidstreamParse(bool midstream_enabled)
     return policy;
 }
 
+void ExceptionPolicySetStatsCounters(ThreadVars *tv, ExceptionPolicyCounters *counter,
+        ExceptionPolicyStatsSetts *setting, enum ExceptionPolicy conf_policy,
+        const char *default_str, bool (*isExceptionPolicyValid)(enum ExceptionPolicy))
+{
+    if (conf_policy != EXCEPTION_POLICY_NOT_SET) {
+        /* set-up policy counters */
+        for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
+            if (isExceptionPolicyValid(i)) {
+                snprintf(setting->eps_name[i], sizeof(setting->eps_name[i]), "%s%s", default_str,
+                        ExceptionPolicyEnumToString(i, true));
+                counter->eps_id[i] = StatsRegisterCounter(setting->eps_name[i], tv);
+            }
+        }
+    }
+}
+
 #ifndef DEBUG
 
 int ExceptionSimulationCommandLineParser(const char *name, const char *arg)

diff --git a/src/util-exception-policy.h b/src/util-exception-policy.h
@@ -31,6 +31,9 @@ void ExceptionPolicyApply(
         Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason);
 enum ExceptionPolicy ExceptionPolicyParse(const char *option, const bool support_flow);
 enum ExceptionPolicy ExceptionPolicyMidstreamParse(bool midstream_enabled);
+void ExceptionPolicySetStatsCounters(ThreadVars *tv, ExceptionPolicyCounters *counter,
+        ExceptionPolicyStatsSetts *setting, enum ExceptionPolicy conf_policy,
+        const char *default_str, bool (*isExceptionPolicyValid)(enum ExceptionPolicy));
 
 extern enum ExceptionPolicy g_eps_master_switch;
 #ifdef DEBUG