detect: absent keyword to test absence of sticky buffer

doc/userguide/rules/payload-keywords.rst

@@ -274,6 +274,27 @@ You can also use the negation (!) before isdataat.
 
 .. image:: payload-keywords/isdataat1.png
 
+absent
+------
+
+The keyword ``absent`` checks that a sticky buffer does not exist.
+It can take an argument "only" to match only on absent buffer :
+
+Example of ``absent`` in a rule:
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  http.referer; absent:only; sid:1; rev:1;)
+
+
+It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance :
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  http.referer; absent: or_else; content: !"abc"; sid:1; rev:1;)
+
+An absent keyword cannot work with positive (non negated) content or pcre.
+
 bsize
 -----
 

src/detect-engine-content-inspection.c

@@ -382,6 +382,14 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                     prev_offset);
         } while(1);
 
+    } else if (smd->type == DETECT_ABSENT) {
+        const DetectAbsentData *id = (DetectAbsentData *)smd->ctx;
+        if (!id->or_else) {
+            // we match only on absent buffer
+            goto no_match;
+        } else {
+            goto match;
+        }
     } else if (smd->type == DETECT_ISDATAAT) {
         SCLogDebug("inspecting isdataat");
 
@@ -648,8 +656,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
             goto match;
         }
         goto no_match_discontinue;
-    }
-    else if (smd->type == DETECT_LUA) {
+    } else if (smd->type == DETECT_LUA) {
         SCLogDebug("lua starting");
 
         if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len,
@@ -760,6 +767,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         return false;
 }
 
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd)
+{
+    // we will match on NULL buffers there is one absent
+    bool absent_data = false;
+    while (1) {
+        if (smd->type == DETECT_ABSENT) {
+            absent_data = true;
+            break;
+        }
+        if (smd->is_last) {
+            break;
+        }
+        // smd does not get reused after this loop
+        smd++;
+    }
+    return absent_data;
+}
+
 #ifdef UNITTESTS
 #include "tests/detect-engine-content-inspection.c"
 #endif

src/detect-engine-content-inspection.h

@@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b,
         const enum DetectContentInspectionType inspection_mode);
 
+/** \brief tells if we should match on absent buffer, because
+ *  all smd entries are negated
+ *  \param smd array of content inspection matches
+ *  \retval bool true to match on absent buffer, false otherwise */
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd);
+
 void DetectEngineContentInspectionRegisterTests(void);
 
 #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */

src/detect-engine-file.c

@@ -192,6 +192,11 @@ uint8_t DetectFileInspectGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx
     SCLogDebug("tx %p tx_id %" PRIu64 " ffc %p ffc->head %p sid %u", tx, tx_id, ffc,
             ffc ? ffc->head : NULL, s->id);
     if (ffc == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, tx, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         SCReturnInt(DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES);
     } else if (ffc->head == NULL) {
         SCReturnInt(DETECT_ENGINE_INSPECT_SIG_NO_MATCH);

src/detect-engine-frame.c

@@ -293,8 +293,12 @@ static int DetectFrameInspectUdp(DetectEngineThreadCtx *det_ctx,
 
     // TODO can we use single here? Could it conflict with TCP?
     InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, 0);
-    if (buffer == NULL)
+    if (buffer == NULL) {
+        if (engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
 
     DEBUG_VALIDATE_BUG_ON(frame->offset >= p->payload_len);
     if (frame->offset >= p->payload_len)
@@ -434,6 +438,9 @@ static int FrameStreamDataInspectFunc(
     InspectionBuffer *buffer =
             InspectionBufferMultipleForListGet(fsd->det_ctx, fsd->list_id, fsd->idx++);
     if (buffer == NULL) {
+        if (fsd->inspect_engine->match_on_null && fsd->idx == 0) {
+            fsd->inspect_result = DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return 0;
     }
     SCLogDebug("buffer %p idx %u", buffer, fsd->idx);

src/detect-engine-register.h

@@ -80,6 +80,7 @@ enum DetectKeywordId {
     DETECT_IPPROTO,
     DETECT_FTPBOUNCE,
     DETECT_ISDATAAT,
+    DETECT_ABSENT,
     DETECT_ID,
     DETECT_RPC,
     DETECT_FLOWVAR,

src/detect-engine.c

@@ -564,6 +564,7 @@ static void AppendFrameInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->sm_list = u->sm_list;
     new_engine->sm_list_base = u->sm_list_base;
     new_engine->smd = smd;
+    new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd);
     new_engine->v1 = u->v1;
     SCLogDebug("sm_list %d new_engine->v1 %p/%p", new_engine->sm_list, new_engine->v1.Callback,
             new_engine->v1.transforms);
@@ -659,6 +660,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->sm_list = t->sm_list;
     new_engine->sm_list_base = t->sm_list_base;
     new_engine->smd = smd;
+    new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd);
     new_engine->progress = t->progress;
     new_engine->v2 = t->v2;
     SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
@@ -2154,6 +2156,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh
     const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
             f, flags, txv, list_id);
     if (unlikely(buffer == NULL)) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH :
                      DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
@@ -2215,6 +2220,13 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
         }
         local_id++;
     } while (1);
+    if (local_id == 0) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+    }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }
 

src/detect-http-client-body.c

@@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
     const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback(
             det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
     if (buffer == NULL || buffer->inspect == NULL) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
 

src/detect-http-header.c

@@ -176,6 +176,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
 
     const int list_id = engine->sm_list;
     InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    bool eof =
+            (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
     if (buffer->inspect == NULL) {
         SCLogDebug("setting up inspect buffer %d", list_id);
 
@@ -189,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
         uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len);
         if (rawdata_len == 0) {
             SCLogDebug("no data");
+            if (engine->match_on_null && eof) {
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
+            }
             goto end;
         }
         /* setup buffer and apply transforms */
@@ -209,14 +214,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
     }
 end:
-    if (flags & STREAM_TOSERVER) {
-        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
-                HTP_REQUEST_HEADERS)
-            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
-    } else {
-        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
-                HTP_RESPONSE_HEADERS)
-            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+    if (eof) {
+        return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }

src/detect-isdataat.c

@@ -35,6 +35,7 @@
 
 #include "detect-isdataat.h"
 #include "detect-content.h"
+#include "detect-bytetest.h"
 #include "detect-uricontent.h"
 #include "detect-engine-build.h"
 
@@ -61,6 +62,97 @@ void DetectIsdataatFree(DetectEngineCtx *, void *);
 
 static int DetectEndsWithSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr);
 
+static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr)
+{
+    SCFree(ptr);
+}
+
+static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr)
+{
+    if (s->init_data->list == DETECT_SM_LIST_NOTSET) {
+        SCLogError("no buffer for absent keyword");
+        return -1;
+    }
+
+    if (DetectBufferGetActiveList(de_ctx, s) == -1)
+        return -1;
+
+    bool or_else = false;
+    if (strcmp(optstr, "or_else") == 0) {
+        or_else = true;
+    } else if (strcmp(optstr, "only") != 0) {
+        SCLogError("unhandled value for absent keyword: %s", optstr);
+        return -1;
+    }
+    DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData));
+    if (unlikely(dad == NULL))
+        return -1;
+
+    dad->or_else = or_else;
+
+    if (SigMatchAppendSMToList(de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) ==
+            NULL) {
+        DetectAbsentFree(de_ctx, dad);
+        return -1;
+    }
+    return 0;
+}
+
+bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *b)
+{
+    bool has_other = false;
+    bool only_absent = false;
+    bool has_absent = false;
+    for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) {
+        if (sm->type == DETECT_ABSENT) {
+            has_absent = true;
+            const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx;
+            if (!dad->or_else) {
+                only_absent = true;
+            }
+        } else {
+            has_other = true;
+            if (sm->type == DETECT_CONTENT) {
+                const DetectContentData *cd = (DetectContentData *)sm->ctx;
+                if (has_absent && (cd->flags & DETECT_CONTENT_NEGATED) == 0) {
+                    SCLogError("signature can't have a buffer both absent and with content");
+                    return false;
+                }
+            } else if (sm->type == DETECT_PCRE) {
+                const DetectPcreData *cd = (DetectPcreData *)sm->ctx;
+                if (has_absent && (cd->flags & DETECT_PCRE_NEGATE) == 0) {
+                    SCLogError("signature can't have a buffer both absent and with pcre content");
+                    return false;
+                }
+            } else if (sm->type == DETECT_ISDATAAT) {
+                const DetectIsdataatData *cd = (DetectIsdataatData *)sm->ctx;
+                if (has_absent && (cd->flags & ISDATAAT_NEGATED) == 0) {
+                    SCLogError("signature can't have a buffer both absent and with pcre content");
+                    return false;
+                }
+            } else if (sm->type == DETECT_BYTETEST) {
+                const DetectBytetestData *cd = (DetectBytetestData *)sm->ctx;
+                if (has_absent && !cd->neg_op) {
+                    SCLogError("signature can't have a buffer both absent and with pcre content");
+                    return false;
+                }
+            } else if (has_absent) {
+                SCLogError("signature can't have a buffer absent with unhandled keyword");
+                return false;
+            }
+        }
+    }
+
+    if (only_absent && has_other) {
+        SCLogError("signature can't have a buffer both absent and tested otherwise");
+        return false;
+    } else if (has_absent && !only_absent && !has_other) {
+        SCLogError("signature with absent: or_else expects something else to test on");
+        return false;
+    }
+    return true;
+}
+
 /**
  * \brief Registration function for isdataat: keyword
  */
@@ -82,6 +174,12 @@ void DetectIsdataatRegister(void)
     sigmatch_table[DETECT_ENDS_WITH].Setup = DetectEndsWithSetup;
     sigmatch_table[DETECT_ENDS_WITH].flags = SIGMATCH_NOOPT;
 
+    sigmatch_table[DETECT_ABSENT].name = "absent";
+    sigmatch_table[DETECT_ABSENT].desc = "test if the buffer is absent";
+    sigmatch_table[DETECT_ABSENT].url = "/rules/payload-keywords.html#absent";
+    sigmatch_table[DETECT_ABSENT].Setup = DetectAbsentSetup;
+    sigmatch_table[DETECT_ABSENT].Free = DetectAbsentFree;
+
     DetectSetupParseRegexes(PARSE_REGEX, &parse_regex);
 }
 

src/detect-isdataat.h

@@ -34,7 +34,14 @@ typedef struct DetectIsdataatData_ {
     uint8_t flags; /* isdataat options*/
 } DetectIsdataatData;
 
+typedef struct DetectAbsentData_ {
+    // absent or something else (or only absent)
+    bool or_else;
+} DetectAbsentData;
+
 /* prototypes */
 void DetectIsdataatRegister (void);
 
+bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *);
+
 #endif /* SURICATA_DETECT_ISDATAAT_H */

src/detect-parse.c

@@ -35,6 +35,7 @@
 
 #include "detect-content.h"
 #include "detect-bsize.h"
+#include "detect-isdataat.h"
 #include "detect-pcre.h"
 #include "detect-uricontent.h"
 #include "detect-reference.h"
@@ -1982,6 +1983,9 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         if (!DetectBsizeValidateContentCallback(s, b)) {
             SCReturnInt(0);
         }
+        if (!DetectAbsentValidateContentCallback(s, b)) {
+            SCReturnInt(0);
+        }
     }
 
     int ts_excl = 0;

src/detect.h

@@ -434,6 +434,8 @@ typedef struct DetectEngineAppInspectionEngine_ {
     uint8_t id;     /**< per sig id used in state keeping */
     bool mpm;
     bool stream;
+    /** will match on a NULL buffer, cf absent keyword */
+    bool match_on_null;
     uint16_t sm_list;
     uint16_t sm_list_base; /**< base buffer being transformed */
     int16_t progress;
@@ -515,6 +517,8 @@ typedef struct DetectEngineFrameInspectionEngine {
     uint8_t dir;
     uint8_t type;
     bool mpm;
+    /** will match on a NULL buffer, cf absent keyword */
+    bool match_on_null;
     uint16_t sm_list;
     uint16_t sm_list_base;
     struct {