diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst index 2e92b7046269..1234e6367954 100644 --- a/doc/userguide/rules/payload-keywords.rst +++ b/doc/userguide/rules/payload-keywords.rst @@ -274,6 +274,28 @@ You can also use the negation (!) before isdataat. .. image:: payload-keywords/isdataat1.png +absent +------ + +The keyword ``absent`` checks that a sticky buffer does not exist. +It can be used without any argument to match only on absent buffer : + +Example of ``absent`` in a rule: + +.. container:: example-rule + + alert http any any -> any any (msg:"HTTP request without referer"; http.referer; absent; sid:1; rev:1;) + + +It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance : + +.. container:: example-rule + + alert http any any -> any any (msg:"HTTP request without referer"; http.referer; absent: or_else; content: !"abc"; sid:1; rev:1;) + +An absent keyword cannot work with positive (non negated) content or pcre. +For files (ie ``file.data``), absent means there are no files in the transaction. + bsize ----- diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index 0e78d4ae85df..d5e66f455238 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -382,6 +382,14 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, prev_offset); } while(1); + } else if (smd->type == DETECT_ABSENT) { + const DetectAbsentData *id = (DetectAbsentData *)smd->ctx; + if (!id->or_else) { + // we match only on absent buffer + goto no_match; + } else { + goto match; + } } else if (smd->type == DETECT_ISDATAAT) { SCLogDebug("inspecting isdataat"); @@ -648,8 +656,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, goto match; } goto no_match_discontinue; - } - else if (smd->type == DETECT_LUA) { + } else if (smd->type == DETECT_LUA) { SCLogDebug("lua starting"); if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len, @@ -760,6 +767,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh return false; } +bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd) +{ + // we will match on NULL buffers there is one absent + bool absent_data = false; + while (1) { + if (smd->type == DETECT_ABSENT) { + absent_data = true; + break; + } + if (smd->is_last) { + break; + } + // smd does not get reused after this loop + smd++; + } + return absent_data; +} + #ifdef UNITTESTS #include "tests/detect-engine-content-inspection.c" #endif diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 2c253b77ad3d..fe53086badd9 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b, const enum DetectContentInspectionType inspection_mode); +/** \brief tells if we should match on absent buffer, because + * there is an absent keyword being used + * \param smd array of content inspection matches + * \retval bool true to match on absent buffer, false otherwise */ +bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd); + void DetectEngineContentInspectionRegisterTests(void); #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */ diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 94f8de15de83..b201ece0a036 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -80,6 +80,7 @@ enum DetectKeywordId { DETECT_IPPROTO, DETECT_FTPBOUNCE, DETECT_ISDATAAT, + DETECT_ABSENT, DETECT_ID, DETECT_RPC, DETECT_FLOWVAR, diff --git a/src/detect-engine.c b/src/detect-engine.c index 819fa04a39cd..71e0b7e1457a 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -659,6 +659,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, new_engine->sm_list = t->sm_list; new_engine->sm_list_base = t->sm_list_base; new_engine->smd = smd; + new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd); new_engine->progress = t->progress; new_engine->v2 = t->v2; SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback, @@ -2153,6 +2154,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms, f, flags, txv, list_id); if (unlikely(buffer == NULL)) { + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } @@ -2214,6 +2218,13 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx, } local_id++; } while (1); + if (local_id == 0) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + } return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-file-data.c b/src/detect-file-data.c index 732dfd63731c..60d1804ec9a0 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -396,7 +396,15 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index f23434d8666e..0f8d94a7b5b1 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -307,7 +307,15 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-filename.c b/src/detect-filename.c index f75fdbd680fe..ef144cf44086 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -244,7 +244,15 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index 5e5604ea594d..7747b61b858b 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx, const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback( det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base); if (buffer == NULL || buffer->inspect == NULL) { + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-http-header.c b/src/detect-http-header.c index be825e5ec714..c0e21f224d7b 100644 --- a/src/detect-http-header.c +++ b/src/detect-http-header.c @@ -176,6 +176,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, const int list_id = engine->sm_list; InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); + bool eof = + (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress); if (buffer->inspect == NULL) { SCLogDebug("setting up inspect buffer %d", list_id); @@ -189,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len); if (rawdata_len == 0) { SCLogDebug("no data"); + if (engine->match_on_null && eof) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } goto end; } /* setup buffer and apply transforms */ @@ -209,14 +214,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, return DETECT_ENGINE_INSPECT_SIG_MATCH; } end: - if (flags & STREAM_TOSERVER) { - if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) > - HTP_REQUEST_HEADERS) - return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; - } else { - if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) > - HTP_RESPONSE_HEADERS) - return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; + if (eof) { + return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; } return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c index 7b4d629ad3a1..c397a7d4e48a 100644 --- a/src/detect-isdataat.c +++ b/src/detect-isdataat.c @@ -35,6 +35,7 @@ #include "detect-isdataat.h" #include "detect-content.h" +#include "detect-bytetest.h" #include "detect-uricontent.h" #include "detect-engine-build.h" @@ -56,11 +57,118 @@ static DetectParseRegex parse_regex; int DetectIsdataatSetup (DetectEngineCtx *, Signature *, const char *); #ifdef UNITTESTS static void DetectIsdataatRegisterTests(void); +static void DetectAbsentRegisterTests(void); #endif void DetectIsdataatFree(DetectEngineCtx *, void *); static int DetectEndsWithSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr); +static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr) +{ + if (s->init_data->list == DETECT_SM_LIST_NOTSET) { + SCLogError("no buffer for absent keyword"); + return -1; + } + + if (DetectBufferGetActiveList(de_ctx, s) == -1) + return -1; + + bool or_else; + if (optstr == NULL) { + or_else = false; + } else if (strcmp(optstr, "or_else") == 0) { + or_else = true; + } else { + SCLogError("unhandled value for absent keyword: %s", optstr); + return -1; + } + if (s->init_data->curbuf == NULL || s->init_data->list != (int)s->init_data->curbuf->id) { + SCLogError("unspected buffer for absent keyword"); + return -1; + } + const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, s->init_data->list); + if (!b || b->frame) { + SCLogError("absent does not work with frames"); + return -1; + } + if (s->init_data->curbuf->tail != NULL) { + SCLogError("absent must come first right after buffer"); + return -1; + } + DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData)); + if (unlikely(dad == NULL)) + return -1; + + dad->or_else = or_else; + + if (SigMatchAppendSMToList(de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) == + NULL) { + DetectAbsentFree(de_ctx, dad); + return -1; + } + return 0; +} + +bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *b) +{ + bool has_other = false; + bool only_absent = false; + bool has_absent = false; + for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_ABSENT) { + has_absent = true; + const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx; + if (!dad->or_else) { + only_absent = true; + } + } else { + has_other = true; + if (sm->type == DETECT_CONTENT) { + const DetectContentData *cd = (DetectContentData *)sm->ctx; + if (has_absent && (cd->flags & DETECT_CONTENT_NEGATED) == 0) { + SCLogError("signature can't have a buffer both absent and with content"); + return false; + } + } else if (sm->type == DETECT_PCRE) { + const DetectPcreData *cd = (DetectPcreData *)sm->ctx; + if (has_absent && (cd->flags & DETECT_PCRE_NEGATE) == 0) { + SCLogError("signature can't have a buffer both absent and with pcre content"); + return false; + } + } else if (sm->type == DETECT_ISDATAAT) { + const DetectIsdataatData *cd = (DetectIsdataatData *)sm->ctx; + if (has_absent && (cd->flags & ISDATAAT_NEGATED) == 0) { + SCLogError("signature can't have a buffer both absent and with pcre content"); + return false; + } + } else if (sm->type == DETECT_BYTETEST) { + const DetectBytetestData *cd = (DetectBytetestData *)sm->ctx; + if (has_absent && !cd->neg_op) { + SCLogError("signature can't have a buffer both absent and with pcre content"); + return false; + } + } else if (has_absent) { + SCLogError("signature can't have a buffer absent with unhandled keyword"); + return false; + } + } + } + + if (only_absent && has_other) { + SCLogError("signature can't have a buffer only absent and tested otherwise"); + return false; + } else if (has_absent && !only_absent && !has_other) { + SCLogError("signature with absent: or_else expects something else to test on"); + return false; + } + return true; +} + /** * \brief Registration function for isdataat: keyword */ @@ -82,6 +190,16 @@ void DetectIsdataatRegister(void) sigmatch_table[DETECT_ENDS_WITH].Setup = DetectEndsWithSetup; sigmatch_table[DETECT_ENDS_WITH].flags = SIGMATCH_NOOPT; + sigmatch_table[DETECT_ABSENT].name = "absent"; + sigmatch_table[DETECT_ABSENT].desc = "test if the buffer is absent"; + sigmatch_table[DETECT_ABSENT].url = "/rules/payload-keywords.html#absent"; + sigmatch_table[DETECT_ABSENT].Setup = DetectAbsentSetup; + sigmatch_table[DETECT_ABSENT].Free = DetectAbsentFree; + sigmatch_table[DETECT_ABSENT].flags = SIGMATCH_OPTIONAL_OPT; +#ifdef UNITTESTS + sigmatch_table[DETECT_ABSENT].RegisterTests = DetectAbsentRegisterTests; +#endif + DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -584,4 +702,46 @@ void DetectIsdataatRegisterTests(void) UtRegisterTest("DetectIsdataatTestPacket02", DetectIsdataatTestPacket02); UtRegisterTest("DetectIsdataatTestPacket03", DetectIsdataatTestPacket03); } + +static int DetectAbsentTestParse01(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF(de_ctx == NULL); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"invalid absent with positive content\"; http.user_agent; " + "absent: or_else; content:\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"invalid absent only with negated content\"; http.user_agent; " + "absent; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent\"; http.user_agent; " + "content:!\"one\"; absent; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent\"; http.user_agent; " + "content:\"one\"; absent: or_else; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"absent without sticky buffer\"; " + "content:!\"one\"; absent: or_else; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, + "alert websocket any any -> any any " + "(msg:\"absent with frame\"; " + "frame: websocket.pdu; absent: or_else; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + DetectEngineCtxFree(de_ctx); + PASS; +} + +void DetectAbsentRegisterTests(void) +{ + UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01); +} #endif diff --git a/src/detect-isdataat.h b/src/detect-isdataat.h index 01ea2e304f42..9748c33f762e 100644 --- a/src/detect-isdataat.h +++ b/src/detect-isdataat.h @@ -34,7 +34,14 @@ typedef struct DetectIsdataatData_ { uint8_t flags; /* isdataat options*/ } DetectIsdataatData; +typedef struct DetectAbsentData_ { + // absent or something else (or only absent) + bool or_else; +} DetectAbsentData; + /* prototypes */ void DetectIsdataatRegister (void); +bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *); + #endif /* SURICATA_DETECT_ISDATAAT_H */ diff --git a/src/detect-parse.c b/src/detect-parse.c index 61318521f45d..d4ee54bd8585 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -35,6 +35,7 @@ #include "detect-content.h" #include "detect-bsize.h" +#include "detect-isdataat.h" #include "detect-pcre.h" #include "detect-uricontent.h" #include "detect-reference.h" @@ -1982,6 +1983,9 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s) if (!DetectBsizeValidateContentCallback(s, b)) { SCReturnInt(0); } + if (!DetectAbsentValidateContentCallback(s, b)) { + SCReturnInt(0); + } } int ts_excl = 0; diff --git a/src/detect.h b/src/detect.h index a4f154523280..ee6f9ae8f602 100644 --- a/src/detect.h +++ b/src/detect.h @@ -432,6 +432,8 @@ typedef struct DetectEngineAppInspectionEngine_ { uint8_t id; /**< per sig id used in state keeping */ bool mpm; bool stream; + /** will match on a NULL buffer, cf absent keyword */ + bool match_on_null; uint16_t sm_list; uint16_t sm_list_base; /**< base buffer being transformed */ int16_t progress;