diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst index 2e92b704626..780ad111f7d 100644 --- a/doc/userguide/rules/payload-keywords.rst +++ b/doc/userguide/rules/payload-keywords.rst @@ -274,6 +274,27 @@ You can also use the negation (!) before isdataat. .. image:: payload-keywords/isdataat1.png +absent +------ + +The keyword ``absent`` checks that a sticky buffer does not exist. +It can be used without any argument to match only on absent buffer : + +Example of ``absent`` in a rule: + +.. container:: example-rule + + alert http any any -> any any (msg:"HTTP request without referer"; :example-rule-emphasis:`http.referer; absent;` sid:1; rev:1;) + + +It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance : + +.. container:: example-rule + + alert http any any -> any any (msg:"HTTP request without referer"; :example-rule-emphasis:`http.referer; absent: or_else;` content: !"abc"; sid:1; rev:1;) + +For files (ie ``file.data``), absent means there are no files in the transaction. + bsize ----- diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c index 3ae77526db0..abd815a20a7 100644 --- a/src/detect-engine-analyzer.c +++ b/src/detect-engine-analyzer.c @@ -39,6 +39,7 @@ #include "detect-pcre.h" #include "detect-bytejump.h" #include "detect-bytetest.h" +#include "detect-isdataat.h" #include "detect-flow.h" #include "detect-tcp-flags.h" #include "detect-tcp-ack.h" @@ -853,6 +854,14 @@ static void DumpMatches(RuleAnalyzer *ctx, JsonBuilder *js, const SigMatchData * jb_close(js); break; } + case DETECT_ABSENT: { + const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx; + jb_open_object(js, "absent"); + jb_set_bool(js, "or_else", dad->or_else); + jb_close(js); + break; + } + case DETECT_IPOPTS: { const DetectIpOptsData *cd = (const DetectIpOptsData *)smd->ctx; diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index d4dab42816d..e43e693b215 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -382,6 +382,13 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, prev_offset); } while(1); + } else if (smd->type == DETECT_ABSENT) { + const DetectAbsentData *id = (DetectAbsentData *)smd->ctx; + if (!id->or_else) { + // we match only on absent buffer + goto no_match; + } + goto match; } else if (smd->type == DETECT_ISDATAAT) { SCLogDebug("inspecting isdataat"); @@ -646,8 +653,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, goto match; } goto no_match_discontinue; - } - else if (smd->type == DETECT_LUA) { + } else if (smd->type == DETECT_LUA) { SCLogDebug("lua starting"); if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len, @@ -758,6 +764,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh return false; } +bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd) +{ + // we will match on NULL buffers there is one absent + bool absent_data = false; + while (1) { + if (smd->type == DETECT_ABSENT) { + absent_data = true; + break; + } + if (smd->is_last) { + break; + } + // smd does not get reused after this loop + smd++; + } + return absent_data; +} + #ifdef UNITTESTS #include "tests/detect-engine-content-inspection.c" #endif diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 2c253b77ad3..fe53086badd 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b, const enum DetectContentInspectionType inspection_mode); +/** \brief tells if we should match on absent buffer, because + * there is an absent keyword being used + * \param smd array of content inspection matches + * \retval bool true to match on absent buffer, false otherwise */ +bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd); + void DetectEngineContentInspectionRegisterTests(void); #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */ diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 5e8687e3468..1c9984ea954 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -1125,6 +1125,9 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) } for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { + // a buffer with absent keyword cannot be used as fast_pattern + if (sm->type == DETECT_ABSENT) + break; if (sm->type != DETECT_CONTENT) continue; diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 7c3b5b4514b..8617901aa8d 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -93,6 +93,7 @@ enum DetectKeywordId { DETECT_LUA, DETECT_ISDATAAT, DETECT_AL_URILEN, + DETECT_ABSENT, /* end of content inspection */ DETECT_METADATA, diff --git a/src/detect-engine.c b/src/detect-engine.c index 58b5c9967c8..2d0b74a10b2 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -664,6 +664,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, new_engine->sm_list = t->sm_list; new_engine->sm_list_base = t->sm_list_base; new_engine->smd = smd; + new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd); new_engine->progress = t->progress; new_engine->v2 = t->v2; SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback, @@ -2158,6 +2159,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms, f, flags, txv, list_id); if (unlikely(buffer == NULL)) { + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } @@ -2219,6 +2223,14 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx, } local_id++; } while (1); + if (local_id == 0) { + // That means we did not get even one buffer value from the multi-buffer + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + } return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-file-data.c b/src/detect-file-data.c index a721c08c7cf..d976b51c00b 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -403,6 +403,14 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC if (ffc == NULL) { return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } + if (ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } int local_file_id = 0; File *file = ffc->head; diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index f23434d8666..0f8d94a7b5b 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -307,7 +307,15 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-filename.c b/src/detect-filename.c index f75fdbd680f..ef144cf4408 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -244,7 +244,15 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index 5e5604ea594..7747b61b858 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx, const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback( det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base); if (buffer == NULL || buffer->inspect == NULL) { + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-http-header.c b/src/detect-http-header.c index be825e5ec71..c0e21f224d7 100644 --- a/src/detect-http-header.c +++ b/src/detect-http-header.c @@ -176,6 +176,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, const int list_id = engine->sm_list; InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); + bool eof = + (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress); if (buffer->inspect == NULL) { SCLogDebug("setting up inspect buffer %d", list_id); @@ -189,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len); if (rawdata_len == 0) { SCLogDebug("no data"); + if (engine->match_on_null && eof) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } goto end; } /* setup buffer and apply transforms */ @@ -209,14 +214,8 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, return DETECT_ENGINE_INSPECT_SIG_MATCH; } end: - if (flags & STREAM_TOSERVER) { - if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) > - HTP_REQUEST_HEADERS) - return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; - } else { - if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) > - HTP_RESPONSE_HEADERS) - return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; + if (eof) { + return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; } return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c index 7b4d629ad3a..6bc4e961c07 100644 --- a/src/detect-isdataat.c +++ b/src/detect-isdataat.c @@ -35,6 +35,7 @@ #include "detect-isdataat.h" #include "detect-content.h" +#include "detect-bytetest.h" #include "detect-uricontent.h" #include "detect-engine-build.h" @@ -56,11 +57,99 @@ static DetectParseRegex parse_regex; int DetectIsdataatSetup (DetectEngineCtx *, Signature *, const char *); #ifdef UNITTESTS static void DetectIsdataatRegisterTests(void); +static void DetectAbsentRegisterTests(void); #endif void DetectIsdataatFree(DetectEngineCtx *, void *); static int DetectEndsWithSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr); +static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr) +{ + if (s->init_data->list == DETECT_SM_LIST_NOTSET) { + SCLogError("no buffer for absent keyword"); + return -1; + } + + if (DetectBufferGetActiveList(de_ctx, s) == -1) + return -1; + + bool or_else; + if (optstr == NULL) { + or_else = false; + } else if (strcmp(optstr, "or_else") == 0) { + or_else = true; + } else { + SCLogError("unhandled value for absent keyword: %s", optstr); + return -1; + } + if (s->init_data->curbuf == NULL || s->init_data->list != (int)s->init_data->curbuf->id) { + SCLogError("unspected buffer for absent keyword"); + return -1; + } + const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, s->init_data->list); + if (!b || b->frame) { + SCLogError("absent does not work with frames"); + return -1; + } + if (s->init_data->curbuf->tail != NULL) { + SCLogError("absent must come first right after buffer"); + return -1; + } + DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData)); + if (unlikely(dad == NULL)) + return -1; + + dad->or_else = or_else; + + if (SigMatchAppendSMToList(de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) == + NULL) { + DetectAbsentFree(de_ctx, dad); + return -1; + } + return 0; +} + +bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *b) +{ + bool has_other = false; + bool only_absent = false; + bool has_absent = false; + for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_ABSENT) { + has_absent = true; + const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx; + if (!dad->or_else) { + only_absent = true; + } + } else { + has_other = true; + if (sm->type == DETECT_CONTENT) { + const DetectContentData *cd = (DetectContentData *)sm->ctx; + if (has_absent && (cd->flags & DETECT_CONTENT_FAST_PATTERN)) { + SCLogError("signature can't have absent and fast_pattern on the same buffer"); + return false; + } + } + } + } + + if (only_absent && has_other) { + SCLogError("signature can't have a buffer tested absent and tested with other keywords " + "such as content"); + return false; + } else if (has_absent && !only_absent && !has_other) { + SCLogError( + "signature with absent: or_else expects other keywords to test on such as content"); + return false; + } + return true; +} + /** * \brief Registration function for isdataat: keyword */ @@ -82,6 +171,16 @@ void DetectIsdataatRegister(void) sigmatch_table[DETECT_ENDS_WITH].Setup = DetectEndsWithSetup; sigmatch_table[DETECT_ENDS_WITH].flags = SIGMATCH_NOOPT; + sigmatch_table[DETECT_ABSENT].name = "absent"; + sigmatch_table[DETECT_ABSENT].desc = "test if the buffer is absent"; + sigmatch_table[DETECT_ABSENT].url = "/rules/payload-keywords.html#absent"; + sigmatch_table[DETECT_ABSENT].Setup = DetectAbsentSetup; + sigmatch_table[DETECT_ABSENT].Free = DetectAbsentFree; + sigmatch_table[DETECT_ABSENT].flags = SIGMATCH_OPTIONAL_OPT; +#ifdef UNITTESTS + sigmatch_table[DETECT_ABSENT].RegisterTests = DetectAbsentRegisterTests; +#endif + DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -584,4 +683,41 @@ void DetectIsdataatRegisterTests(void) UtRegisterTest("DetectIsdataatTestPacket02", DetectIsdataatTestPacket02); UtRegisterTest("DetectIsdataatTestPacket03", DetectIsdataatTestPacket03); } + +static int DetectAbsentTestParse01(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF(de_ctx == NULL); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"invalid absent only with negated content\"; http.user_agent; " + "absent; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent\"; http.user_agent; " + "content:!\"one\"; absent; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent\"; http.user_agent; " + "content:\"one\"; absent: or_else; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"absent without sticky buffer\"; " + "content:!\"one\"; absent: or_else; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, + "alert websocket any any -> any any " + "(msg:\"absent with frame\"; " + "frame: websocket.pdu; absent: or_else; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + DetectEngineCtxFree(de_ctx); + PASS; +} + +void DetectAbsentRegisterTests(void) +{ + UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01); +} #endif diff --git a/src/detect-isdataat.h b/src/detect-isdataat.h index 01ea2e304f4..c0cff884838 100644 --- a/src/detect-isdataat.h +++ b/src/detect-isdataat.h @@ -34,7 +34,14 @@ typedef struct DetectIsdataatData_ { uint8_t flags; /* isdataat options*/ } DetectIsdataatData; +typedef struct DetectAbsentData_ { + /** absent or try to match with other keywords (false means only absent) */ + bool or_else; +} DetectAbsentData; + /* prototypes */ void DetectIsdataatRegister (void); +bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *); + #endif /* SURICATA_DETECT_ISDATAAT_H */ diff --git a/src/detect-parse.c b/src/detect-parse.c index 984501c1dd8..3b03dfb92b3 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -35,6 +35,7 @@ #include "detect-content.h" #include "detect-bsize.h" +#include "detect-isdataat.h" #include "detect-pcre.h" #include "detect-uricontent.h" #include "detect-reference.h" @@ -1971,6 +1972,9 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s) if (!DetectBsizeValidateContentCallback(s, b)) { SCReturnInt(0); } + if (!DetectAbsentValidateContentCallback(s, b)) { + SCReturnInt(0); + } } int ts_excl = 0; diff --git a/src/detect.h b/src/detect.h index cf8f4335b19..5e77c5a45de 100644 --- a/src/detect.h +++ b/src/detect.h @@ -432,6 +432,8 @@ typedef struct DetectEngineAppInspectionEngine_ { uint8_t id; /**< per sig id used in state keeping */ bool mpm; bool stream; + /** will match on a NULL buffer (so an absent buffer) */ + bool match_on_null; uint16_t sm_list; uint16_t sm_list_base; /**< base buffer being transformed */ int16_t progress;