var-names: reimplement var name handling

Implement a new design for handling var name id's. The old logic
was aware of detection engine versions and generally didn't work
well for multi-tenancy cases. Other than memory leaks and crashes,
logging of var names worked or failed based on which tenant was
loaded last.

This patch implements a new approach, where there is a global store
of vars and their id's for the lifetime of the program.

Overall Design:

Base Store: "base"

Used during keyword registration. Operates under lock. Base is shared
between all detect engines, detect engine versions and tenants.
Each variable name is ref counted.

During the freeing of a detect engine / tenant, unregistration decreases
the ref cnt.

Base has both a string to id and a id to string hash table. String to
id is used during parsing/registration. id to string during unregistration.

Active Store Pointer (atomic)

The "active" store atomic pointer points to the active lookup store. The call
to `VarNameStoreActivate` will build a new lookup store and hot swap
the pointer.

Ensuring memory safety. During the hot swap, the pointer is replaced, so
any new call to the lookup functions will automatically use the new store.
This leaves the case of any lookup happening concurrently with the pointer
swap. For this case we add the old store to a free list. It gets a timestamp
before which it cannot be freed.

Free List

The free list contains old stores that are waiting to get removed. They
contain a timestamp that is checked before they are freed.

Bug: OISF#6044.
Bug: OISF#6201.

src/detect-engine-build.c

@@ -1967,7 +1967,7 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
     ThresholdHashAllocate(de_ctx);
 
     if (!DetectEngineMultiTenantEnabled()) {
-        VarNameStoreActivateStaging();
+        VarNameStoreActivate();
     }
     return 0;
 }

src/detect-engine.c

@@ -2041,7 +2041,6 @@ static DetectEngineCtx *DetectEngineCtxInitReal(enum DetectEngineType type, cons
     }
 
     de_ctx->version = DetectEngineGetVersion();
-    VarNameStoreSetupStaging(de_ctx->version);
     SCLogDebug("dectx with version %u", de_ctx->version);
     return de_ctx;
 error:
@@ -2172,8 +2171,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     DetectPortCleanupList(de_ctx, de_ctx->udp_whitelist);
 
     DetectBufferTypeFreeDetectEngine(de_ctx);
-    /* freed our var name hash */
-    VarNameStoreFree(de_ctx->version);
 
     SCFree(de_ctx);
     //DetectAddressGroupPrintMemory();
@@ -3765,7 +3762,7 @@ int DetectEngineMultiTenantSetup(void)
             goto error;
         }
 
-        VarNameStoreActivateStaging();
+        VarNameStoreActivate();
 
     } else {
         SCLogDebug("multi-detect not enabled (multi tenancy)");

src/detect-flowbits.c

@@ -109,7 +109,7 @@ static int FlowbitOrAddData(DetectEngineCtx *de_ctx, DetectFlowbitsData *cd, cha
     if (unlikely(cd->or_list == NULL))
         return -1;
     for (uint8_t j = 0; j < cd->or_list_size ; j++) {
-        cd->or_list[j] = VarNameStoreSetupAdd(strarr[j], VAR_TYPE_FLOW_BIT);
+        cd->or_list[j] = VarNameStoreRegister(strarr[j], VAR_TYPE_FLOW_BIT);
         de_ctx->max_fb_id = MAX(cd->or_list[j], de_ctx->max_fb_id);
     }
 
@@ -310,7 +310,7 @@ static int DetectFlowbitParse(
         }
         cd->cmd = cmd;
     } else {
-        cd->idx = VarNameStoreSetupAdd(name, VAR_TYPE_FLOW_BIT);
+        cd->idx = VarNameStoreRegister(name, VAR_TYPE_FLOW_BIT);
         de_ctx->max_fb_id = MAX(cd->idx, de_ctx->max_fb_id);
         cd->cmd = cmd;
         cd->or_list_size = 0;
@@ -386,8 +386,13 @@ void DetectFlowbitFree (DetectEngineCtx *de_ctx, void *ptr)
     DetectFlowbitsData *fd = (DetectFlowbitsData *)ptr;
     if (fd == NULL)
         return;
-    if (fd->or_list != NULL)
+    VarNameStoreUnregister(fd->idx, VAR_TYPE_FLOW_BIT);
+    if (fd->or_list != NULL) {
+        for (uint8_t i = 0; i < fd->or_list_size; i++) {
+            VarNameStoreUnregister(fd->or_list[i], VAR_TYPE_FLOW_BIT);
+        }
         SCFree(fd->or_list);
+    }
     SCFree(fd);
 }
 
@@ -590,7 +595,7 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
 
     /* walk array to see if all bits make sense */
     for (uint32_t i = 0; i < array_size; i++) {
-        char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
+        const char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
         if (varname == NULL)
             continue;
 
@@ -643,7 +648,6 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
                         "stateful rules that set flowbit %s", s->id, varname);
             }
         }
-        SCFree(varname);
     }
 
     if (rule_engine_analysis_set) {
@@ -673,7 +677,7 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
 
     jb_open_array(js, "flowbits");
     for (uint32_t x = 0; x < elements; x++) {
-        char *varname = VarNameStoreSetupLookup(x, VAR_TYPE_FLOW_BIT);
+        const char *varname = VarNameStoreSetupLookup(x, VAR_TYPE_FLOW_BIT);
         if (varname == NULL)
             continue;
 
@@ -733,7 +737,6 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
             }
             jb_close(js);
         }
-        SCFree(varname);
         jb_close(js);
     }
     jb_close(js); // array
@@ -911,8 +914,8 @@ static int FlowBitsTestSig04(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"isset option\"; flowbits:isset,fbt; content:\"GET \"; sid:1;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("fbt", VAR_TYPE_FLOW_BIT);
-    FAIL_IF(idx != 1);
+    idx = VarNameStoreRegister("fbt", VAR_TYPE_FLOW_BIT);
+    FAIL_IF(idx == 0);
 
     SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
@@ -994,7 +997,7 @@ static int FlowBitsTestSig06(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit set\"; flowbits:set,myflow; sid:10;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreRegister("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
@@ -1068,7 +1071,7 @@ static int FlowBitsTestSig07(void)
     s = s->next = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit unset\"; flowbits:unset,myflow2; sid:11;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreRegister("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
@@ -1144,7 +1147,7 @@ static int FlowBitsTestSig08(void)
     s = s->next  = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit unset\"; flowbits:toggle,myflow2; sid:11;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreRegister("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 

src/detect-flowint.c

@@ -329,7 +329,7 @@ static DetectFlowintData *DetectFlowintParse(DetectEngineCtx *de_ctx, const char
         SCLogError(SC_ERR_MEM_ALLOC, "malloc from strdup failed");
         goto error;
     }
-    sfd->idx = VarNameStoreSetupAdd(varname, VAR_TYPE_FLOW_INT);
+    sfd->idx = VarNameStoreRegister(varname, VAR_TYPE_FLOW_INT);
     SCLogDebug("sfd->name %s id %u", sfd->name, sfd->idx);
     sfd->modifier = modifier;
 
@@ -416,6 +416,7 @@ void DetectFlowintFree(DetectEngineCtx *de_ctx, void *tmp)
 {
     DetectFlowintData *sfd =(DetectFlowintData*) tmp;
     if (sfd != NULL) {
+        VarNameStoreUnregister(sfd->idx, VAR_TYPE_FLOW_INT);
         if (sfd->name != NULL)
             SCFree(sfd->name);
         if (sfd->targettype == FLOWINT_TARGET_VAR)

src/detect-flowvar.c

@@ -78,6 +78,9 @@ static void DetectFlowvarDataFree(DetectEngineCtx *de_ctx, void *ptr)
         SCReturn;
 
     DetectFlowvarData *fd = (DetectFlowvarData *)ptr;
+    /* leave unregistration to pcre keyword */
+    if (!fd->post_match)
+        VarNameStoreUnregister(fd->idx, VAR_TYPE_FLOW_VAR);
 
     if (fd->name)
         SCFree(fd->name);
@@ -172,7 +175,7 @@ static int DetectFlowvarSetup (DetectEngineCtx *de_ctx, Signature *s, const char
     fd->name = SCStrdup(varname);
     if (unlikely(fd->name == NULL))
         goto error;
-    fd->idx = VarNameStoreSetupAdd(varname, VAR_TYPE_FLOW_VAR);
+    fd->idx = VarNameStoreRegister(varname, VAR_TYPE_FLOW_VAR);
 
     /* Okay so far so good, lets get this into a SigMatch
      * and put it in the Signature. */
@@ -267,6 +270,7 @@ int DetectFlowvarPostMatchSetup(DetectEngineCtx *de_ctx, Signature *s, uint32_t
 
     /* we only need the idx */
     fv->idx = idx;
+    fv->post_match = true;
 
     sm = SigMatchAlloc();
     if (unlikely(sm == NULL))

src/detect-flowvar.h

@@ -28,8 +28,10 @@ typedef struct DetectFlowvarData_ {
     char *name;
     uint32_t idx;
     uint8_t *content;
-    uint8_t content_len;
-    uint8_t flags;
+    uint16_t content_len;
+    /** set to true if used in a post-match */
+    bool post_match;
+    uint32_t flags;
 } DetectFlowvarData;
 
 /* prototypes */

src/detect-hostbits.c

@@ -387,7 +387,7 @@ int DetectHostbitSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     if (unlikely(cd == NULL))
         goto error;
 
-    cd->idx = VarNameStoreSetupAdd(fb_name, VAR_TYPE_HOST_BIT);
+    cd->idx = VarNameStoreRegister(fb_name, VAR_TYPE_HOST_BIT);
     cd->cmd = fb_cmd;
     cd->tracker = hb_dir;
     cd->type = VAR_TYPE_HOST_BIT;
@@ -443,6 +443,7 @@ void DetectHostbitFree (DetectEngineCtx *de_ctx, void *ptr)
 
     if (fd == NULL)
         return;
+    VarNameStoreUnregister(fd->idx, VAR_TYPE_HOST_BIT);
 
     SCFree(fd);
 }
@@ -760,8 +761,8 @@ static int HostBitsTestSig04(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"isset option\"; hostbits:isset,fbt; content:\"GET \"; sid:1;)");
     FAIL_IF_NULL(s);
 
-    idx = VarNameStoreSetupAdd("fbt", VAR_TYPE_HOST_BIT);
-    FAIL_IF(idx != 1);
+    idx = VarNameStoreRegister("fbt", VAR_TYPE_HOST_BIT);
+    FAIL_IF(idx == 0);
 
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);