Merge pull request #38 from deadbits/vdb_dupes

VectorDB dupes

vigil/dispatch.py

@@ -41,7 +41,7 @@ def __init__(self,
 
         if self.auto_update:
             if self.db_client is None:
-                logger.warn('{self.name} Auto-update disabled: db client is None')
+                logger.warn(f'{self.name} Auto-update disabled: db client is None')
             else:
                 logger.info(f'{self.name} Auto-update vectordb enabled: threshold={self.update_threshold}')
 
@@ -59,7 +59,7 @@ def perform_scan(self, input_prompt: str, input_resp: str = None) -> dict:
         if not input_prompt:
             resp.errors.append('Input prompt value is empty')
             resp.status = 'failed'
-            logger.error('{self.name} Input prompt value is empty')
+            logger.error(f'{self.name} Input prompt value is empty')
             return resp.dict()
 
         logger.info(f'{self.name} Dispatching scan request id={resp.uuid}')

vigil/scanners/vectordb.py

@@ -23,14 +23,17 @@ def analyze(self, scan_obj: ScanModel, scan_id: uuid.uuid4) -> ScanModel:
             logger.error(f'Failed to perform vector scan; id="{scan_id}" error="{err}"')
             return scan_obj
 
+        existing_texts = []
+
         for match in zip(matches["documents"][0], matches["metadatas"][0], matches["distances"][0]):
             distance = match[2]
 
-            if distance < self.threshold:
+            if distance < self.threshold and match[0] not in existing_texts:
                 # with chromadb a lower distance means the vectors are more similar
                 m = VectorMatch(text=match[0], metadata=match[1], distance=match[2])
                 logger.warning(f'Matched vector text="{m.text}" threshold="{self.threshold}" distance="{m.distance}" id="{scan_id}"')
                 scan_obj.results.append(m)
+                existing_texts.append(m.text)
 
         if len(scan_obj.results) == 0:
             logger.info(f'No matches found; id="{scan_id}"')