Merge pull request #1157 from weaviate/add-kagome-kr-tokenizer

Add support for the kagome_kr tokenizer
weaviate · Jul 4, 2024 · a09598b · a09598b
2 parents 2541408 + 60d7849
commit a09598b
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py
@@ -134,6 +134,8 @@ class Tokenization(str, Enum):
             Tokenize using GSE (for Chinese and Japanese).
         `TRIGRAM`
             Tokenize into trigrams.
+        `KAGOME_KR`
+            Tokenize using the 'Kagome' tokenizer and a Korean MeCab dictionary (for Korean).
     """
 
     WORD = "word"
@@ -142,6 +144,7 @@ class Tokenization(str, Enum):
     FIELD = "field"
     GSE = "gse"
     TRIGRAM = "trigram"
+    KAGOME_KR = "kagome_kr"
 
 
 class GenerativeSearches(str, Enum):