Add Swedish language support

Yavari · Aug 30, 2019 · 0dd5e0e · 0dd5e0e
1 parent cbd2a8d
commit 0dd5e0e
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 2 deletions.
diff --git a/ElasticSearch/Dockerfile b/ElasticSearch/Dockerfile
@@ -9,6 +9,7 @@ COPY elasticsearch.yml ./config/elasticsearch.yml
 RUN bin/elasticsearch-plugin install http://dl.bintray.com/content/imotov/elasticsearch-plugins/org/elasticsearch/elasticsearch-analysis-morphology/5.6.3/elasticsearch-analysis-morphology-5.6.3.zip
 RUN bin/elasticsearch-plugin install analysis-stempel
 RUN bin/elasticsearch-plugin install analysis-smartcn
+RUN bin/elasticsearch-plugin install analysis-icu
 
 CMD ["elasticsearch"]
 

diff --git a/Pipeline/Dockerfile b/Pipeline/Dockerfile
@@ -17,6 +17,7 @@ RUN apt-get update && apt-get install -y \
     tesseract-ocr-spa \
     tesseract-ocr-nld \
     tesseract-ocr-pol \
+    tesseract-ocr-swe \
     default-jre \
     default-jdk \
     readpst

diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ Ambar defines a new way to implement a full-text document search into yor workfl
 * Search By Size (size>1M)
 * Search By Tags (tags:ocr)
 * Search As You Type
-* Supported language analyzers: English `ambar_en`, Russian `ambar_ru`, German `ambar_de`, Italian `ambar_it`, Polish `ambar_pl`, Chinese `ambar_cn`, CJK `ambar_cjk`
+* Supported language analyzers: English `ambar_en`, Russian `ambar_ru`, German `ambar_de`, Italian `ambar_it`, Polish `ambar_pl`, Chinese `ambar_cn`, CJK `ambar_cjk`, Swedish `ambar_sv`
 
 ### Crawling
 
@@ -44,7 +44,7 @@ Crawling is automatic, no schedule is needed since the crawler monitors fs event
 * OCR over images
 * Email messages with attachments
 * Adobe PDF (with OCR)
-* OCR languages: Eng, Rus, Ita, Deu, Fra, Spa, Pl, Nld
+* OCR languages: Eng, Rus, Ita, Deu, Fra, Spa, Pl, Nld, Swe
 * OpenOffice documents
 * RTF, Plaintext
 * HTML / XHTML

diff --git a/ServiceApi/src/services/EsProxy/AmbarFileDataMapping.json b/ServiceApi/src/services/EsProxy/AmbarFileDataMapping.json
@@ -92,6 +92,18 @@
                 "german_stemmer": {
                     "type": "stemmer",
                     "language": "light_german"
+                },
+                "swedish_stemmer": {
+                  "type": "stemmer",
+                  "language": "swedish"
+                },
+                "swedish_stop": {
+                  "type": "stop",
+                  "stopwords": "_swedish_"
+                },
+                "swedish_icu_folding": {
+                  "type": "icu_folding",
+                  "unicodeSetFilter": "[^åäöÅÄÖ]"
                 }
             },
             "analyzer": {
@@ -164,6 +176,15 @@
                     "filter": [
                         "lowercase"
                     ]
+                },
+                "ambar_sv": {
+                  "tokenizer": "standard",
+                  "filter": [
+                    "lowercase",
+                    "swedish_icu_folding",
+                    "swedish_stop",
+                    "swedish_stemmer"
+                  ]
                 }
             }
         }