[BUGFIX:BP:11.1] Fix autosuggest with non-ascii terms

The introduced ASCII folding filters or language depending normalization filters lead to issue with the auto suggest function due to the differing stemming behaviour. To fix this issue the original token is preserved if possible, this e.g. allows suggestions for search terms with and without accents. As this extension might lead to unwanted duplicates a new field textSpellExact is introduced, which considers non-ascii characters as given. Resolves: TYPO3-Solr#3096
dkd-friedrich · Dec 8, 2021 · 735979b · 735979b
1 parent 32c1984
commit 735979b
Show file tree

Hide file tree

Showing 43 changed files with 892 additions and 25 deletions.
diff --git a/Documentation/Appendix/DynamicFieldTypes.rst b/Documentation/Appendix/DynamicFieldTypes.rst
@@ -73,6 +73,8 @@ Extension               Type                                 Multivalue  Comment
 \*_textExactM           textExact                            Yes
 \*_textSpellS           textSpell                            No
 \*_textSpellM           textSpell                            Yes
+\*_textSpellExactS      textSpellExact                       No
+\*_textSpellExactM      textSpellExact                       Yes
 \*_phoneticS            Phonetic                             No
 \*_phoneticM            Phonetic                             Yes
 \*_point                point                                No

diff --git a/Documentation/Configuration/Reference/TxSolrSuggest.rst b/Documentation/Configuration/Reference/TxSolrSuggest.rst
@@ -34,6 +34,9 @@ suggestField
 
 Sets the Solr index field used to get suggestions from. A general advice is to use a field without stemming on it. For practical reasons this is currently the spell checker field.
 
+Note: With EXT:solr 11.1.0 ASCII folding and language depending normalization filters were introduced, but due to the special behaviour of the auto suggestions ascii-terms were not treated correctly. So with 11.1.3 the untouched tokens are also kept, as this might lead to duplicate
+suggestions, a new field for exact suggestions is introduced, if you want to avoid duplicates and use stricter suggestions, just configure `spellExact` as suggest field. 
+
 forceHttps
 ----------
 

diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/arabic/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/arabic/schema.xml
@@ -202,4 +202,30 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+			<filter class="solr.ArabicNormalizationFilterFactory"/>
+			<filter class="solr.ArabicStemFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.ArabicNormalizationFilterFactory"/>
+			<filter class="solr.ArabicStemFilterFactory"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
+
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/armenian/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/armenian/schema.xml
@@ -169,4 +169,23 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+			<filter class="solr.LowerCaseFilterFactory"/>
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+			<filter class="solr.LowerCaseFilterFactory"/>
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
+
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/basque/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/basque/schema.xml
@@ -184,4 +184,26 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/brazilian_portuguese/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/brazilian_portuguese/schema.xml
@@ -187,4 +187,27 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
+
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/bulgarian/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/bulgarian/schema.xml
@@ -189,4 +189,29 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.BulgarianStemFilterFactory"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.BulgarianStemFilterFactory"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
+
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/burmese/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/burmese/schema.xml
@@ -105,4 +105,17 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.ICUTokenizerFactory"/>
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.FlattenGraphFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.ICUTokenizerFactory" />
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+		</analyzer>
+	</fieldType>
+
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/catalan/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/catalan/schema.xml
@@ -182,4 +182,27 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
+
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/chinese/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/chinese/schema.xml
@@ -116,4 +116,21 @@
 			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
 		</analyzer>
 	</fieldType>
+
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+			<filter class="solr.CJKWidthFilterFactory"/>
+			<filter class="solr.CJKBigramFilterFactory" han="true" hiragana="true" katakana="true" hangul="true" outputUnigrams="false"/>
+			<filter class="solr.LowerCaseFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+			<filter class="solr.CJKWidthFilterFactory"/>
+			<filter class="solr.CJKBigramFilterFactory" han="true" hiragana="true" katakana="true" hangul="true" outputUnigrams="false" />
+			<filter class="solr.LowerCaseFilterFactory"/>
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+		</analyzer>
+	</fieldType>
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/czech/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/czech/schema.xml
@@ -184,4 +184,26 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.CzechStemFilterFactory"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+
+			<filter class="solr.CzechStemFilterFactory"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/danish/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/danish/schema.xml
@@ -213,4 +213,24 @@
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/dutch/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/dutch/schema.xml
@@ -176,7 +176,7 @@
 			<filter class="solr.LowerCaseFilterFactory"/>
 
 			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
-			<filter class="solr.ASCIIFoldingFilterFactory"/>
+			<filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>
 			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 		</analyzer>
 		<analyzer type="query">
@@ -186,9 +186,29 @@
 
 			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
 			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
-			<filter class="solr.ASCIIFoldingFilterFactory"/>
+			<filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>
 			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
 </schema>
diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/english/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/english/schema.xml
@@ -177,7 +177,30 @@
 
 			<!-- no synonyms here because we do not want to add them as spell suggestion -->
 			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
-			<filter class="solr.ASCIIFoldingFilterFactory"/>
+			<filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
+
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<!-- no synonyms here because we do not want to add them as spell suggestion -->
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
 			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 		</analyzer>
 		<analyzer type="query">
@@ -187,7 +210,6 @@
 
 			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
 			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
-			<filter class="solr.ASCIIFoldingFilterFactory"/>
 			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 		</analyzer>
 	</fieldType>

diff --git a/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/finnish/schema.xml b/Resources/Private/Solr/configsets/ext_solr_11_1_0/conf/finnish/schema.xml
@@ -176,7 +176,7 @@
 			<filter class="solr.LowerCaseFilterFactory"/>
 
 			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
-			<filter class="solr.ASCIIFoldingFilterFactory"/>
+			<filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>
 			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 		</analyzer>
 		<analyzer type="query">
@@ -186,9 +186,29 @@
 
 			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
 			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
-			<filter class="solr.ASCIIFoldingFilterFactory"/>
+			<filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true"/>
 			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 		</analyzer>
 	</fieldType>
 
+	<!-- Setup simple analysis for more exact spell checking, considers non-ascii charaters as they are -->
+	<fieldType name="textSpellExact" class="solr.TextField" positionIncrementGap="100" omitNorms="true">
+		<analyzer type="index">
+			<tokenizer class="solr.StandardTokenizerFactory"/>
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+		<analyzer type="query">
+			<tokenizer class="solr.StandardTokenizerFactory" />
+
+			<filter class="solr.LowerCaseFilterFactory"/>
+
+			<filter class="solr.ManagedSynonymGraphFilterFactory" managed="${solr.core.name}" />
+			<filter class="solr.ManagedStopFilterFactory" managed="${solr.core.name}"/>
+			<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+		</analyzer>
+	</fieldType>
 </schema>