From ef18d3fb5b21dd86a56caac0c0d64ad36aa666be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 12 Mar 2019 23:46:55 +0100 Subject: [PATCH] Add analysis modes to restrict token filter use contexts (#36103) Currently token filter settings are treated as fixed once they are declared and used in an analyzer. This is done to prevent changes in analyzers that are already used actively to index documents, since changes to the analysis chain could corrupt the index. However, it would be safe to allow updates to token filters at search time ("search_analyzer"). This change introduces a new property of token filters that allows to mark them as only being usable at search or at index time. Any analyzer that uses these tokenfilters inherits that property and can be rejected if they are used in other contexts. This is a first step towards making specific token filters (e.g. synonym filter) updateable. Relates to #29051 --- .../index/analysis/AnalysisMode.java | 82 +++++++++ .../index/analysis/AnalysisRegistry.java | 5 +- .../index/analysis/CustomAnalyzer.java | 11 ++ .../index/analysis/NamedAnalyzer.java | 48 ++++- .../index/analysis/TokenFilterFactory.java | 9 + .../index/mapper/TypeParsers.java | 20 +++ .../index/analysis/AnalysisRegistryTests.java | 26 +++ .../index/analysis/NamedAnalyzerTests.java | 79 ++++++++ .../index/mapper/TypeParsersTests.java | 169 ++++++++++++++++++ 9 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java create mode 100644 server/src/test/java/org/elasticsearch/index/analysis/NamedAnalyzerTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java new file mode 100644 index 0000000000000..ea9e1e0c6aa7f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +/** + * Enum representing the mode in which token filters and analyzers are allowed to operate. + * While most token filters are allowed both in index and search time analyzers, some are + * restricted to be used only at index time, others at search time. + */ +public enum AnalysisMode { + + /** + * AnalysisMode representing analysis components that can be used only at index time + */ + INDEX_TIME("index time") { + @Override + public AnalysisMode merge(AnalysisMode other) { + if (other == AnalysisMode.SEARCH_TIME) { + throw new IllegalStateException("Cannot merge SEARCH_TIME and INDEX_TIME analysis mode."); + } + return AnalysisMode.INDEX_TIME; + } + }, + /** + * AnalysisMode representing analysis components that can be used only at search time + */ + SEARCH_TIME("search time") { + @Override + public AnalysisMode merge(AnalysisMode other) { + if (other == AnalysisMode.INDEX_TIME) { + throw new IllegalStateException("Cannot merge SEARCH_TIME and INDEX_TIME analysis mode."); + } + return AnalysisMode.SEARCH_TIME; + } + }, + /** + * AnalysisMode representing analysis components that can be used both at index and search time + */ + ALL("all") { + @Override + public AnalysisMode merge(AnalysisMode other) { + return other; + } + }; + + private String readableName; + + AnalysisMode(String name) { + this.readableName = name; + } + + public String getReadableName() { + return this.readableName; + } + + /** + * Returns a mode that is compatible with both this mode and the other mode, that is: + * + */ + abstract AnalysisMode merge(AnalysisMode other); +} diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 165256940bb81..eff594d1d1152 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -20,11 +20,11 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.elasticsearch.Version; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -456,6 +456,7 @@ public IndexAnalyzers build(IndexSettings indexSettings, if (defaultAnalyzer == null) { throw new IllegalArgumentException("no default analyzer configured"); } + defaultAnalyzer.checkAllowedInMode(AnalysisMode.ALL); if (analyzers.containsKey("default_index")) { throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use " + "[index.analysis.analyzer.default] instead for index [" + index.getName() + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index 87a540312b411..a41ee33564400 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -36,6 +36,7 @@ public final class CustomAnalyzer extends Analyzer { private final int positionIncrementGap; private final int offsetGap; + private final AnalysisMode analysisMode; public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, TokenFilterFactory[] tokenFilters) { @@ -50,6 +51,12 @@ public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, C this.tokenFilters = tokenFilters; this.positionIncrementGap = positionIncrementGap; this.offsetGap = offsetGap; + // merge and transfer token filter analysis modes with analyzer + AnalysisMode mode = AnalysisMode.ALL; + for (TokenFilterFactory f : tokenFilters) { + mode = mode.merge(f.getAnalysisMode()); + } + this.analysisMode = mode; } /** @@ -84,6 +91,10 @@ public int getOffsetGap(String field) { return this.offsetGap; } + public AnalysisMode getAnalysisMode() { + return this.analysisMode; + } + @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = tokenizerFactory.create(); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java index 416967e94f5f0..4831d88f3aa1f 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java @@ -21,7 +21,10 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; +import org.elasticsearch.index.mapper.MapperException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** @@ -34,6 +37,7 @@ public class NamedAnalyzer extends DelegatingAnalyzerWrapper { private final AnalyzerScope scope; private final Analyzer analyzer; private final int positionIncrementGap; + private final AnalysisMode analysisMode; public NamedAnalyzer(NamedAnalyzer analyzer, int positionIncrementGap) { this(analyzer.name(), analyzer.scope(), analyzer.analyzer(), positionIncrementGap); @@ -43,12 +47,17 @@ public NamedAnalyzer(String name, AnalyzerScope scope, Analyzer analyzer) { this(name, scope, analyzer, Integer.MIN_VALUE); } - public NamedAnalyzer(String name, AnalyzerScope scope, Analyzer analyzer, int positionIncrementGap) { + NamedAnalyzer(String name, AnalyzerScope scope, Analyzer analyzer, int positionIncrementGap) { super(ERROR_STRATEGY); this.name = name; this.scope = scope; this.analyzer = analyzer; this.positionIncrementGap = positionIncrementGap; + if (analyzer instanceof org.elasticsearch.index.analysis.CustomAnalyzer) { + this.analysisMode = ((org.elasticsearch.index.analysis.CustomAnalyzer) analyzer).getAnalysisMode(); + } else { + this.analysisMode = AnalysisMode.ALL; + } } /** @@ -65,6 +74,13 @@ public AnalyzerScope scope() { return this.scope; } + /** + * Returns whether this analyzer can be updated + */ + public AnalysisMode getAnalysisMode() { + return this.analysisMode; + } + /** * The actual analyzer. */ @@ -85,9 +101,37 @@ public int getPositionIncrementGap(String fieldName) { return super.getPositionIncrementGap(fieldName); } + /** + * Checks the wrapped analyzer for the provided restricted {@link AnalysisMode} and throws + * an error if the analyzer is not allowed to run in that mode. The error contains more detailed information about + * the offending filters that caused the analyzer to not be allowed in this mode. + */ + public void checkAllowedInMode(AnalysisMode mode) { + Objects.requireNonNull(mode); + if (this.getAnalysisMode() == AnalysisMode.ALL) { + return; // everything allowed if this analyzer is in ALL mode + } + if (this.getAnalysisMode() != mode) { + if (analyzer instanceof CustomAnalyzer) { + TokenFilterFactory[] tokenFilters = ((CustomAnalyzer) analyzer).tokenFilters(); + List offendingFilters = new ArrayList<>(); + for (TokenFilterFactory tokenFilter : tokenFilters) { + if (tokenFilter.getAnalysisMode() != mode) { + offendingFilters.add(tokenFilter.name()); + } + } + throw new MapperException("analyzer [" + name + "] contains filters " + offendingFilters + + " that are not allowed to run in " + mode.getReadableName() + " mode."); + } else { + throw new MapperException( + "analyzer [" + name + "] contains components that are not allowed to run in " + mode.getReadableName() + " mode."); + } + } + } + @Override public String toString() { - return "analyzer name[" + name + "], analyzer [" + analyzer + "]"; + return "analyzer name[" + name + "], analyzer [" + analyzer + "], analysisMode [" + analysisMode + "]"; } /** It is an error if this is ever used, it means we screwed up! */ diff --git a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java index b7ed6fd9e9e24..3dbd571288575 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java @@ -74,6 +74,15 @@ default TokenFilterFactory getSynonymFilter() { return this; } + /** + * Get the {@link AnalysisMode} this filter is allowed to be used in. The default is + * {@link AnalysisMode#ALL}. Instances need to override this method to define their + * own restrictions. + */ + default AnalysisMode getAnalysisMode() { + return AnalysisMode.ALL; + } + /** * A TokenFilterFactory that does no filtering to its TokenStream */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index 8cf66009ea140..77d7be62fc1b9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -80,6 +81,7 @@ private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, St if (analyzer == null) { throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } + analyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME); searchAnalyzer = analyzer; iterator.remove(); } else if (propName.equals("search_quote_analyzer")) { @@ -87,11 +89,29 @@ private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, St if (analyzer == null) { throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } + analyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME); searchQuoteAnalyzer = analyzer; iterator.remove(); } } + // check analyzers are allowed to work in the respective AnalysisMode + { + if (indexAnalyzer != null) { + if (searchAnalyzer == null) { + indexAnalyzer.checkAllowedInMode(AnalysisMode.ALL); + } else { + indexAnalyzer.checkAllowedInMode(AnalysisMode.INDEX_TIME); + } + } + if (searchAnalyzer != null) { + searchAnalyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME); + } + if (searchQuoteAnalyzer != null) { + searchQuoteAnalyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME); + } + } + if (indexAnalyzer == null && searchAnalyzer != null) { throw new MapperParsingException("analyzer on field [" + name + "] must be set when search_analyzer is set"); } diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 04dc98deb7bf5..b836a5d0372b8 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -20,6 +20,8 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.en.EnglishAnalyzer; @@ -31,6 +33,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; @@ -102,6 +105,29 @@ public void testOverrideDefaultAnalyzer() throws IOException { assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); } + public void testOverrideDefaultAnalyzerWithoutAnalysisModeAll() throws IOException { + Version version = VersionUtils.randomVersion(random()); + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + TokenFilterFactory tokenFilter = new AbstractTokenFilterFactory(IndexSettingsModule.newIndexSettings("index", settings), + "my_filter", Settings.EMPTY) { + @Override + public AnalysisMode getAnalysisMode() { + return randomFrom(AnalysisMode.SEARCH_TIME, AnalysisMode.INDEX_TIME); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return null; + } + }; + Analyzer analyzer = new CustomAnalyzer("tokenizerName", null, new CharFilterFactory[0], new TokenFilterFactory[] { tokenFilter }); + MapperException ex = expectThrows(MapperException.class, + () -> emptyRegistry.build(IndexSettingsModule.newIndexSettings("index", settings), + singletonMap("default", new PreBuiltAnalyzerProvider("my_analyzer", AnalyzerScope.INDEX, analyzer)), emptyMap(), + emptyMap(), emptyMap(), emptyMap())); + assertEquals("analyzer [my_analyzer] contains filters [my_filter] that are not allowed to run in all mode.", ex.getMessage()); + } + public void testOverrideDefaultIndexAnalyzerIsUnsupported() { Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_alpha1, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); diff --git a/server/src/test/java/org/elasticsearch/index/analysis/NamedAnalyzerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/NamedAnalyzerTests.java new file mode 100644 index 0000000000000..e0f4a37c57fa2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/analysis/NamedAnalyzerTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.elasticsearch.index.mapper.MapperException; +import org.elasticsearch.test.ESTestCase; + +public class NamedAnalyzerTests extends ESTestCase { + + public void testCheckAllowedInMode() { + try (NamedAnalyzer testAnalyzer = new NamedAnalyzer("my_analyzer", AnalyzerScope.INDEX, + createAnalyzerWithMode("my_analyzer", AnalysisMode.INDEX_TIME), Integer.MIN_VALUE)) { + testAnalyzer.checkAllowedInMode(AnalysisMode.INDEX_TIME); + MapperException ex = expectThrows(MapperException.class, () -> testAnalyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME)); + assertEquals("analyzer [my_analyzer] contains filters [my_analyzer] that are not allowed to run in search time mode.", + ex.getMessage()); + ex = expectThrows(MapperException.class, () -> testAnalyzer.checkAllowedInMode(AnalysisMode.ALL)); + assertEquals("analyzer [my_analyzer] contains filters [my_analyzer] that are not allowed to run in all mode.", ex.getMessage()); + } + + try (NamedAnalyzer testAnalyzer = new NamedAnalyzer("my_analyzer", AnalyzerScope.INDEX, + createAnalyzerWithMode("my_analyzer", AnalysisMode.SEARCH_TIME), Integer.MIN_VALUE)) { + testAnalyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME); + MapperException ex = expectThrows(MapperException.class, () -> testAnalyzer.checkAllowedInMode(AnalysisMode.INDEX_TIME)); + assertEquals("analyzer [my_analyzer] contains filters [my_analyzer] that are not allowed to run in index time mode.", + ex.getMessage()); + ex = expectThrows(MapperException.class, () -> testAnalyzer.checkAllowedInMode(AnalysisMode.ALL)); + assertEquals("analyzer [my_analyzer] contains filters [my_analyzer] that are not allowed to run in all mode.", ex.getMessage()); + } + + try (NamedAnalyzer testAnalyzer = new NamedAnalyzer("my_analyzer", AnalyzerScope.INDEX, + createAnalyzerWithMode("my_analyzer", AnalysisMode.ALL), Integer.MIN_VALUE)) { + testAnalyzer.checkAllowedInMode(AnalysisMode.ALL); + testAnalyzer.checkAllowedInMode(AnalysisMode.INDEX_TIME); + testAnalyzer.checkAllowedInMode(AnalysisMode.SEARCH_TIME); + } + } + + private Analyzer createAnalyzerWithMode(String name, AnalysisMode mode) { + TokenFilterFactory tokenFilter = new TokenFilterFactory() { + + @Override + public String name() { + return name; + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return null; + } + + @Override + public AnalysisMode getAnalysisMode() { + return mode; + } + }; + return new CustomAnalyzer("tokenizerName", null, new CharFilterFactory[0], + new TokenFilterFactory[] { tokenFilter }); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java new file mode 100644 index 0000000000000..a2966053ae7dc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -0,0 +1,169 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.AnalysisMode; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.CustomAnalyzer; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TypeParsersTests extends ESTestCase { + + private static final IndexMetaData EMPTY_INDEX_METADATA = IndexMetaData.builder("") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(0).build(); + private static final IndexSettings indexSettings = new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY); + + public void testParseTextFieldCheckAnalyzerAnalysisMode() { + TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField"); + Map fieldNode = new HashMap(); + fieldNode.put("analyzer", "my_analyzer"); + Mapper.TypeParser.ParserContext parserContext = mock(Mapper.TypeParser.ParserContext.class); + + // check AnalysisMode.ALL works + Map analyzers = new HashMap<>(); + analyzers.put("my_analyzer", + new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", AnalysisMode.ALL))); + + IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, + new NamedAnalyzer("default", AnalyzerScope.INDEX, null), null, null, analyzers, null, null); + when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers); + TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext); + + // check that "analyzer" set to something that only supports AnalysisMode.SEARCH_TIME or AnalysisMode.INDEX_TIME is blocked + AnalysisMode mode = randomFrom(AnalysisMode.SEARCH_TIME, AnalysisMode.INDEX_TIME); + analyzers = new HashMap<>(); + analyzers.put("my_analyzer", new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, + createAnalyzerWithMode("my_analyzer", mode))); + indexAnalyzers = new IndexAnalyzers(indexSettings, new NamedAnalyzer("default", AnalyzerScope.INDEX, null), null, null, analyzers, + null, null); + when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers); + MapperException ex = expectThrows(MapperException.class, + () -> TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext)); + assertEquals("analyzer [my_named_analyzer] contains filters [my_analyzer] that are not allowed to run in all mode.", + ex.getMessage()); + } + + public void testParseTextFieldCheckSearchAnalyzerAnalysisMode() { + TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField"); + for (String settingToTest : new String[] { "search_analyzer", "search_quote_analyzer" }) { + Map fieldNode = new HashMap(); + fieldNode.put(settingToTest, "my_analyzer"); + fieldNode.put("analyzer", "standard"); + if (settingToTest.equals("search_quote_analyzer")) { + fieldNode.put("search_analyzer", "standard"); + } + Mapper.TypeParser.ParserContext parserContext = mock(Mapper.TypeParser.ParserContext.class); + + // check AnalysisMode.ALL and AnalysisMode.SEARCH_TIME works + Map analyzers = new HashMap<>(); + AnalysisMode mode = randomFrom(AnalysisMode.ALL, AnalysisMode.SEARCH_TIME); + analyzers.put("my_analyzer", + new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode))); + analyzers.put("standard", new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); + + IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, new NamedAnalyzer("default", AnalyzerScope.INDEX, null), null, + null, analyzers, null, null); + when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers); + TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext); + + // check that "analyzer" set to AnalysisMode.INDEX_TIME is blocked + mode = AnalysisMode.INDEX_TIME; + analyzers = new HashMap<>(); + analyzers.put("my_analyzer", + new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode))); + analyzers.put("standard", new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); + indexAnalyzers = new IndexAnalyzers(indexSettings, new NamedAnalyzer("default", AnalyzerScope.INDEX, null), null, null, + analyzers, null, null); + when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers); + MapperException ex = expectThrows(MapperException.class, + () -> TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext)); + assertEquals("analyzer [my_named_analyzer] contains filters [my_analyzer] that are not allowed to run in search time mode.", + ex.getMessage()); + } + } + + public void testParseTextFieldCheckAnalyzerWithSearchAnalyzerAnalysisMode() { + TextFieldMapper.Builder builder = new TextFieldMapper.Builder("textField"); + Map fieldNode = new HashMap(); + fieldNode.put("analyzer", "my_analyzer"); + Mapper.TypeParser.ParserContext parserContext = mock(Mapper.TypeParser.ParserContext.class); + + // check that "analyzer" set to AnalysisMode.INDEX_TIME is blocked if there is no search analyzer + AnalysisMode mode = AnalysisMode.INDEX_TIME; + Map analyzers = new HashMap<>(); + analyzers.put("my_analyzer", + new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode))); + IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, new NamedAnalyzer("default", AnalyzerScope.INDEX, null), null, + null, analyzers, null, null); + when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers); + MapperException ex = expectThrows(MapperException.class, + () -> TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext)); + assertEquals("analyzer [my_named_analyzer] contains filters [my_analyzer] that are not allowed to run in all mode.", + ex.getMessage()); + + // check AnalysisMode.INDEX_TIME is okay if search analyzer is also set + fieldNode.put("search_analyzer", "standard"); + analyzers = new HashMap<>(); + mode = randomFrom(AnalysisMode.ALL, AnalysisMode.INDEX_TIME); + analyzers.put("my_analyzer", + new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode))); + analyzers.put("standard", new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); + + indexAnalyzers = new IndexAnalyzers(indexSettings, new NamedAnalyzer("default", AnalyzerScope.INDEX, null), null, null, analyzers, + null, null); + when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers); + TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext); + } + + private Analyzer createAnalyzerWithMode(String name, AnalysisMode mode) { + TokenFilterFactory tokenFilter = new AbstractTokenFilterFactory(indexSettings, name, Settings.EMPTY) { + @Override + public AnalysisMode getAnalysisMode() { + return mode; + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return null; + } + }; + return new CustomAnalyzer("tokenizerName", null, new CharFilterFactory[0], + new TokenFilterFactory[] { tokenFilter }); + } +}