Skip to content

Commit

Permalink
Register smartcn analyzer, tokenizer and tokenfilter
Browse files Browse the repository at this point in the history
When the plugin starts, it should register `smartcn` analyzer, `smartcn_sentence` tokenizer and `smartcn_word` token filter.

Closes #12.
(cherry picked from commit b8cd4c4)
  • Loading branch information
dadoonet committed Mar 26, 2014
1 parent 9e780fa commit 9308c0a
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.indices.analysis.smartcn;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.cn.smart.WordTokenFilter;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.analysis.*;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;

import java.io.Reader;

/**
* Registers indices level analysis components so, if not explicitly configured, will be shared
* among all indices.
*/
public class SmartChineseIndicesAnalysis extends AbstractComponent {

@Inject
public SmartChineseIndicesAnalysis(Settings settings, IndicesAnalysisService indicesAnalysisService) {
super(settings);

// Register smartcn analyzer
indicesAnalysisService.analyzerProviderFactories().put("smartcn", new PreBuiltAnalyzerProviderFactory("smartcn", AnalyzerScope.INDICES, new SmartChineseAnalyzer(Lucene.ANALYZER_VERSION)));

// Register smartcn_word token filter
indicesAnalysisService.tokenFilterFactories().put("smartcn_word", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
@Override public String name() {
return "smartcn_word";
}

@Override public TokenStream create(TokenStream tokenStream) {
return new WordTokenFilter(tokenStream);
}
}));

// Register smartcn_sentence tokenizer
indicesAnalysisService.tokenizerFactories().put("smartcn_sentence", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() {
@Override
public String name() {
return "smartcn_sentence";
}

@Override
public Tokenizer create(Reader reader) {
return new SentenceTokenizer(reader);
}
}));


}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.indices.analysis.smartcn;

import org.elasticsearch.common.inject.AbstractModule;

/**
*/
public class SmartChineseIndicesAnalysisModule extends AbstractModule {

@Override
protected void configure() {
bind(SmartChineseIndicesAnalysis.class).asEagerSingleton();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@

package org.elasticsearch.plugin.analysis.smartcn;

import org.elasticsearch.common.collect.ImmutableList;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.SmartChineseAnalysisBinderProcessor;
import org.elasticsearch.indices.analysis.smartcn.SmartChineseIndicesAnalysisModule;
import org.elasticsearch.plugins.AbstractPlugin;

import java.util.Collection;

/**
*
*/
Expand All @@ -38,6 +43,11 @@ public String description() {
return "Smart Chinese analysis support";
}

@Override
public Collection<Class<? extends Module>> modules() {
return ImmutableList.<Class<? extends Module>>of(SmartChineseIndicesAnalysisModule.class);
}

public void onModule(AnalysisModule module) {
module.addProcessor(new SmartChineseAnalysisBinderProcessor());
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Licensed to Elasticsearch (the "Author") under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Author licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.analysis;

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;

import java.util.concurrent.ExecutionException;

import static org.hamcrest.CoreMatchers.*;

@ElasticsearchIntegrationTest.ClusterScope(numNodes = 1, scope = ElasticsearchIntegrationTest.Scope.SUITE)
public class SimpleSmartChineseIntegrationTests extends ElasticsearchIntegrationTest {

@Test
public void testSmartcnAnalyzer() throws ExecutionException, InterruptedException {
AnalyzeResponse response = client().admin().indices()
.prepareAnalyze("叻出色").setAnalyzer("smartcn")
.execute().get();

assertThat(response, notNullValue());
assertThat(response.getTokens().size(), is(2));
}

@Test
public void testSmartcnTokenizer() throws ExecutionException, InterruptedException {
AnalyzeResponse response = client().admin().indices()
.prepareAnalyze("叻出色").setTokenizer("smartcn_sentence")
.execute().get();

assertThat(response, notNullValue());
assertThat(response.getTokens().size(), is(1));
}

@Test
public void testSmartcnTokenFilter() throws ExecutionException, InterruptedException {
AnalyzeResponse response = client().admin().indices()
.prepareAnalyze("叻出色").setTokenFilters("smartcn_word")
.execute().get();

assertThat(response, notNullValue());
assertThat(response.getTokens().size(), is(3));
}
}

0 comments on commit 9308c0a

Please sign in to comment.