From ab99b96dd11bd8f0c427daaa776439f5a40eb93a Mon Sep 17 00:00:00 2001 From: Umesh Dangat Date: Sat, 6 Apr 2019 16:44:07 -0700 Subject: [PATCH 1/4] update for ES7.0.0-rc2. Still have a few Todos and some failing tests --- build.gradle | 10 ++--- gradle/wrapper/gradle-wrapper.properties | 2 +- .../com/o19s/es/explore/ExplorerQuery.java | 43 ++++++++++++------- .../com/o19s/es/explore/ExplorerScorer.java | 13 +++++- .../es/explore/PostingsExplorerQuery.java | 41 ++++++++++++------ .../com/o19s/es/ltr/LtrQueryParserPlugin.java | 6 ++- .../es/ltr/action/AddFeaturesToSetAction.java | 13 ++---- .../o19s/es/ltr/action/CachesStatsAction.java | 22 ++++------ .../o19s/es/ltr/action/ClearCachesAction.java | 13 ++---- .../ltr/action/CreateModelFromSetAction.java | 12 ++---- .../es/ltr/action/FeatureStoreAction.java | 12 ++---- .../o19s/es/ltr/action/ListStoresAction.java | 22 ++++------ .../TransportAddFeatureToSetAction.java | 16 +++---- .../ltr/action/TransportCacheStatsAction.java | 6 +-- .../action/TransportClearCachesAction.java | 2 +- .../TransportCreateModelFromSetAction.java | 8 +--- .../action/TransportFeatureStoreAction.java | 10 +---- .../ltr/action/TransportListStoresAction.java | 4 +- .../o19s/es/ltr/feature/PrebuiltFeature.java | 5 ++- .../es/ltr/feature/store/ScriptFeature.java | 18 ++++++-- .../es/ltr/feature/store/StoredFeature.java | 2 +- .../es/ltr/logging/LoggingFetchSubPhase.java | 3 +- .../es/ltr/query/DerivedExpressionQuery.java | 19 ++++++-- .../com/o19s/es/ltr/query/NoopScorer.java | 11 +++++ .../com/o19s/es/ltr/query/RankerQuery.java | 23 +++++++--- .../ranker/ranklib/RankLibScriptEngine.java | 3 +- .../o19s/es/ltr/rest/RestAddFeatureToSet.java | 3 +- .../es/ltr/rest/RestCreateModelFromSet.java | 2 +- .../es/ltr/rest/RestFeatureStoreCaches.java | 7 +-- .../es/ltr/rest/RestSimpleFeatureStore.java | 10 +++-- .../o19s/es/explore/ExplorerQueryTests.java | 4 +- .../ltr/action/AddFeaturesToSetActionIT.java | 22 +++++----- .../es/ltr/action/BaseIntegrationTest.java | 8 ++-- .../es/ltr/action/ListStoresActionIT.java | 2 +- .../ValidatingFeatureStoreActionIT.java | 5 ++- .../logging/LoggingFetchSubPhaseTests.java | 13 ++++-- .../es/ltr/query/LtrQueryBuilderTests.java | 5 --- .../com/o19s/es/ltr/query/LtrQueryTests.java | 21 +++++---- .../ltr/query/StoredLtrQueryBuilderTests.java | 6 --- .../o19s/es/ltr/query/StoredLtrQueryIT.java | 18 ++++---- 40 files changed, 252 insertions(+), 213 deletions(-) diff --git a/build.gradle b/build.gradle index 7e64db08..dd246e9d 100644 --- a/build.gradle +++ b/build.gradle @@ -6,12 +6,12 @@ buildscript { } dependencies { - classpath "org.elasticsearch.gradle:build-tools:6.6.0" + classpath "org.elasticsearch.gradle:build-tools:7.0.0-rc2" } } group = 'com.o19s' -version = '1.1.0-es6.6.0' +version = '1.1.0-es7.0.0-rc2' apply plugin: 'java' apply plugin: 'idea' @@ -46,15 +46,15 @@ repositories { } dependencies { - compile "org.apache.lucene:lucene-expressions:7.6.0" + compile "org.apache.lucene:lucene-expressions:8.0.0" compile 'org.antlr:antlr4-runtime:4.5.1-1' compile 'org.ow2.asm:asm:5.0.4' compile 'org.ow2.asm:asm-commons:5.0.4' compile 'org.ow2.asm:asm-tree:5.0.4' - compile 'org.elasticsearch:elasticsearch:6.6.0' + compile 'org.elasticsearch:elasticsearch:7.0.0-rc2' compile 'com.o19s:RankyMcRankFace:0.1.1' compile "com.github.spullara.mustache.java:compiler:0.9.3" - testCompile 'org.elasticsearch.test:framework:6.6.0' + testCompile 'org.elasticsearch.test:framework:7.0.0-rc2' } dependencyLicenses { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 25fe2a65..663c4485 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-bin.zip diff --git a/src/main/java/com/o19s/es/explore/ExplorerQuery.java b/src/main/java/com/o19s/es/explore/ExplorerQuery.java index 450047bb..1ad1b4a0 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/ExplorerQuery.java @@ -13,12 +13,13 @@ * limitations under the License. * */ + package com.o19s.es.explore; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; +import org.apache.lucene.index.TermStates; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreScorer; @@ -27,6 +28,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; @@ -53,8 +55,13 @@ private boolean isCollectionScoped() { || type.endsWith(("_ttf")); } - public Query getQuery() { return this.query; } - public String getType() { return this.type; } + public Query getQuery() { + return this.query; + } + + public String getType() { + return this.type; + } @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override @@ -72,7 +79,7 @@ private boolean equalsTo(ExplorerQuery other) { public Query rewrite(IndexReader reader) throws IOException { Query rewritten = query.rewrite(reader); - if(rewritten != query) { + if (rewritten != query) { return new ExplorerQuery(rewritten, type); } @@ -85,21 +92,24 @@ public int hashCode() { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - if (!needsScores) { - return searcher.createWeight(query, false, boost); + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + if (!scoreMode.needsScores()) { + return searcher.createWeight(query, scoreMode, boost); } - final Weight subWeight = searcher.createWeight(query, true, boost); + final Weight subWeight = searcher.createWeight(query, scoreMode, boost); Set terms = new HashSet<>(); subWeight.extractTerms(terms); - if(isCollectionScoped()) { + if (isCollectionScoped()) { ClassicSimilarity sim = new ClassicSimilarity(); StatisticsHelper df_stats = new StatisticsHelper(); StatisticsHelper idf_stats = new StatisticsHelper(); StatisticsHelper ttf_stats = new StatisticsHelper(); - for(Term term : terms) { - TermContext ctx = TermContext.build(searcher.getTopReaderContext(), term); + for (Term term : terms) { + TermStates + ctx = + TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores()); TermStatistics tStats = searcher.termStatistics(term, ctx); df_stats.add(tStats.docFreq()); idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs())); @@ -113,7 +123,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores, float bo */ float constantScore; - if(terms.size() > 0) { + if (terms.size() > 0) { switch (type) { case ("sum_classic_idf"): constantScore = idf_stats.getSum(); @@ -185,7 +195,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio @Override public Scorer scorer(LeafReaderContext context) throws IOException { - return new ConstantScoreScorer(this, constantScore, DocIdSetIterator.all(context.reader().maxDoc())); + return new ConstantScoreScorer(this, constantScore, scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); } @Override @@ -198,15 +208,16 @@ public boolean isCacheable(LeafReaderContext ctx) { // Rewrite this into a boolean query where we can inject our PostingsExplorerQuery BooleanQuery.Builder qb = new BooleanQuery.Builder(); for (Term t : terms) { - qb.add(new BooleanClause(new PostingsExplorerQuery(t, PostingsExplorerQuery.Type.TF), BooleanClause.Occur.SHOULD)); + qb.add(new BooleanClause(new PostingsExplorerQuery(t, PostingsExplorerQuery.Type.TF), + BooleanClause.Occur.SHOULD)); } // FIXME: completely refactor this class and stop accepting a random query but a list of terms directly // rewriting at this point is wrong, additionally we certainly build the TermContext twice for every terms // problem is that we rely on extractTerms which happen too late in the process Query q = qb.build().rewrite(searcher.getIndexReader()); - return new ExplorerQuery.ExplorerWeight(this, searcher.createWeight(q, true, boost), type); + return new ExplorerQuery.ExplorerWeight(this, searcher.createWeight(q, scoreMode, boost), type); } - throw new IllegalArgumentException( "Unknown ExplorerQuery type [" + type + "]" ); + throw new IllegalArgumentException("Unknown ExplorerQuery type [" + type + "]"); } static class ExplorerWeight extends Weight { diff --git a/src/main/java/com/o19s/es/explore/ExplorerScorer.java b/src/main/java/com/o19s/es/explore/ExplorerScorer.java index f3e2a7bf..8bc686ba 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerScorer.java +++ b/src/main/java/com/o19s/es/explore/ExplorerScorer.java @@ -37,7 +37,7 @@ public float score() throws IOException { // Grab freq from subscorer, or the children if available if(subScorer.getChildren().size() > 0) { - for(ChildScorer child : subScorer.getChildren()) { + for(ChildScorable child : subScorer.getChildren()) { assert child.child instanceof PostingsExplorerQuery.PostingsExplorerScorer; if(child.child.docID() == docID()) { tf_stats.add(child.child.score()); @@ -83,4 +83,15 @@ public int docID() { public DocIdSetIterator iterator() { return subScorer.iterator(); } + + /** + * Return the maximum score that documents between the last {@code target} + * that this iterator was {@link #advanceShallow(int) shallow-advanced} to + * included and {@code upTo} included. + */ + @Override + public float getMaxScore(int upTo) throws IOException { + //TODO?? + return Float.POSITIVE_INFINITY; + } } diff --git a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java index 904ca31f..7fa183c8 100644 --- a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java @@ -22,13 +22,14 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -63,8 +64,8 @@ public String toString(String field) { @Override public boolean equals(Object obj) { return this.sameClassAs(obj) - && this.term.equals(((PostingsExplorerQuery)obj).term) - && this.type.equals(((PostingsExplorerQuery)obj).type); + && this.term.equals(((PostingsExplorerQuery) obj).term) + && this.type.equals(((PostingsExplorerQuery) obj).type); } @Override @@ -73,10 +74,13 @@ public int hashCode() { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { IndexReaderContext context = searcher.getTopReaderContext(); - assert needsScores : "Should not be used in filtering mode"; - return new PostingsExplorerWeight(this, this.term, TermContext.build(context, this.term), this.type); + assert scoreMode.needsScores() : "Should not be used in filtering mode"; + return new PostingsExplorerWeight(this, this.term, TermStates.build(context, this.term, + scoreMode.needsScores()), + this.type); } /** @@ -100,10 +104,10 @@ public Scorer apply(Weight weight, TermsEnum termsEnum) throws IOException { static class PostingsExplorerWeight extends Weight { private final Term term; - private final TermContext termStates; + private final TermStates termStates; private final Type type; - PostingsExplorerWeight(Query query, Term term, TermContext termStates, Type type) { + PostingsExplorerWeight(Query query, Term term, TermStates termStates, Type type) { super(query); this.term = term; this.termStates = termStates; @@ -120,15 +124,17 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio Scorer scorer = this.scorer(context); int newDoc = scorer.iterator().advance(doc); if (newDoc == doc) { - return Explanation.match(scorer.score(), "weight(" + this.getQuery() + " in doc " + newDoc + ")"); + return Explanation + .match(scorer.score(), "weight(" + this.getQuery() + " in doc " + newDoc + ")"); } - return Explanation.noMatch("no matching term" ); + return Explanation.noMatch("no matching term"); } @Override public Scorer scorer(LeafReaderContext context) throws IOException { - assert this.termStates != null && this.termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context)); - TermState state = this.termStates.get(context.ord); + assert this.termStates != null && this.termStates + .wasBuiltFor(ReaderUtil.getTopLevelContext(context)); + TermState state = this.termStates.get(context); if (state == null) { return null; } else { @@ -172,5 +178,16 @@ static class TFScorer extends PostingsExplorerScorer { public float score() throws IOException { return this.postingsEnum.freq(); } + + /** + * Return the maximum score that documents between the last {@code target} + * that this iterator was {@link #advanceShallow(int) shallow-advanced} to + * included and {@code upTo} included. + */ + @Override + public float getMaxScore(int upTo) throws IOException { + //TODO? + return Float.POSITIVE_INFINITY; + } } } diff --git a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java index be688f19..615f4160 100644 --- a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java +++ b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java @@ -16,7 +16,6 @@ */ package com.o19s.es.ltr; -import ciir.umass.edu.learning.RankerFactory; import com.o19s.es.explore.ExplorerQueryBuilder; import com.o19s.es.ltr.action.AddFeaturesToSetAction; import com.o19s.es.ltr.action.CachesStatsAction; @@ -53,6 +52,7 @@ import com.o19s.es.ltr.rest.RestSimpleFeatureStore; import com.o19s.es.ltr.utils.FeatureStoreLoader; import com.o19s.es.ltr.utils.Suppliers; + import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.miscellaneous.LengthFilter; import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter; @@ -100,6 +100,8 @@ import java.util.List; import java.util.function.Supplier; +import ciir.umass.edu.learning.RankerFactory; + import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.Collections.unmodifiableList; @@ -253,6 +255,6 @@ public List getPreConfiguredTokenFilters() { public List getPreConfiguredTokenizers() { return Collections.singletonList(PreConfiguredTokenizer.singleton("ltr_keyword", - () -> new KeywordTokenizer(KeywordTokenizer.DEFAULT_BUFFER_SIZE), null)); + () -> new KeywordTokenizer(KeywordTokenizer.DEFAULT_BUFFER_SIZE))); } } diff --git a/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java b/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java index 67df5018..9ab6e7ef 100644 --- a/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java @@ -36,8 +36,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class AddFeaturesToSetAction extends Action { +public class AddFeaturesToSetAction extends Action { public static final AddFeaturesToSetAction INSTANCE = new AddFeaturesToSetAction(); public static final String NAME = "cluster:admin/ltr/store/add-features-to-set"; @@ -45,19 +44,13 @@ protected AddFeaturesToSetAction() { super(NAME); } - @Override - public AddFeaturesToSetRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new AddFeaturesToSetRequestBuilder(client); - } - @Override public AddFeaturesToSetResponse newResponse() { return new AddFeaturesToSetResponse(); } - public static class AddFeaturesToSetRequestBuilder extends ActionRequestBuilder { - protected AddFeaturesToSetRequestBuilder(ElasticsearchClient client) { + public static class AddFeaturesToSetRequestBuilder extends ActionRequestBuilder { + public AddFeaturesToSetRequestBuilder(ElasticsearchClient client) { super(client, INSTANCE, new AddFeaturesToSetRequest()); } } diff --git a/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java b/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java index 5a5a3a9b..432a1069 100644 --- a/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java +++ b/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java @@ -37,8 +37,7 @@ import java.util.List; import java.util.Map; -public class CachesStatsAction extends Action { +public class CachesStatsAction extends Action { public static final String NAME = "cluster:admin/ltr/caches/stats"; public static final CachesStatsAction INSTANCE = new CachesStatsAction(); @@ -46,22 +45,11 @@ protected CachesStatsAction() { super(NAME); } - @Override - public CacheStatsRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new CacheStatsRequestBuilder(client); - } - @Override public CachesStatsNodesResponse newResponse() { return new CachesStatsNodesResponse(); } - public static class CacheStatsRequestBuilder extends ActionRequestBuilder { - protected CacheStatsRequestBuilder(ElasticsearchClient client) { - super(client, INSTANCE, new CachesStatsNodesRequest()); - } - } public static class CachesStatsNodesRequest extends BaseNodesRequest { } @@ -309,4 +297,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } } + + public static class CachesStatsActionBuilder extends + ActionRequestBuilder { + public CachesStatsActionBuilder(ElasticsearchClient client){ + super(client, INSTANCE, new CachesStatsNodesRequest()); + } + } + } diff --git a/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java b/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java index a4ac4e22..b197317c 100644 --- a/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java +++ b/src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java @@ -35,8 +35,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class ClearCachesAction extends Action { +public class ClearCachesAction extends Action { public static final String NAME = "cluster:admin/ltr/caches"; public static final ClearCachesAction INSTANCE = new ClearCachesAction(); @@ -44,19 +43,13 @@ private ClearCachesAction() { super(NAME); } - @Override - public RequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new RequestBuilder(client); - } - @Override public ClearCachesNodesResponse newResponse() { return new ClearCachesNodesResponse(); } - public static class RequestBuilder extends ActionRequestBuilder { - RequestBuilder(ElasticsearchClient client) { + public static class RequestBuilder extends ActionRequestBuilder { + public RequestBuilder(ElasticsearchClient client) { super(client, ClearCachesAction.INSTANCE, new ClearCachesNodesRequest()); } } diff --git a/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java index 3762c68b..a0ba7f56 100644 --- a/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java @@ -35,8 +35,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class CreateModelFromSetAction extends Action { +public class CreateModelFromSetAction extends Action { public static final String NAME = "cluster:admin/ltr/store/create-model-from-set"; public static final CreateModelFromSetAction INSTANCE = new CreateModelFromSetAction(); @@ -44,11 +43,6 @@ protected CreateModelFromSetAction() { super(NAME); } - @Override - public CreateModelFromSetRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new CreateModelFromSetRequestBuilder(client); - } - /** * Creates a new response instance. */ @@ -58,9 +52,9 @@ public CreateModelFromSetResponse newResponse() { } public static class CreateModelFromSetRequestBuilder extends ActionRequestBuilder { + CreateModelFromSetResponse> { - protected CreateModelFromSetRequestBuilder(ElasticsearchClient client) { + public CreateModelFromSetRequestBuilder(ElasticsearchClient client) { super(client, INSTANCE, new CreateModelFromSetRequest()); } diff --git a/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java index 57f64a8a..00cf5b04 100644 --- a/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java +++ b/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java @@ -37,8 +37,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class FeatureStoreAction extends Action { +public class FeatureStoreAction extends Action { public static final String NAME = "cluster:admin/ltr/featurestore/data"; public static final FeatureStoreAction INSTANCE = new FeatureStoreAction(); @@ -51,14 +50,9 @@ public FeatureStoreResponse newResponse() { return new FeatureStoreResponse(); } - @Override - public FeatureStoreRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new FeatureStoreRequestBuilder(client, this); - } - public static class FeatureStoreRequestBuilder - extends ActionRequestBuilder { - FeatureStoreRequestBuilder(ElasticsearchClient client, FeatureStoreAction action) { + extends ActionRequestBuilder { + public FeatureStoreRequestBuilder(ElasticsearchClient client, FeatureStoreAction action) { super(client, action, new FeatureStoreRequest()); } } diff --git a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java index 891298f2..73d0b90d 100644 --- a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java +++ b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java @@ -36,8 +36,7 @@ import java.util.Objects; import java.util.stream.Collectors; -public class ListStoresAction extends Action { +public class ListStoresAction extends Action { public static final String NAME = "cluster:admin/ltr/featurestore/list"; public static final ListStoresAction INSTANCE = new ListStoresAction(); @@ -50,18 +49,6 @@ public ListStoresActionResponse newResponse() { return new ListStoresActionResponse(); } - @Override - public ListStoresActionRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new ListStoresActionRequestBuilder(client); - } - - public static class ListStoresActionRequestBuilder extends ActionRequestBuilder { - protected ListStoresActionRequestBuilder(ElasticsearchClient client) { - super(client, INSTANCE, new ListStoresActionRequest()); - } - } - public static class ListStoresActionRequest extends MasterNodeReadRequest { @Override public ActionRequestValidationException validate() { @@ -155,4 +142,11 @@ public Map getCounts() { return counts; } } + + public static class ListStoresActionBuilder extends + ActionRequestBuilder { + public ListStoresActionBuilder(ElasticsearchClient client, ListStoresAction action){ + super(client, action, new ListStoresActionRequest()); + } + } } diff --git a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java index ea55b264..7c18a8e0 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java @@ -24,7 +24,6 @@ import com.o19s.es.ltr.feature.store.StoredFeature; import com.o19s.es.ltr.feature.store.StoredFeatureSet; import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; -import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -67,19 +66,13 @@ public TransportAddFeatureToSetAction(Settings settings, ThreadPool threadPool, IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, TransportSearchAction searchAction, TransportGetAction getAction, TransportFeatureStoreAction featureStoreAction) { - super(settings, AddFeaturesToSetAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, - AddFeaturesToSetRequest::new); + super(AddFeaturesToSetAction.NAME, transportService, actionFilters, AddFeaturesToSetRequest::new); this.clusterService = clusterService; this.searchAction = searchAction; this.getAction = getAction; this.featureStoreAction = featureStoreAction; } - @Override - protected final void doExecute(AddFeaturesToSetRequest request, ActionListener listener) { - throw new UnsupportedOperationException("attempt to execute a TransportAddFeatureToSetAction without a task"); - } - @Override protected void doExecute(Task task, AddFeaturesToSetRequest request, ActionListener listener) { if (!clusterService.state().routingTable().hasIndex(request.getStore())) { @@ -181,7 +174,8 @@ private void fetchFeaturesFromStore() { srequest.source().query(bq); srequest.source().fetchSource(true); srequest.source().size(StoredFeatureSet.MAX_FEATURES); - ActionFuture resp = searchAction.execute(srequest); + //do we need this? + //ActionFuture resp = searchAction.execute(srequest); searchAction.execute(srequest, wrap(this::onSearchResponse, this::onSearchFailure)); } @@ -215,10 +209,10 @@ private void onGetResponse(GetResponse getResponse) { private void onSearchResponse(SearchResponse sr) { try { - if (sr.getHits().getTotalHits() > StoredFeatureSet.MAX_FEATURES) { + if (sr.getHits().getTotalHits().value > StoredFeatureSet.MAX_FEATURES) { throw new IllegalArgumentException("The feature query [" + featureNamesQuery + "] returns too many features"); } - if (sr.getHits().getTotalHits() == 0) { + if (sr.getHits().getTotalHits().value == 0) { throw new IllegalArgumentException("The feature query [" + featureNamesQuery + "] returned no features"); } final List features = new ArrayList<>(sr.getHits().getHits().length); diff --git a/src/main/java/com/o19s/es/ltr/action/TransportCacheStatsAction.java b/src/main/java/com/o19s/es/ltr/action/TransportCacheStatsAction.java index b8616fca..3291e149 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportCacheStatsAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportCacheStatsAction.java @@ -42,9 +42,9 @@ public TransportCacheStatsAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Caches caches) { - super(settings, CachesStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, - CachesStatsNodesRequest::new, CachesStatsNodeRequest::new, ThreadPool.Names.MANAGEMENT, - CachesStatsAction.CachesStatsNodeResponse.class); + super(CachesStatsAction.NAME, threadPool, clusterService, transportService, + actionFilters, CachesStatsNodesRequest::new, CachesStatsNodeRequest::new, + ThreadPool.Names.MANAGEMENT, CachesStatsAction.CachesStatsNodeResponse.class); this.caches = caches; } diff --git a/src/main/java/com/o19s/es/ltr/action/TransportClearCachesAction.java b/src/main/java/com/o19s/es/ltr/action/TransportClearCachesAction.java index ed976cfa..8d722059 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportClearCachesAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportClearCachesAction.java @@ -45,7 +45,7 @@ public TransportClearCachesAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Caches caches) { - super(settings, ClearCachesAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + super(ClearCachesAction.NAME, threadPool, clusterService, transportService, actionFilters, ClearCachesNodesRequest::new, ClearCachesNodeRequest::new, ThreadPool.Names.MANAGEMENT, ClearCachesNodeResponse.class); this.caches = caches; } diff --git a/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java index 1ec532cb..f6b214f3 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java @@ -50,18 +50,12 @@ public TransportCreateModelFromSetAction(Settings settings, ThreadPool threadPoo IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, TransportGetAction getAction, TransportFeatureStoreAction featureStoreAction) { - super(settings, CreateModelFromSetAction.NAME, threadPool, transportService, - actionFilters, indexNameExpressionResolver, CreateModelFromSetRequest::new); + super(CreateModelFromSetAction.NAME, transportService, actionFilters, CreateModelFromSetRequest::new); this.clusterService = clusterService; this.getAction = getAction; this.featureStoreAction = featureStoreAction; } - @Override - protected final void doExecute(CreateModelFromSetRequest request, ActionListener listener) { - throw new UnsupportedOperationException("attempt to execute a TransportCreateModelFromSetAction without a task"); - } - @Override protected void doExecute(Task task, CreateModelFromSetRequest request, ActionListener listener) { if (!clusterService.state().routingTable().hasIndex(request.getStore())) { diff --git a/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java index a4ed7a6b..44db0e55 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java @@ -62,19 +62,13 @@ public TransportFeatureStoreAction(Settings settings, ThreadPool threadPool, Tra ClusterService clusterService, Client client, LtrRankerParserFactory factory, TransportClearCachesAction clearCachesAction) { - super(settings, FeatureStoreAction.NAME, false, threadPool, transportService, actionFilters, - indexNameExpressionResolver, FeatureStoreRequest::new); + super(FeatureStoreAction.NAME, false, transportService, actionFilters, FeatureStoreRequest::new); this.factory = factory; this.clusterService = clusterService; this.clearCachesAction = clearCachesAction; this.client = client; } - @Override - protected void doExecute(FeatureStoreRequest request, ActionListener listener) { - throw new UnsupportedOperationException("attempt to execute a TransportFeatureStoreAction without a task"); - } - @Override protected void doExecute(Task task, FeatureStoreRequest request, ActionListener listener) { if (!clusterService.state().routingTable().hasIndex(request.getStore())) { @@ -157,7 +151,7 @@ private void validate(FeatureValidation validation, Runnable onSuccess) { ValidatingLtrQueryBuilder ltrBuilder = new ValidatingLtrQueryBuilder(element, validation, factory); - SearchRequestBuilder builder = SearchAction.INSTANCE.newRequestBuilder(client); + SearchRequestBuilder builder = new SearchRequestBuilder(client, SearchAction.INSTANCE); builder.setIndices(validation.getIndex()); builder.setQuery(ltrBuilder); builder.setFrom(0); diff --git a/src/main/java/com/o19s/es/ltr/action/TransportListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/TransportListStoresAction.java index 734ed402..73d7beab 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportListStoresAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportListStoresAction.java @@ -62,8 +62,8 @@ public class TransportListStoresAction extends TransportMasterNodeReadAction params, String templateLanguage, @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); - out.writeStringList(queryParams); + out.writeStringArray(queryParams.toArray(new String[0])); out.writeString(templateLanguage); out.writeString(template); out.writeBoolean(templateAsString); diff --git a/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java b/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java index 9e936a79..129b0438 100644 --- a/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java +++ b/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.elasticsearch.common.collect.Tuple; @@ -90,7 +91,7 @@ void doLog(Query query, List loggers, IndexSearcher searcher, Se int endDoc = 0; int docBase = 0; Scorer scorer = null; - Weight weight = searcher.createWeight(searcher.rewrite(query), true, 1F); + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE, 1F); // Loop logic borrowed from lucene QueryRescorer while (hitUpto < reordered.length) { SearchHit hit = reordered[hitUpto]; diff --git a/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java b/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java index c38a683e..ee71dc3b 100644 --- a/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -94,8 +95,8 @@ public String toString(String field) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - if (!needsScores) { + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + if (!scoreMode.needsScores()) { // If scores are not needed simply return a constant score on all docs return new ConstantScoreWeight(this.query, boost) { @Override @@ -105,7 +106,8 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { - return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); + return new ConstantScoreScorer(this, score(), + scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); } }; } @@ -215,6 +217,17 @@ public float score() throws IOException { public DocIdSetIterator iterator() { return iterator; } + + /** + * Return the maximum score that documents between the last {@code target} + * that this iterator was {@link #advanceShallow(int) shallow-advanced} to + * included and {@code upTo} included. + */ + @Override + public float getMaxScore(int upTo) throws IOException { + //TODO?? + return Float.POSITIVE_INFINITY; + } } static class FVDoubleValuesSource extends DoubleValuesSource { diff --git a/src/main/java/com/o19s/es/ltr/query/NoopScorer.java b/src/main/java/com/o19s/es/ltr/query/NoopScorer.java index 2605b8b8..cfbebfa8 100644 --- a/src/main/java/com/o19s/es/ltr/query/NoopScorer.java +++ b/src/main/java/com/o19s/es/ltr/query/NoopScorer.java @@ -56,4 +56,15 @@ public float score() throws IOException { public DocIdSetIterator iterator() { return _noopIter; } + + /** + * Return the maximum score that documents between the last {@code target} + * that this iterator was {@link #advanceShallow(int) shallow-advanced} to + * included and {@code upTo} included. + */ + @Override + public float getMaxScore(int upTo) throws IOException { + //TODO?? + return Float.POSITIVE_INFINITY; + } } diff --git a/src/main/java/com/o19s/es/ltr/query/RankerQuery.java b/src/main/java/com/o19s/es/ltr/query/RankerQuery.java index 775442e6..628c6f77 100644 --- a/src/main/java/com/o19s/es/ltr/query/RankerQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/RankerQuery.java @@ -37,6 +37,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -170,13 +171,14 @@ public FeatureSet featureSet() { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - if (!needsScores) { + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + if (!scoreMode.needsScores()) { // If scores are not needed simply return a constant score on all docs return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { - return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); + return new ConstantScoreScorer(this, score(), + scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); } @Override @@ -196,7 +198,7 @@ public boolean isCacheable(LeafReaderContext ctx) { if (q instanceof LtrRewritableQuery) { q = ((LtrRewritableQuery)q).ltrRewrite(vectorSupplier); } - weights.add(searcher.createWeight(q, true, boost)); + weights.add(searcher.createWeight(q, ScoreMode.COMPLETE, boost)); } return new RankerWeight(this, weights, ltrRankerWrapper, features); } @@ -245,7 +247,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio subs.add(Explanation.noMatch(featureString + " [no match, default value 0.0 used]")); } else { subs.add(Explanation.match(explain.getValue(), featureString, explain)); - d.setFeatureScore(ordinal, explain.getValue()); + d.setFeatureScore(ordinal, explain.getValue().floatValue()); } } float modelScore = ranker.score(d); @@ -320,6 +322,17 @@ public float score() throws IOException { public DocIdSetIterator iterator() { return iterator; } + + /** + * Return the maximum score that documents between the last {@code target} + * that this iterator was {@link #advanceShallow(int) shallow-advanced} to + * included and {@code upTo} included. + */ + @Override + public float getMaxScore(int upTo) throws IOException { + //TODO?? + return Float.POSITIVE_INFINITY; + } } } diff --git a/src/main/java/com/o19s/es/ltr/ranker/ranklib/RankLibScriptEngine.java b/src/main/java/com/o19s/es/ltr/ranker/ranklib/RankLibScriptEngine.java index c5956e3e..2cfd3cd6 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/ranklib/RankLibScriptEngine.java +++ b/src/main/java/com/o19s/es/ltr/ranker/ranklib/RankLibScriptEngine.java @@ -18,7 +18,6 @@ import com.o19s.es.ltr.ranker.LtrRanker; import com.o19s.es.ltr.ranker.parser.LtrRankerParserFactory; -import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; @@ -36,7 +35,7 @@ * So this code acts as a hook for deserializing Ranklib models from ranklib XML * and as a convenient means for caching those deserialized model */ -public class RankLibScriptEngine extends AbstractComponent implements ScriptEngine { +public class RankLibScriptEngine implements ScriptEngine { public static final String NAME = "ranklib"; public static final String EXTENSION = "ranklib"; diff --git a/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java b/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java index 74eda159..6eb052ac 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java @@ -16,7 +16,6 @@ package com.o19s.es.ltr.rest; -import com.o19s.es.ltr.action.AddFeaturesToSetAction; import com.o19s.es.ltr.action.AddFeaturesToSetAction.AddFeaturesToSetRequestBuilder; import com.o19s.es.ltr.feature.FeatureValidation; import com.o19s.es.ltr.feature.store.StoredFeature; @@ -75,7 +74,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli "or directly in the body not both"); } - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client); builder.request().setStore(store); builder.request().setFeatureSet(setName); builder.request().setFeatureNameQuery(featureQuery); diff --git a/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java b/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java index 315c7fd7..e0a65d1e 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java @@ -63,7 +63,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli String routing = request.param("routing"); ParserState state = new ParserState(); request.withContentOrSourceParamParserOrNull((p) -> ParserState.parse(p, state)); - CreateModelFromSetRequestBuilder builder = CreateModelFromSetAction.INSTANCE.newRequestBuilder(client); + CreateModelFromSetRequestBuilder builder = new CreateModelFromSetRequestBuilder(client); if (expectedVersion != null) { builder.withVersion(store, request.param("name"), expectedVersion, state.model.name, state.model.model); } else { diff --git a/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java b/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java index 7b3808ee..896a6710 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java @@ -16,9 +16,9 @@ package com.o19s.es.ltr.rest; -import com.o19s.es.ltr.action.CachesStatsAction; import com.o19s.es.ltr.action.ClearCachesAction; import com.o19s.es.ltr.action.ClearCachesAction.ClearCachesNodesResponse; + import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -66,13 +66,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } } + @SuppressWarnings({"rawtypes", "unchecked"}) private RestChannelConsumer getStats(NodeClient client) { - return (channel) -> CachesStatsAction.INSTANCE.newRequestBuilder(client).execute(new NodesResponseRestListener<>(channel)); + return (channel) -> new ClearCachesAction.RequestBuilder(client).execute(new NodesResponseRestListener(channel)); } private RestChannelConsumer clearCache(RestRequest request, NodeClient client) { String storeName = indexName(request); - ClearCachesAction.RequestBuilder builder = ClearCachesAction.INSTANCE.newRequestBuilder(client); + ClearCachesAction.RequestBuilder builder = new ClearCachesAction.RequestBuilder(client); builder.request().clearStore(storeName); return (channel) -> builder.execute(new RestBuilderListener(channel) { @Override diff --git a/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java b/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java index 9b048478..ab56fa1c 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java @@ -18,7 +18,6 @@ import com.o19s.es.ltr.action.ClearCachesAction; import com.o19s.es.ltr.action.FeatureStoreAction; -import com.o19s.es.ltr.action.ListStoresAction; import com.o19s.es.ltr.feature.FeatureValidation; import com.o19s.es.ltr.feature.store.StorableElement; import com.o19s.es.ltr.feature.store.StoredFeature; @@ -30,6 +29,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -208,7 +209,7 @@ public RestResponse buildResponse(IndicesExistsResponse indicesExistsResponse, } RestChannelConsumer listStores(NodeClient client) { - return (channel) -> ListStoresAction.INSTANCE.newRequestBuilder(client).execute( + return (channel) -> new SearchRequestBuilder(client, SearchAction.INSTANCE).execute( new RestToXContentListener<>(channel)); } @@ -248,7 +249,8 @@ RestChannelConsumer addOrUpdate(NodeClient client, String type, String indexName return (channel) -> channel.sendResponse(new BytesRestResponse(channel, RestStatus.METHOD_NOT_ALLOWED, iae)); } } - FeatureStoreAction.FeatureStoreRequestBuilder builder = FeatureStoreAction.INSTANCE.newRequestBuilder(client); + FeatureStoreAction.FeatureStoreRequestBuilder builder = new FeatureStoreAction.FeatureStoreRequestBuilder( + client, FeatureStoreAction.INSTANCE); if (request.method() == RestRequest.Method.PUT) { builder.request().setAction(FeatureStoreAction.FeatureStoreRequest.Action.CREATE); } else { @@ -275,7 +277,7 @@ RestChannelConsumer delete(NodeClient client, String type, String indexName, Res // wrap the response so we can send another request to clear the cache // usually we send only one transport request from the rest layer // it's still unclear which direction we should take (thick or thin REST layer?) - ClearCachesAction.RequestBuilder clearCache = ClearCachesAction.INSTANCE.newRequestBuilder(client); + ClearCachesAction.RequestBuilder clearCache = new ClearCachesAction.RequestBuilder(client); switch (type) { case StoredFeature.TYPE: clearCache.request().clearFeature(indexName, name); diff --git a/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java b/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java index 893ddd04..544b79c2 100644 --- a/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java +++ b/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java @@ -110,7 +110,7 @@ public void testBooleanQuery() throws Exception { // Verify tf score TopDocs docs = searcher.search(eq, 4); - assertThat(docs.getMaxScore(), equalTo(3.0f)); + assertThat(docs.scoreDocs[0].score, equalTo(3.0f)); } public void testUniqueTerms() throws Exception { @@ -136,7 +136,7 @@ public void testUniqueTerms() throws Exception { // Verify score is 5 (5 unique terms) TopDocs docs = searcher.search(eq, 4); - assertThat(docs.getMaxScore(), equalTo(5.0f)); + assertThat(docs.scoreDocs[0].score, equalTo(5.0f)); } public void testInvalidStat() throws Exception { diff --git a/src/test/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java b/src/test/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java index 1910d755..9c9be4a2 100644 --- a/src/test/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java +++ b/src/test/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java @@ -45,7 +45,7 @@ public void testAddToSetWithQuery() throws Exception { addElement(randomFeature("another")); - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_feature_set"); builder.request().setFeatureNameQuery("feature*"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -59,7 +59,7 @@ public void testAddToSetWithQuery() throws Exception { assertEquals(features.size(), set.size()); assertTrue(features.stream().map(StoredFeature::name).allMatch(set::hasFeature)); - builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_feature_set"); builder.request().setFeatureNameQuery("another"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -78,7 +78,7 @@ public void testAddToSetWithList() throws Exception { features.add(feat); } - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_feature_set"); builder.request().setFeatures(features); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -92,7 +92,7 @@ public void testAddToSetWithList() throws Exception { assertEquals(features.size(), set.size()); assertTrue(features.stream().map(StoredFeature::name).allMatch(set::hasFeature)); - builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_feature_set"); builder.request().setFeatures(Collections.singletonList(randomFeature("another_feature"))); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -105,7 +105,7 @@ public void testAddToSetWithList() throws Exception { } public void testFailuresWhenEmpty() throws Exception { - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_broken_set"); builder.request().setFeatureNameQuery("doesnotexist*"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -118,7 +118,7 @@ public void testFailuresWhenEmpty() throws Exception { public void testFailuresOnDuplicates() throws Exception { addElement(randomFeature("duplicated")); - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("duplicated_set"); builder.request().setFeatureNameQuery("duplicated*"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -128,7 +128,7 @@ public void testFailuresOnDuplicates() throws Exception { assertEquals(1, resp.getResponse().getVersion()); - AddFeaturesToSetRequestBuilder builder2 = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder2= new AddFeaturesToSetRequestBuilder(client()); builder2.request().setFeatureSet("duplicated_set"); builder2.request().setFeatureNameQuery("duplicated"); builder2.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -143,7 +143,7 @@ public void testMergeWithQuery() throws Exception { addElement(randomFeature("duplicated")); addElement(randomFeature("new_feature")); - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder= new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("merged_set"); builder.request().setFeatureNameQuery("duplicated*"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -153,7 +153,7 @@ public void testMergeWithQuery() throws Exception { assertEquals(DocWriteResponse.Result.CREATED, resp.getResponse().getResult()); assertEquals(1, resp.getResponse().getVersion()); - builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + builder= new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("merged_set"); builder.request().setFeatureNameQuery("*"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -176,7 +176,7 @@ public void testMergeWithList() throws Exception { features.add(feat); } - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder= new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_feature_set"); builder.request().setFeatures(features); builder.request().setMerge(true); @@ -191,7 +191,7 @@ public void testMergeWithList() throws Exception { assertEquals(features.size(), set.size()); assertTrue(features.stream().map(StoredFeature::name).allMatch(set::hasFeature)); - builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + builder= new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("new_feature_set"); builder.request().setFeatures(asList(randomFeature("another_feature"), randomFeature("feature0"))); builder.request().setMerge(true); diff --git a/src/test/java/com/o19s/es/ltr/action/BaseIntegrationTest.java b/src/test/java/com/o19s/es/ltr/action/BaseIntegrationTest.java index 292312ab..15aff3cf 100644 --- a/src/test/java/com/o19s/es/ltr/action/BaseIntegrationTest.java +++ b/src/test/java/com/o19s/es/ltr/action/BaseIntegrationTest.java @@ -35,7 +35,6 @@ import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; -import org.elasticsearch.script.SearchScript; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; @@ -48,6 +47,8 @@ import static com.o19s.es.ltr.feature.store.ScriptFeature.FEATURE_VECTOR; public abstract class BaseIntegrationTest extends ESSingleNodeTestCase { + + public static final ScriptContext AGGS_CONTEXT = new ScriptContext<>("aggs", ScoreScript.Factory.class); @Override protected Collection> getPlugins() { return Arrays.asList(LtrQueryParserPlugin.class, NativeScriptPlugin.class); @@ -105,7 +106,8 @@ protected LtrRankerParserFactory parserFactory() { public FeatureStoreResponse addElement(StorableElement element, @Nullable FeatureValidation validation, String store) throws ExecutionException, InterruptedException { - FeatureStoreRequestBuilder builder = FeatureStoreAction.INSTANCE.newRequestBuilder(client()); + FeatureStoreRequestBuilder builder = + new FeatureStoreRequestBuilder(client(), FeatureStoreAction.INSTANCE); builder.request().setStorableElement(element); builder.request().setAction(FeatureStoreAction.FeatureStoreRequest.Action.CREATE); builder.request().setStore(store); @@ -147,7 +149,7 @@ public String getType() { @Override public FactoryType compile(String scriptName, String scriptSource, ScriptContext context, Map params) { - if (!context.equals(ScoreScript.CONTEXT) && (!context.equals(SearchScript.AGGS_CONTEXT))) { + if (!context.equals(ScoreScript.CONTEXT) && (!context.equals(AGGS_CONTEXT))) { throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]"); } diff --git a/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java b/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java index d2f20b36..5aa5c0c3 100644 --- a/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java +++ b/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java @@ -41,7 +41,7 @@ public void testListStore() throws Exception { infos.put(IndexFeatureStore.storeName(store), new IndexStoreInfo(store, IndexFeatureStore.VERSION, addElements(store))); } - ListStoresActionResponse resp = ListStoresAction.INSTANCE.newRequestBuilder(client()).execute().get(); + ListStoresActionResponse resp = new ListStoresAction.ListStoresActionBuilder(client(), ListStoresAction.INSTANCE).execute().get(); assertEquals(infos.size(), resp.getStores().size()); assertEquals(infos.keySet(), resp.getStores().keySet()); for (String k : infos.keySet()) { diff --git a/src/test/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java b/src/test/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java index d973494c..e55b7d3b 100644 --- a/src/test/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java +++ b/src/test/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java @@ -90,7 +90,7 @@ public void testValidationOnAddFeatureToSet() { StoredFeature brokenFeature = new StoredFeature("broken", singletonList("query_string"), "mustache", brokenQuery); Map params = new HashMap<>(); params.put("query_string", "a query"); - AddFeaturesToSetAction.AddFeaturesToSetRequestBuilder request = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetAction.AddFeaturesToSetRequestBuilder request = new AddFeaturesToSetAction.AddFeaturesToSetRequestBuilder(client()); request.request().setStore(IndexFeatureStore.DEFAULT_STORE); request.request().setValidation(new FeatureValidation("test_index", params)); request.request().setFeatures(Arrays.asList(feature, brokenFeature)); @@ -112,7 +112,8 @@ public void testValidationOnCreateModelFromSet() throws ExecutionException, Inte StoredFeatureSet brokenFeatureSet = new StoredFeatureSet("my_feature_set", Arrays.asList(feature, brokenFeature)); // Store a broken feature set addElement(brokenFeatureSet); - CreateModelFromSetAction.CreateModelFromSetRequestBuilder request = CreateModelFromSetAction.INSTANCE.newRequestBuilder(client()); + CreateModelFromSetAction.CreateModelFromSetRequestBuilder request = + new CreateModelFromSetAction.CreateModelFromSetRequestBuilder(client()); request.request().setValidation(new FeatureValidation("test_index", params)); StoredLtrModel.LtrModelDefinition definition = new StoredLtrModel.LtrModelDefinition("model/linear", "{\"test\": 2.1, \"broken\": 4.3}", true); diff --git a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java index 441c5d83..f8b378ee 100644 --- a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java +++ b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; @@ -164,6 +165,14 @@ public SearchHit[] selectRandomHits() throws IOException { int maxHits = TestUtil.nextInt(random(), minHits, minHits+10); List hits = new ArrayList<>(maxHits); searcher.search(new MatchAllDocsQuery(), new SimpleCollector() { + /** + * Indicates what features are required from the scorer. + */ + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + LeafReaderContext context; @Override @@ -183,10 +192,6 @@ public void collect(int doc) throws IOException { } } - @Override - public boolean needsScores() { - return false; - } }); assert hits.size() >= minHits; Collections.shuffle(hits, random()); diff --git a/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java b/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java index da61ab2f..f8a74ab6 100644 --- a/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java +++ b/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java @@ -166,11 +166,6 @@ protected boolean builderGeneratesCacheableQueries() { return false; } - @Override - protected boolean isCachable(LtrQueryBuilder queryBuilder) { - return false; - } - @Override protected LtrQueryBuilder doCreateTestQueryBuilder() { LtrQueryBuilder builder = new LtrQueryBuilder(); diff --git a/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java b/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java index 7f6e4681..006e6836 100644 --- a/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java +++ b/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java @@ -47,15 +47,16 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.AfterEffectB; import org.apache.lucene.search.similarities.AxiomaticF3LOG; import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.BasicModelBE; +import org.apache.lucene.search.similarities.BasicModelG; import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.DFISimilarity; @@ -140,7 +141,7 @@ public void setupIndex() throws IOException { new LMJelinekMercerSimilarity(0.2F), new AxiomaticF3LOG(0.5F, 10), new DFISimilarity(new IndependenceChiSquared()), - new DFRSimilarity(new BasicModelBE(), new AfterEffectB(), new NormalizationH1()), + new DFRSimilarity(new BasicModelG(), new AfterEffectB(), new NormalizationH1()), new IBSimilarity(new DistributionLL(), new LambdaDF(), new NormalizationH3()) ); similarity = sims.get(random().nextInt(sims.size())); @@ -181,15 +182,19 @@ public void reset() { RankerQuery query = RankerQuery.buildLogQuery(logger, set, null, Collections.emptyMap()); searcherUnderTest.search(query, new SimpleCollector() { + private LeafReaderContext context; - private Scorer scorer; + private Scorable scorer; + /** + * Indicates what features are required from the scorer. + */ @Override - public boolean needsScores() { - return true; + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; } @Override - public void setScorer(Scorer scorer) throws IOException { + public void setScorer(Scorable scorer) throws IOException { this.scorer = scorer; } @@ -324,7 +329,7 @@ private void assertScoresMatch(List features, float[] scores, // It produces 0.56103003 for feat:0 in doc1 using score() but 0.5610301 using explain Explanation expl = searcherUnderTest.explain(ltrQuery, docId); - assertEquals("Explain scores match with similarity " + similarity.getClass(), expl.getValue(), + assertEquals("Explain scores match with similarity " + similarity.getClass(), expl.getValue().floatValue(), queryScore, 5 * Math.ulp(modelScore)); checkFeatureNames(expl, features); } diff --git a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java index c27a730b..21d42ee5 100644 --- a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java +++ b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java @@ -217,12 +217,6 @@ protected void doAssertLuceneQuery(StoredLtrQueryBuilder queryBuilder, assertThat(rquery.ranker().newFeatureVector(null), instanceOf(DenseFeatureVector.class)); } - @Override - protected boolean isCachable(StoredLtrQueryBuilder queryBuilder) { - // This query is not cachable as it needs a ScriptService - // see QueryShardContext#failIfFrozen() - return false; - } // Hack to inject our MemStore public static class TestPlugin extends LtrQueryParserPlugin { diff --git a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java index ae61d71d..dccb4f86 100644 --- a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java +++ b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java @@ -18,13 +18,11 @@ package com.o19s.es.ltr.query; import com.o19s.es.ltr.LtrTestUtils; -import com.o19s.es.ltr.action.AddFeaturesToSetAction; import com.o19s.es.ltr.action.AddFeaturesToSetAction.AddFeaturesToSetRequestBuilder; import com.o19s.es.ltr.action.BaseIntegrationTest; import com.o19s.es.ltr.action.CachesStatsAction; import com.o19s.es.ltr.action.CachesStatsAction.CachesStatsNodesResponse; import com.o19s.es.ltr.action.ClearCachesAction; -import com.o19s.es.ltr.action.CreateModelFromSetAction; import com.o19s.es.ltr.action.CreateModelFromSetAction.CreateModelFromSetRequestBuilder; import com.o19s.es.ltr.feature.store.ScriptFeature; import com.o19s.es.ltr.feature.store.StoredFeature; @@ -72,7 +70,7 @@ public void testScriptFeatureUseCase() throws Exception { addElement(new StoredFeature("feature6", Arrays.asList("query", "extra_multiplier_ltr"), ScriptFeature.TEMPLATE_LANGUAGE, "{\"lang\": \"native\", \"source\": \"feature_extractor\", \"params\": { \"dependent_feature\": \"feature1\"," + " \"extra_script_params\" : {\"extra_multiplier_ltr\": \"extra_multiplier\"}}}")); - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("my_set"); builder.request().setFeatureNameQuery("feature1"); @@ -81,7 +79,7 @@ public void testScriptFeatureUseCase() throws Exception { builder.request().setFeatureNameQuery("feature6"); long version = builder.get().getResponse().getVersion(); - CreateModelFromSetRequestBuilder createModelFromSetRequestBuilder = CreateModelFromSetAction.INSTANCE.newRequestBuilder(client()); + CreateModelFromSetRequestBuilder createModelFromSetRequestBuilder = new CreateModelFromSetRequestBuilder(client()); createModelFromSetRequestBuilder.withVersion(IndexFeatureStore.DEFAULT_STORE, "my_set", version, "my_model", new StoredLtrModel.LtrModelDefinition("model/linear", SIMPLE_SCRIPT_MODEL, true)); createModelFromSetRequestBuilder.get(); @@ -119,7 +117,7 @@ public void testFullUsecase() throws Exception { "{\"lang\": \"native\", \"source\": \"feature_extractor\", \"params\": { \"dependent_feature\": \"feature1\"}}")); - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("my_set"); builder.request().setFeatureNameQuery("feature1"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); @@ -140,7 +138,7 @@ public void testFullUsecase() throws Exception { builder.request().setFeatureNameQuery("feature6"); long version = builder.get().getResponse().getVersion(); - CreateModelFromSetRequestBuilder createModelFromSetRequestBuilder = CreateModelFromSetAction.INSTANCE.newRequestBuilder(client()); + CreateModelFromSetRequestBuilder createModelFromSetRequestBuilder = new CreateModelFromSetRequestBuilder(client()); createModelFromSetRequestBuilder.withVersion(IndexFeatureStore.DEFAULT_STORE, "my_set", version, "my_model", new StoredLtrModel.LtrModelDefinition("model/linear", SIMPLE_MODEL, true)); createModelFromSetRequestBuilder.get(); @@ -261,7 +259,7 @@ public void testFullUsecase() throws Exception { assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThan(0.2876f + 2.876f)); StoredLtrModel model = getElement(StoredLtrModel.class, StoredLtrModel.TYPE, "my_model"); - CachesStatsNodesResponse stats = CachesStatsAction.INSTANCE.newRequestBuilder(client()).execute().get(); + CachesStatsNodesResponse stats = new CachesStatsAction.CachesStatsActionBuilder(client()).execute().get(); assertEquals(1, stats.getAll().getTotal().getCount()); assertEquals(model.compile(parserFactory()).ramBytesUsed(), stats.getAll().getTotal().getRam()); assertEquals(1, stats.getAll().getModels().getCount()); @@ -271,11 +269,11 @@ public void testFullUsecase() throws Exception { assertEquals(0, stats.getAll().getFeaturesets().getCount()); assertEquals(0, stats.getAll().getFeaturesets().getRam()); - ClearCachesAction.RequestBuilder clearCache = ClearCachesAction.INSTANCE.newRequestBuilder(client()); + ClearCachesAction.RequestBuilder clearCache = new ClearCachesAction.RequestBuilder(client()); clearCache.request().clearModel(IndexFeatureStore.DEFAULT_STORE, "my_model"); clearCache.get(); - stats = CachesStatsAction.INSTANCE.newRequestBuilder(client()).execute().get(); + stats = new CachesStatsAction.CachesStatsActionBuilder(client()).execute().get(); assertEquals(0, stats.getAll().getTotal().getCount()); assertEquals(0, stats.getAll().getTotal().getRam()); @@ -285,7 +283,7 @@ public void testInvalidDerived() throws Exception { addElement(new StoredFeature("bad_df", Collections.singletonList("query"), "derived_expression", "what + is + this")); - AddFeaturesToSetRequestBuilder builder = AddFeaturesToSetAction.INSTANCE.newRequestBuilder(client()); + AddFeaturesToSetRequestBuilder builder = new AddFeaturesToSetRequestBuilder(client()); builder.request().setFeatureSet("my_bad_set"); builder.request().setFeatureNameQuery("bad_df"); builder.request().setStore(IndexFeatureStore.DEFAULT_STORE); From f7cfabb10aca0bd5b453a54520935b5d7d91126c Mon Sep 17 00:00:00 2001 From: Umesh Dangat Date: Wed, 10 Apr 2019 13:30:53 -0700 Subject: [PATCH 2/4] update to ES7.0.0, fix some tests --- build.gradle | 8 ++--- .../com/o19s/es/explore/ExplorerQuery.java | 13 ++++---- .../es/ltr/logging/LoggingFetchSubPhase.java | 33 ++++++++----------- .../logging/LoggingFetchSubPhaseTests.java | 14 -------- .../es/ltr/query/LtrQueryBuilderTests.java | 6 ++++ .../ltr/query/StoredLtrQueryBuilderTests.java | 5 +++ 6 files changed, 35 insertions(+), 44 deletions(-) diff --git a/build.gradle b/build.gradle index dd246e9d..8f6662ce 100644 --- a/build.gradle +++ b/build.gradle @@ -6,12 +6,12 @@ buildscript { } dependencies { - classpath "org.elasticsearch.gradle:build-tools:7.0.0-rc2" + classpath "org.elasticsearch.gradle:build-tools:7.0.0" } } group = 'com.o19s' -version = '1.1.0-es7.0.0-rc2' +version = '1.1.0-es7.0.0' apply plugin: 'java' apply plugin: 'idea' @@ -51,10 +51,10 @@ dependencies { compile 'org.ow2.asm:asm:5.0.4' compile 'org.ow2.asm:asm-commons:5.0.4' compile 'org.ow2.asm:asm-tree:5.0.4' - compile 'org.elasticsearch:elasticsearch:7.0.0-rc2' + compile 'org.elasticsearch:elasticsearch:7.0.0' compile 'com.o19s:RankyMcRankFace:0.1.1' compile "com.github.spullara.mustache.java:compiler:0.9.3" - testCompile 'org.elasticsearch.test:framework:7.0.0-rc2' + testCompile 'org.elasticsearch.test:framework:7.0.0' } dependencyLicenses { diff --git a/src/main/java/com/o19s/es/explore/ExplorerQuery.java b/src/main/java/com/o19s/es/explore/ExplorerQuery.java index 1ad1b4a0..caa47bf5 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/ExplorerQuery.java @@ -107,13 +107,14 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo StatisticsHelper ttf_stats = new StatisticsHelper(); for (Term term : terms) { - TermStates - ctx = - TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores()); + TermStates ctx = TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores()); TermStatistics tStats = searcher.termStatistics(term, ctx); - df_stats.add(tStats.docFreq()); - idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs())); - ttf_stats.add(tStats.totalTermFreq()); + if(tStats!=null){ + df_stats.add(tStats.docFreq()); + idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs())); + ttf_stats.add(tStats.totalTermFreq()); + + } } /* diff --git a/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java b/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java index 129b0438..1c37d65f 100644 --- a/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java +++ b/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.FetchPhaseExecutionException; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.QueryRescorer; @@ -72,11 +71,8 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept loggers.add(query.v2()); }); - try { - doLog(builder.build(), loggers, context.searcher(), hits); - } catch (LtrLoggingException e) { - throw new FetchPhaseExecutionException(context, e.getMessage(), e); - } + doLog(builder.build(), loggers, context.searcher(), hits); + } void doLog(Query query, List loggers, IndexSearcher searcher, SearchHit[] hits) throws IOException { @@ -110,7 +106,7 @@ void doLog(Query query, List loggers, IndexSearcher searcher, Se scorer = weight.scorer(readerContext); } - if(scorer != null) { + if (scorer != null) { int targetDoc = docID - docBase; int actualDoc = scorer.docID(); if (actualDoc < targetDoc) { @@ -133,15 +129,17 @@ private Tuple extractQuery(LoggingSearchExtBuilder. } return toLogger(logSpec, inspectQuery(q) .orElseThrow(() -> new IllegalArgumentException("Query named [" + logSpec.getNamedQuery() + - "] must be a [sltr] query [" + - ((q instanceof BoostQuery) ? ((BoostQuery)q).getQuery().getClass().getSimpleName() : q.getClass().getSimpleName()) + - "] found"))); + "] must be a [sltr] query [" + + ((q instanceof BoostQuery) ? ((BoostQuery) q).getQuery().getClass().getSimpleName( + + ) : q.getClass().getSimpleName()) + + "] found"))); } private Tuple extractRescore(LoggingSearchExtBuilder.LogSpec logSpec, List contexts) { if (logSpec.getRescoreIndex() >= contexts.size()) { - throw new IllegalArgumentException("rescore index [" + logSpec.getRescoreIndex()+"] is out of bounds, only " + + throw new IllegalArgumentException("rescore index [" + logSpec.getRescoreIndex() + "] is out of bounds, only " + "[" + contexts.size() + "] rescore context(s) are available"); } RescoreContext context = contexts.get(logSpec.getRescoreIndex()); @@ -153,8 +151,8 @@ private Tuple extractRescore(LoggingSearchExtBuilde QueryRescorer.QueryRescoreContext qrescore = (QueryRescorer.QueryRescoreContext) context; return toLogger(logSpec, inspectQuery(qrescore.query()) .orElseThrow(() -> new IllegalArgumentException("Expected a [sltr] query but found a " + - "[" + qrescore.query().getClass().getSimpleName() + "] " + - "at index [" + logSpec.getRescoreIndex() + "]"))); + "[" + qrescore.query().getClass().getSimpleName() + "] " + + "at index [" + logSpec.getRescoreIndex() + "]"))); } private Optional inspectQuery(Query q) { @@ -200,7 +198,7 @@ static class HitLogConsumer implements LogLtrRanker.LogConsumer { } private void rebuild() { - List> ini = new ArrayList<>(set.size()) ; + List> ini = new ArrayList<>(set.size()); for (int i = 0; i < set.size(); i++) { Map defaultKeyVal = new HashMap<>(); @@ -217,11 +215,6 @@ private void rebuild() { public void accept(int featureOrdinal, float score) { assert currentLog != null; assert currentHit != null; - if (Float.isNaN(score)) { - // NOTE: should we fail on Float#isInfinite() as well? - throw new LtrLoggingException("Feature [" + set.feature(featureOrdinal).name() +"] produced a NaN value " + - "for doc [" + currentHit.getId() + "]" ); - } currentLog.get(featureOrdinal).put("value", score); } @@ -230,7 +223,7 @@ void nextDoc(SearchHit hit) { hit.fields(new HashMap<>()); } DocumentField logs = hit.getFields() - .computeIfAbsent(FIELD_NAME,(k) -> newLogField()); + .computeIfAbsent(FIELD_NAME, (k) -> newLogField()); Map>> entries = logs.getValue(); rebuild(); currentHit = hit; diff --git a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java index f8b378ee..2ae6d0b5 100644 --- a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java +++ b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java @@ -16,7 +16,6 @@ package com.o19s.es.ltr.logging; -import com.o19s.es.ltr.LtrTestUtils; import com.o19s.es.ltr.feature.PrebuiltFeature; import com.o19s.es.ltr.feature.PrebuiltFeatureSet; import com.o19s.es.ltr.feature.PrebuiltLtrModel; @@ -34,7 +33,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; @@ -63,7 +61,6 @@ import java.util.Map; import java.util.UUID; -import static java.util.Collections.singletonList; import static org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction.Modifier.LN2P; import static org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType.FLOAT; @@ -149,17 +146,6 @@ public void testLogging() throws IOException { } } - public void testBogusQuery() throws IOException { - PrebuiltFeatureSet set = new PrebuiltFeatureSet("test", - singletonList(new PrebuiltFeature("test", new BoostQuery(new MatchAllDocsQuery(), Float.NaN)))); - LoggingFetchSubPhase.HitLogConsumer logger1 = new LoggingFetchSubPhase.HitLogConsumer("logger1", set, true); - RankerQuery q = RankerQuery.build(new PrebuiltLtrModel("test", LtrTestUtils.buildRandomRanker(set.size()), set)); - Query lq = q.toLoggerQuery(logger1, true); - LoggingFetchSubPhase subPhase = new LoggingFetchSubPhase(); - SearchHit[] hits = selectRandomHits(); - expectThrows(LtrLoggingException.class, () -> subPhase.doLog(lq, singletonList(logger1), searcher, hits)); - } - public SearchHit[] selectRandomHits() throws IOException { int minHits = TestUtil.nextInt(random(), 5, 10); int maxHits = TestUtil.nextInt(random(), minHits, minHits+10); diff --git a/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java b/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java index f8a74ab6..eb84ca58 100644 --- a/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java +++ b/src/test/java/com/o19s/es/ltr/query/LtrQueryBuilderTests.java @@ -227,4 +227,10 @@ public void testMustRewrite() throws IOException { protected void doAssertLuceneQuery(LtrQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException { assertThat(query, instanceOf(RankerQuery.class)); } + + @Override + protected boolean isCacheable(LtrQueryBuilder queryBuilder){ + return false; + } + } diff --git a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java index 21d42ee5..4934a60b 100644 --- a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java +++ b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryBuilderTests.java @@ -229,4 +229,9 @@ protected FeatureStoreLoader getFeatureStoreLoader() { return LtrTestUtils.wrapMemStore(StoredLtrQueryBuilderTests.store); } } + + @Override + protected boolean isCacheable(StoredLtrQueryBuilder queryBuilder){ + return false; + } } From 46ed7d7e97720650618d6f2bde6f121bf4f38924 Mon Sep 17 00:00:00 2001 From: Umesh Dangat Date: Thu, 11 Apr 2019 09:11:06 -0700 Subject: [PATCH 3/4] fix the remaining tests --- .../com/o19s/es/ltr/action/ListStoresAction.java | 4 ++-- .../o19s/es/ltr/rest/RestFeatureStoreCaches.java | 3 ++- .../o19s/es/ltr/rest/RestSimpleFeatureStore.java | 5 ++--- .../com/o19s/es/ltr/action/ListStoresActionIT.java | 2 +- .../java/com/o19s/es/ltr/query/LtrQueryTests.java | 2 +- .../com/o19s/es/ltr/query/StoredLtrQueryIT.java | 14 +++++++------- .../rest-api-spec/test/fstore/20_features.yml | 12 ++++++------ .../rest-api-spec/test/fstore/30_featuresets.yml | 12 ++++++------ .../rest-api-spec/test/fstore/40_models.yml | 12 ++++++------ .../test/fstore/80_search_w_partial_models.yml | 4 ++-- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java index 73d0b90d..fbeb4a5b 100644 --- a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java +++ b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java @@ -145,8 +145,8 @@ public Map getCounts() { public static class ListStoresActionBuilder extends ActionRequestBuilder { - public ListStoresActionBuilder(ElasticsearchClient client, ListStoresAction action){ - super(client, action, new ListStoresActionRequest()); + public ListStoresActionBuilder(ElasticsearchClient client){ + super(client, INSTANCE, new ListStoresActionRequest()); } } } diff --git a/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java b/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java index 896a6710..31b0c67b 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java @@ -16,6 +16,7 @@ package com.o19s.es.ltr.rest; +import com.o19s.es.ltr.action.CachesStatsAction; import com.o19s.es.ltr.action.ClearCachesAction; import com.o19s.es.ltr.action.ClearCachesAction.ClearCachesNodesResponse; @@ -68,7 +69,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli @SuppressWarnings({"rawtypes", "unchecked"}) private RestChannelConsumer getStats(NodeClient client) { - return (channel) -> new ClearCachesAction.RequestBuilder(client).execute(new NodesResponseRestListener(channel)); + return (channel) -> new CachesStatsAction.CachesStatsActionBuilder(client).execute(new NodesResponseRestListener(channel)); } private RestChannelConsumer clearCache(RestRequest request, NodeClient client) { diff --git a/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java b/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java index ab56fa1c..4c30e3cd 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestSimpleFeatureStore.java @@ -18,6 +18,7 @@ import com.o19s.es.ltr.action.ClearCachesAction; import com.o19s.es.ltr.action.FeatureStoreAction; +import com.o19s.es.ltr.action.ListStoresAction; import com.o19s.es.ltr.feature.FeatureValidation; import com.o19s.es.ltr.feature.store.StorableElement; import com.o19s.es.ltr.feature.store.StoredFeature; @@ -29,8 +30,6 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -209,7 +208,7 @@ public RestResponse buildResponse(IndicesExistsResponse indicesExistsResponse, } RestChannelConsumer listStores(NodeClient client) { - return (channel) -> new SearchRequestBuilder(client, SearchAction.INSTANCE).execute( + return (channel) -> new ListStoresAction.ListStoresActionBuilder(client).execute( new RestToXContentListener<>(channel)); } diff --git a/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java b/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java index 5aa5c0c3..db6feed5 100644 --- a/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java +++ b/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java @@ -41,7 +41,7 @@ public void testListStore() throws Exception { infos.put(IndexFeatureStore.storeName(store), new IndexStoreInfo(store, IndexFeatureStore.VERSION, addElements(store))); } - ListStoresActionResponse resp = new ListStoresAction.ListStoresActionBuilder(client(), ListStoresAction.INSTANCE).execute().get(); + ListStoresActionResponse resp = new ListStoresAction.ListStoresActionBuilder(client()).execute().get(); assertEquals(infos.size(), resp.getStores().size()); assertEquals(infos.keySet(), resp.getStores().keySet()); for (String k : infos.keySet()) { diff --git a/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java b/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java index 006e6836..adb731ac 100644 --- a/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java +++ b/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java @@ -273,7 +273,7 @@ public void checkModelWithFeatures(List features, int[] modelFe // each RankList appears to correspond to a // query RankerTrainer trainer = new RankerTrainer(); - Ranker ranker = trainer.train(/*what type of model ot train*/RANKER_TYPE.LAMBDAMART, + Ranker ranker = trainer.train(/*what type of model ot train*/RANKER_TYPE.RANKNET, /*The training data*/ samples /*which features to use*/, featuresToUse /*how to score ranking*/, new NDCGScorer()); diff --git a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java index dccb4f86..6368a526 100644 --- a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java +++ b/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java @@ -97,7 +97,7 @@ public void testScriptFeatureUseCase() throws Exception { .setRescoreQueryWeight(1)); SearchResponse sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThanOrEqualTo(29.0f)); assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThanOrEqualTo(30.0f)); } @@ -158,7 +158,7 @@ public void testFullUsecase() throws Exception { .setRescoreQueryWeight(1)); SearchResponse sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); if (negativeScore) { assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThanOrEqualTo(-10.0f)); @@ -179,7 +179,7 @@ public void testFullUsecase() throws Exception { .setRescoreQueryWeight(1)); sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); if (negativeScore) { assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThanOrEqualTo(-10.0f)); @@ -210,7 +210,7 @@ public void testFullUsecase() throws Exception { .setRescoreQueryWeight(1)); sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThan(0.0f)); assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThanOrEqualTo(1.0f)); @@ -225,7 +225,7 @@ public void testFullUsecase() throws Exception { .setQueryWeight(0) .setRescoreQueryWeight(1)); sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThan(28.0f)); assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThan(30.0f)); @@ -240,7 +240,7 @@ public void testFullUsecase() throws Exception { .setQueryWeight(0) .setRescoreQueryWeight(1)); sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); assertThat(sr.getHits().getAt(0).getScore(), Matchers.lessThan(-28.0f)); assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThan(-30.0f)); @@ -255,7 +255,7 @@ public void testFullUsecase() throws Exception { .setQueryWeight(0) .setRescoreQueryWeight(1)); sr = sb.get(); - assertEquals(1, sr.getHits().getTotalHits()); + assertEquals(1, sr.getHits().getTotalHits().value); assertThat(sr.getHits().getAt(0).getScore(), Matchers.greaterThan(0.2876f + 2.876f)); StoredLtrModel model = getElement(StoredLtrModel.class, StoredLtrModel.TYPE, "my_model"); diff --git a/src/test/resources/rest-api-spec/test/fstore/20_features.yml b/src/test/resources/rest-api-spec/test/fstore/20_features.yml index 36df7d94..633baf82 100644 --- a/src/test/resources/rest-api-spec/test/fstore/20_features.yml +++ b/src/test/resources/rest-api-spec/test/fstore/20_features.yml @@ -60,19 +60,19 @@ - do: ltr.list_feature: {} - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_feature: prefix: my - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_feature: prefix: hop - - match: { hits.total: 0 } + - match: { hits.total.value: 0 } - do: ltr.delete_feature: @@ -155,21 +155,21 @@ ltr.list_feature: store: mystore - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_feature: store: mystore prefix: my - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_feature: store: mystore prefix: hop - - match: { hits.total: 0 } + - match: { hits.total.value: 0 } - do: ltr.delete_feature: diff --git a/src/test/resources/rest-api-spec/test/fstore/30_featuresets.yml b/src/test/resources/rest-api-spec/test/fstore/30_featuresets.yml index 35fa0681..a3eed333 100644 --- a/src/test/resources/rest-api-spec/test/fstore/30_featuresets.yml +++ b/src/test/resources/rest-api-spec/test/fstore/30_featuresets.yml @@ -73,19 +73,19 @@ - do: ltr.list_featureset: {} - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_featureset: prefix: my - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_featureset: prefix: hop - - match: { hits.total: 0 } + - match: { hits.total.value: 0 } - do: ltr.delete_featureset: @@ -180,21 +180,21 @@ ltr.list_featureset: store: mystore - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_featureset: store: mystore prefix: my - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_featureset: store: mystore prefix: hop - - match: { hits.total: 0 } + - match: { hits.total.value: 0 } - do: ltr.delete_featureset: diff --git a/src/test/resources/rest-api-spec/test/fstore/40_models.yml b/src/test/resources/rest-api-spec/test/fstore/40_models.yml index 82578c03..c939a6c4 100644 --- a/src/test/resources/rest-api-spec/test/fstore/40_models.yml +++ b/src/test/resources/rest-api-spec/test/fstore/40_models.yml @@ -66,19 +66,19 @@ - do: ltr.list_model: {} - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_model: prefix: my - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_model: prefix: hop - - match: { hits.total: 0 } + - match: { hits.total.value: 0 } - do: ltr.delete_model: @@ -164,21 +164,21 @@ ltr.list_model: store: mystore - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_model: store: mystore prefix: my - - match: { hits.total: 1 } + - match: { hits.total.value: 1 } - do: ltr.list_model: store: mystore prefix: hop - - match: { hits.total: 0 } + - match: { hits.total.value: 0 } - do: ltr.delete_model: diff --git a/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml b/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml index de005bf7..4434990e 100644 --- a/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml +++ b/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml @@ -84,10 +84,10 @@ setup: 1 1.0 - -2.0 + 2.0 - 2.0 + 4.0 From 1f9c18c386e8acf4bd155099903d4cea62f17559 Mon Sep 17 00:00:00 2001 From: Umesh Dangat Date: Tue, 30 Apr 2019 17:57:53 -0700 Subject: [PATCH 4/4] address PR comments --- .../com/o19s/es/explore/ExplorerQuery.java | 10 +++---- .../com/o19s/es/explore/ExplorerScorer.java | 1 - .../es/explore/PostingsExplorerQuery.java | 1 - .../TransportAddFeatureToSetAction.java | 2 -- .../es/ltr/logging/LtrLoggingException.java | 26 ------------------- .../es/ltr/query/DerivedExpressionQuery.java | 1 - .../com/o19s/es/ltr/query/NoopScorer.java | 1 - .../com/o19s/es/ltr/query/RankerQuery.java | 1 - 8 files changed, 3 insertions(+), 40 deletions(-) delete mode 100644 src/main/java/com/o19s/es/ltr/logging/LtrLoggingException.java diff --git a/src/main/java/com/o19s/es/explore/ExplorerQuery.java b/src/main/java/com/o19s/es/explore/ExplorerQuery.java index caa47bf5..2e73cee3 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/ExplorerQuery.java @@ -55,13 +55,9 @@ private boolean isCollectionScoped() { || type.endsWith(("_ttf")); } - public Query getQuery() { - return this.query; - } + public Query getQuery() { return this.query; } - public String getType() { - return this.type; - } + public String getType() { return this.type; } @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override @@ -109,7 +105,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo for (Term term : terms) { TermStates ctx = TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores()); TermStatistics tStats = searcher.termStatistics(term, ctx); - if(tStats!=null){ + if(tStats != null){ df_stats.add(tStats.docFreq()); idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs())); ttf_stats.add(tStats.totalTermFreq()); diff --git a/src/main/java/com/o19s/es/explore/ExplorerScorer.java b/src/main/java/com/o19s/es/explore/ExplorerScorer.java index 8bc686ba..b31b2906 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerScorer.java +++ b/src/main/java/com/o19s/es/explore/ExplorerScorer.java @@ -91,7 +91,6 @@ public DocIdSetIterator iterator() { */ @Override public float getMaxScore(int upTo) throws IOException { - //TODO?? return Float.POSITIVE_INFINITY; } } diff --git a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java index 7fa183c8..8e60e26b 100644 --- a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java @@ -186,7 +186,6 @@ public float score() throws IOException { */ @Override public float getMaxScore(int upTo) throws IOException { - //TODO? return Float.POSITIVE_INFINITY; } } diff --git a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java index 7c18a8e0..c404453b 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java @@ -174,8 +174,6 @@ private void fetchFeaturesFromStore() { srequest.source().query(bq); srequest.source().fetchSource(true); srequest.source().size(StoredFeatureSet.MAX_FEATURES); - //do we need this? - //ActionFuture resp = searchAction.execute(srequest); searchAction.execute(srequest, wrap(this::onSearchResponse, this::onSearchFailure)); } diff --git a/src/main/java/com/o19s/es/ltr/logging/LtrLoggingException.java b/src/main/java/com/o19s/es/ltr/logging/LtrLoggingException.java deleted file mode 100644 index 18829db6..00000000 --- a/src/main/java/com/o19s/es/ltr/logging/LtrLoggingException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright [2017] Wikimedia Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.o19s.es.ltr.logging; - -/** - * Error happening while logging feature values. - */ -public class LtrLoggingException extends RuntimeException { - public LtrLoggingException(String s) { - super(s); - } -} diff --git a/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java b/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java index ee71dc3b..f834b2c4 100644 --- a/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java @@ -225,7 +225,6 @@ public DocIdSetIterator iterator() { */ @Override public float getMaxScore(int upTo) throws IOException { - //TODO?? return Float.POSITIVE_INFINITY; } } diff --git a/src/main/java/com/o19s/es/ltr/query/NoopScorer.java b/src/main/java/com/o19s/es/ltr/query/NoopScorer.java index cfbebfa8..82700c7b 100644 --- a/src/main/java/com/o19s/es/ltr/query/NoopScorer.java +++ b/src/main/java/com/o19s/es/ltr/query/NoopScorer.java @@ -64,7 +64,6 @@ public DocIdSetIterator iterator() { */ @Override public float getMaxScore(int upTo) throws IOException { - //TODO?? return Float.POSITIVE_INFINITY; } } diff --git a/src/main/java/com/o19s/es/ltr/query/RankerQuery.java b/src/main/java/com/o19s/es/ltr/query/RankerQuery.java index 628c6f77..8f834f22 100644 --- a/src/main/java/com/o19s/es/ltr/query/RankerQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/RankerQuery.java @@ -330,7 +330,6 @@ public DocIdSetIterator iterator() { */ @Override public float getMaxScore(int upTo) throws IOException { - //TODO?? return Float.POSITIVE_INFINITY; } }