update for ES7.0.0

build.gradle

@@ -6,12 +6,12 @@ buildscript {
   }
 
   dependencies {
-    classpath "org.elasticsearch.gradle:build-tools:6.7.1"
+    classpath "org.elasticsearch.gradle:build-tools:7.0.0"
   }
 }
 
 group = 'com.o19s'
-version = '1.1.1-es6.7.1'
+version = '1.1.1-es7.0.0'
 
 apply plugin: 'java'
 apply plugin: 'idea'
@@ -46,15 +46,16 @@ repositories {
 }
 
 dependencies {
-  compile "org.apache.lucene:lucene-expressions:7.7.0"
+  compile "org.apache.lucene:lucene-expressions:8.0.0"
   compile 'org.antlr:antlr4-runtime:4.5.1-1'
   compile 'org.ow2.asm:asm:5.0.4'
   compile 'org.ow2.asm:asm-commons:5.0.4'
   compile 'org.ow2.asm:asm-tree:5.0.4'
-  compile 'org.elasticsearch:elasticsearch:6.7.1'
+  compile 'org.elasticsearch:elasticsearch:7.0.0'
   compile 'com.o19s:RankyMcRankFace:0.1.1'
   compile "com.github.spullara.mustache.java:compiler:0.9.3"
-  testCompile 'org.elasticsearch.test:framework:6.7.1'
+  testCompile 'org.elasticsearch.test:framework:7.0.0'
+
 }
 
 dependencyLicenses {

gradle/wrapper/gradle-wrapper.properties

@@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 distributionUrl=https\://services.gradle.org/distributions/gradle-5.4-bin.zip
 zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
+zipStorePath=wrapper/dists

src/main/java/com/o19s/es/explore/ExplorerQuery.java

@@ -13,12 +13,13 @@
  * limitations under the License.
  *
  */
+
 package com.o19s.es.explore;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.TermStates;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreScorer;
@@ -27,6 +28,7 @@
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.Weight;
@@ -54,6 +56,7 @@ private boolean isCollectionScoped() {
     }
 
     public Query getQuery() { return this.query; }
+
     public String getType() { return this.type; }
 
     @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
@@ -72,7 +75,7 @@ private boolean equalsTo(ExplorerQuery other) {
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = query.rewrite(reader);
 
-        if(rewritten != query) {
+        if (rewritten != query) {
             return new ExplorerQuery(rewritten, type);
         }
 
@@ -85,25 +88,29 @@ public int hashCode() {
     }
 
     @Override
-    public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
-        if (!needsScores) {
-            return searcher.createWeight(query, false, boost);
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
+            throws IOException {
+        if (!scoreMode.needsScores()) {
+            return searcher.createWeight(query, scoreMode, boost);
         }
-        final Weight subWeight = searcher.createWeight(query, true, boost);
+        final Weight subWeight = searcher.createWeight(query, scoreMode, boost);
         Set<Term> terms = new HashSet<>();
         subWeight.extractTerms(terms);
-        if(isCollectionScoped()) {
+        if (isCollectionScoped()) {
             ClassicSimilarity sim = new ClassicSimilarity();
             StatisticsHelper df_stats = new StatisticsHelper();
             StatisticsHelper idf_stats = new StatisticsHelper();
             StatisticsHelper ttf_stats = new StatisticsHelper();
 
-            for(Term term : terms) {
-                TermContext ctx = TermContext.build(searcher.getTopReaderContext(), term);
+            for (Term term : terms) {
+                TermStates ctx = TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores());
                 TermStatistics tStats = searcher.termStatistics(term, ctx);
-                df_stats.add(tStats.docFreq());
-                idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs()));
-                ttf_stats.add(tStats.totalTermFreq());
+                if(tStats != null){
+                    df_stats.add(tStats.docFreq());
+                    idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs()));
+                    ttf_stats.add(tStats.totalTermFreq());
+
+                }
             }
 
             /*
@@ -113,7 +120,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores, float bo
              */
             float constantScore;
 
-            if(terms.size() > 0) {
+            if (terms.size() > 0) {
                 switch (type) {
                     case ("sum_classic_idf"):
                         constantScore = idf_stats.getSum();
@@ -185,7 +192,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
 
                 @Override
                 public Scorer scorer(LeafReaderContext context) throws IOException {
-                    return new ConstantScoreScorer(this, constantScore, DocIdSetIterator.all(context.reader().maxDoc()));
+                    return new ConstantScoreScorer(this, constantScore, scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
                 }
 
                 @Override
@@ -198,15 +205,16 @@ public boolean isCacheable(LeafReaderContext ctx) {
             // Rewrite this into a boolean query where we can inject our PostingsExplorerQuery
             BooleanQuery.Builder qb = new BooleanQuery.Builder();
             for (Term t : terms) {
-                qb.add(new BooleanClause(new PostingsExplorerQuery(t, PostingsExplorerQuery.Type.TF), BooleanClause.Occur.SHOULD));
+                qb.add(new BooleanClause(new PostingsExplorerQuery(t, PostingsExplorerQuery.Type.TF),
+                        BooleanClause.Occur.SHOULD));
             }
             // FIXME: completely refactor this class and stop accepting a random query but a list of terms directly
             // rewriting at this point is wrong, additionally we certainly build the TermContext twice for every terms
             // problem is that we rely on extractTerms which happen too late in the process
             Query q = qb.build().rewrite(searcher.getIndexReader());
-            return new ExplorerQuery.ExplorerWeight(this, searcher.createWeight(q, true, boost), type);
+            return new ExplorerQuery.ExplorerWeight(this, searcher.createWeight(q, scoreMode, boost), type);
         }
-        throw new IllegalArgumentException( "Unknown ExplorerQuery type [" + type + "]" );
+        throw new IllegalArgumentException("Unknown ExplorerQuery type [" + type + "]");
     }
 
     static class ExplorerWeight extends Weight {

src/main/java/com/o19s/es/explore/ExplorerScorer.java

@@ -37,7 +37,7 @@ public float score() throws IOException {
 
         // Grab freq from subscorer, or the children if available
         if(subScorer.getChildren().size() > 0) {
-            for(ChildScorer child : subScorer.getChildren()) {
+            for(ChildScorable child : subScorer.getChildren()) {
                 assert child.child instanceof PostingsExplorerQuery.PostingsExplorerScorer;
                 if(child.child.docID() == docID()) {
                     tf_stats.add(child.child.score());
@@ -83,4 +83,14 @@ public int docID() {
     public DocIdSetIterator iterator() {
         return subScorer.iterator();
     }
+
+    /**
+     * Return the maximum score that documents between the last {@code target}
+     * that this iterator was {@link #advanceShallow(int) shallow-advanced} to
+     * included and {@code upTo} included.
+     */
+    @Override
+    public float getMaxScore(int upTo) throws IOException {
+        return Float.POSITIVE_INFINITY;
+    }
 }

src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java

@@ -22,13 +22,14 @@
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
 import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.TermStates;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 
@@ -63,8 +64,8 @@ public String toString(String field) {
     @Override
     public boolean equals(Object obj) {
         return this.sameClassAs(obj)
-                && this.term.equals(((PostingsExplorerQuery)obj).term)
-                && this.type.equals(((PostingsExplorerQuery)obj).type);
+                && this.term.equals(((PostingsExplorerQuery) obj).term)
+                && this.type.equals(((PostingsExplorerQuery) obj).type);
     }
 
     @Override
@@ -73,10 +74,13 @@ public int hashCode() {
     }
 
     @Override
-    public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
+            throws IOException {
         IndexReaderContext context = searcher.getTopReaderContext();
-        assert needsScores : "Should not be used in filtering mode";
-        return new PostingsExplorerWeight(this, this.term, TermContext.build(context, this.term), this.type);
+        assert scoreMode.needsScores() : "Should not be used in filtering mode";
+        return new PostingsExplorerWeight(this, this.term, TermStates.build(context, this.term,
+                scoreMode.needsScores()),
+                this.type);
     }
 
     /**
@@ -100,10 +104,10 @@ public Scorer apply(Weight weight, TermsEnum termsEnum) throws IOException {
 
     static class PostingsExplorerWeight extends Weight {
         private final Term term;
-        private final TermContext termStates;
+        private final TermStates termStates;
         private final Type type;
 
-        PostingsExplorerWeight(Query query, Term term, TermContext termStates, Type type) {
+        PostingsExplorerWeight(Query query, Term term, TermStates termStates, Type type) {
             super(query);
             this.term = term;
             this.termStates = termStates;
@@ -120,15 +124,17 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
             Scorer scorer = this.scorer(context);
             int newDoc = scorer.iterator().advance(doc);
             if (newDoc == doc) {
-                return Explanation.match(scorer.score(), "weight(" + this.getQuery()  + " in doc " + newDoc + ")");
+                return Explanation
+                        .match(scorer.score(), "weight(" + this.getQuery() + " in doc " + newDoc + ")");
             }
-            return Explanation.noMatch("no matching term" );
+            return Explanation.noMatch("no matching term");
         }
 
         @Override
         public Scorer scorer(LeafReaderContext context) throws IOException {
-            assert this.termStates != null && this.termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context));
-            TermState state = this.termStates.get(context.ord);
+            assert this.termStates != null && this.termStates
+                    .wasBuiltFor(ReaderUtil.getTopLevelContext(context));
+            TermState state = this.termStates.get(context);
             if (state == null) {
                 return null;
             } else {
@@ -172,5 +178,15 @@ static class TFScorer extends PostingsExplorerScorer {
         public float score() throws IOException {
             return this.postingsEnum.freq();
         }
+
+        /**
+         * Return the maximum score that documents between the last {@code target}
+         * that this iterator was {@link #advanceShallow(int) shallow-advanced} to
+         * included and {@code upTo} included.
+         */
+        @Override
+        public float getMaxScore(int upTo) throws IOException {
+            return Float.POSITIVE_INFINITY;
+        }
     }
 }

src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java

@@ -16,7 +16,6 @@
  */
 package com.o19s.es.ltr;
 
-import ciir.umass.edu.learning.RankerFactory;
 import com.o19s.es.explore.ExplorerQueryBuilder;
 import com.o19s.es.ltr.action.AddFeaturesToSetAction;
 import com.o19s.es.ltr.action.CachesStatsAction;
@@ -53,6 +52,7 @@
 import com.o19s.es.ltr.rest.RestSimpleFeatureStore;
 import com.o19s.es.ltr.utils.FeatureStoreLoader;
 import com.o19s.es.ltr.utils.Suppliers;
+
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.miscellaneous.LengthFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
@@ -100,6 +100,8 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import ciir.umass.edu.learning.RankerFactory;
+
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.unmodifiableList;
@@ -253,6 +255,6 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
 
     public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
         return Collections.singletonList(PreConfiguredTokenizer.singleton("ltr_keyword",
-                () -> new KeywordTokenizer(KeywordTokenizer.DEFAULT_BUFFER_SIZE), null));
+                () -> new KeywordTokenizer(KeywordTokenizer.DEFAULT_BUFFER_SIZE)));
     }
 }

src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java

@@ -36,28 +36,21 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class AddFeaturesToSetAction extends Action<AddFeaturesToSetAction.AddFeaturesToSetRequest,
-        AddFeaturesToSetAction.AddFeaturesToSetResponse, AddFeaturesToSetAction.AddFeaturesToSetRequestBuilder> {
+public class AddFeaturesToSetAction extends Action<AddFeaturesToSetAction.AddFeaturesToSetResponse> {
     public static final AddFeaturesToSetAction INSTANCE = new AddFeaturesToSetAction();
     public static final String NAME = "cluster:admin/ltr/store/add-features-to-set";
 
     protected AddFeaturesToSetAction() {
         super(NAME);
     }
 
-    @Override
-    public AddFeaturesToSetRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new AddFeaturesToSetRequestBuilder(client);
-    }
-
     @Override
     public AddFeaturesToSetResponse newResponse() {
         return new AddFeaturesToSetResponse();
     }
 
-    public static class AddFeaturesToSetRequestBuilder extends ActionRequestBuilder<AddFeaturesToSetRequest,
-            AddFeaturesToSetResponse, AddFeaturesToSetRequestBuilder> {
-        protected AddFeaturesToSetRequestBuilder(ElasticsearchClient client) {
+    public static class AddFeaturesToSetRequestBuilder extends ActionRequestBuilder<AddFeaturesToSetRequest, AddFeaturesToSetResponse> {
+        public AddFeaturesToSetRequestBuilder(ElasticsearchClient client) {
             super(client, INSTANCE, new AddFeaturesToSetRequest());
         }
     }

src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java

@@ -37,31 +37,19 @@
 import java.util.List;
 import java.util.Map;
 
-public class CachesStatsAction extends Action<CachesStatsAction.CachesStatsNodesRequest,
-        CachesStatsAction.CachesStatsNodesResponse, CachesStatsAction.CacheStatsRequestBuilder> {
+public class CachesStatsAction extends Action<CachesStatsAction.CachesStatsNodesResponse> {
     public static final String NAME = "cluster:admin/ltr/caches/stats";
     public static final CachesStatsAction INSTANCE = new CachesStatsAction();
 
     protected CachesStatsAction() {
         super(NAME);
     }
 
-    @Override
-    public CacheStatsRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new CacheStatsRequestBuilder(client);
-    }
-
     @Override
     public CachesStatsNodesResponse newResponse() {
         return new CachesStatsNodesResponse();
     }
 
-    public static class CacheStatsRequestBuilder extends ActionRequestBuilder<CachesStatsAction.CachesStatsNodesRequest,
-            CachesStatsAction.CachesStatsNodesResponse, CacheStatsRequestBuilder> {
-        protected CacheStatsRequestBuilder(ElasticsearchClient client) {
-            super(client, INSTANCE, new CachesStatsNodesRequest());
-        }
-    }
 
     public static class CachesStatsNodesRequest extends BaseNodesRequest<CachesStatsNodesRequest> {
     }
@@ -309,4 +297,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             }
         }
     }
+
+    public static class CachesStatsActionBuilder extends
+        ActionRequestBuilder<CachesStatsNodesRequest, CachesStatsNodesResponse> {
+        public CachesStatsActionBuilder(ElasticsearchClient client){
+            super(client, INSTANCE, new CachesStatsNodesRequest());
+        }
+    }
+
 }