[Upgrade] Lucene 9.5.0 release (#6078)

Upgrades to official release of Lucene 9.5. Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
opensearch-project · Jan 31, 2023 · 8d5d2e6 · 8d5d2e6
1 parent c557f27
commit 8d5d2e6
Show file tree

Hide file tree

Showing 53 changed files with 63 additions and 34 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -89,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - OpenJDK Update (January 2023 Patch releases) ([#6074](https://github.com/opensearch-project/OpenSearch/pull/6074))
 - Bumps `Mockito` from 4.7.0 to 5.1.0, `ByteBuddy` from 1.12.18 to 1.12.22 ([#6076](https://github.com/opensearch-project/OpenSearch/pull/6076))
 - Bumps `joda` from 2.10.13 to 2.12.2 ([#6083](https://github.com/opensearch-project/OpenSearch/pull/6083))
+- Upgrade to Lucene 9.5.0 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/6078))
 
 ### Changed
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.5.0-snapshot-0878271
+lucene            = 9.5.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 19.0.2+7

diff --git a/...is-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java b/...is-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java
@@ -50,6 +50,7 @@
 import static org.opensearch.client.Requests.searchRequest;
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.opensearch.index.query.QueryBuilders.boolQuery;
+import static org.opensearch.index.query.QueryBuilders.matchAllQuery;
 import static org.opensearch.index.query.QueryBuilders.matchPhrasePrefixQuery;
 import static org.opensearch.index.query.QueryBuilders.matchPhraseQuery;
 import static org.opensearch.index.query.QueryBuilders.matchQuery;
@@ -61,6 +62,7 @@
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.startsWith;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 
 public class HighlighterWithAnalyzersTests extends OpenSearchIntegTestCase {
     @Override
@@ -270,9 +272,13 @@ public void testPhrasePrefix() throws IOException {
         refresh();
         logger.info("--> highlighting and searching on field0");
 
-        SearchSourceBuilder source = searchSource().query(matchPhrasePrefixQuery("field0", "bro"))
-            .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>"));
+        SearchSourceBuilder source = searchSource().query(matchAllQuery());
         SearchResponse searchResponse = client().search(searchRequest("first_test_index").source(source)).actionGet();
+        assertHitCount(searchResponse, 2);
+
+        source = searchSource().query(matchPhrasePrefixQuery("field0", "bro"))
+            .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>"));
+        searchResponse = client().search(searchRequest("first_test_index").source(source)).actionGet();
 
         assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
 
@@ -415,6 +421,7 @@ public void testPhrasePrefix() throws IOException {
     public static XContentBuilder type1TermVectorMapping() throws IOException {
         return XContentFactory.jsonBuilder()
             .startObject()
+            .startObject("_doc")
             .startObject("properties")
             .startObject("field1")
             .field("type", "text")
@@ -425,6 +432,7 @@ public static XContentBuilder type1TermVectorMapping() throws IOException {
             .field("term_vector", "with_positions_offsets")
             .endObject()
             .endObject()
+            .endObject()
             .endObject();
     }
 }
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.5.0-snapshot-0878271.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.5.0-snapshot-0878271.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1
@@ -0,0 +1 @@
+62f3d3630ecc14069d5c24b9693df5a2787f8202
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0-snapshot-0878271.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1
@@ -0,0 +1 @@
+11c816250e4ff106151fd8cb69e61ead4fb4a8dd
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0-snapshot-0878271.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1
@@ -0,0 +1 @@
+4555fc64ce9d63c6c1f4983e816526f896606e9f
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0-snapshot-0878271.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1
@@ -0,0 +1 @@
+2fd55607da8adf8234169d9f4120119c05a8175c
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0-snapshot-0878271.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1
@@ -0,0 +1 @@
+c97163035f1d46249505d15fbdc427b6ae03549a
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0-snapshot-0878271.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1
@@ -0,0 +1 @@
+a08ae3e2212837c98d5e1174cf1a293a5c8c0373
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0-snapshot-0878271.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1
@@ -0,0 +1 @@
+483906d2ebad0ff910d52382c2f027b511788b58
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0-snapshot-0878271.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0-snapshot-0878271.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1
@@ -0,0 +1 @@
+cb00cd193e2f927e6357cf33e0880034571ea528
diff --git a/server/licenses/lucene-analysis-common-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-analysis-common-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.5.0.jar.sha1
@@ -0,0 +1 @@
+f68660102455a466f98cac0501723bed7e7c6407
diff --git a/server/licenses/lucene-backward-codecs-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-backward-codecs-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1
@@ -0,0 +1 @@
+66be22239d5058b0b8a8aeba03dc047a276efafa
diff --git a/server/licenses/lucene-core-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-core-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-core-9.5.0.jar.sha1 b/server/licenses/lucene-core-9.5.0.jar.sha1
@@ -0,0 +1 @@
+bba4ba5d30e71a5f0017e45e8469db8cff8ad102
diff --git a/server/licenses/lucene-grouping-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-grouping-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.5.0.jar.sha1 b/server/licenses/lucene-grouping-9.5.0.jar.sha1
@@ -0,0 +1 @@
+b15fe0a55a82168c810d0447ec2e244d16d94f01
diff --git a/server/licenses/lucene-highlighter-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-highlighter-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.5.0.jar.sha1 b/server/licenses/lucene-highlighter-9.5.0.jar.sha1
@@ -0,0 +1 @@
+8b5e3cea3370838bda1d730cf55176c24a763d2e
diff --git a/server/licenses/lucene-join-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-join-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-join-9.5.0.jar.sha1 b/server/licenses/lucene-join-9.5.0.jar.sha1
@@ -0,0 +1 @@
+85a4208614a5660297effce441883687b010073b
diff --git a/server/licenses/lucene-memory-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-memory-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-memory-9.5.0.jar.sha1 b/server/licenses/lucene-memory-9.5.0.jar.sha1
@@ -0,0 +1 @@
+d632e63d08837be715046c8ccb4fb804acd3d7e4
diff --git a/server/licenses/lucene-misc-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-misc-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-misc-9.5.0.jar.sha1 b/server/licenses/lucene-misc-9.5.0.jar.sha1
@@ -0,0 +1 @@
+64773801b8ba8141f4256d22da598de40d6f3033
diff --git a/server/licenses/lucene-queries-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-queries-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-queries-9.5.0.jar.sha1 b/server/licenses/lucene-queries-9.5.0.jar.sha1
@@ -0,0 +1 @@
+ae7930fa0ea91198905d695a222ef7f09de1c2dd
diff --git a/server/licenses/lucene-queryparser-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-queryparser-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.5.0.jar.sha1 b/server/licenses/lucene-queryparser-9.5.0.jar.sha1
@@ -0,0 +1 @@
+b3e3d9434bcfce242e242364c64eab0888d285e2
diff --git a/server/licenses/lucene-sandbox-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-sandbox-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.5.0.jar.sha1 b/server/licenses/lucene-sandbox-9.5.0.jar.sha1
@@ -0,0 +1 @@
+408be287b0f421cf8afd655579a0cc65aba61b28
diff --git a/server/licenses/lucene-spatial-extras-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-spatial-extras-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1
@@ -0,0 +1 @@
+4960c09365d20f024bf668f60d7c3b4e54f03750
diff --git a/server/licenses/lucene-spatial3d-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-spatial3d-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.5.0.jar.sha1
@@ -0,0 +1 @@
+01ed55dd384e654076b76d7083feee6ac3efe02b
diff --git a/server/licenses/lucene-suggest-9.5.0-snapshot-0878271.jar.sha1 b/server/licenses/lucene-suggest-9.5.0-snapshot-0878271.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.5.0.jar.sha1 b/server/licenses/lucene-suggest-9.5.0.jar.sha1
@@ -0,0 +1 @@
+195624eb3a09b3dc37cc5fa51ae2f7bd61553985
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -325,6 +325,20 @@ private boolean termArraysEquals(List<Term[]> termArrays1, List<Term[]> termArra
 
     @Override
     public void visit(QueryVisitor visitor) {
-        visitor.visitLeaf(this);
+        if (visitor.acceptField(field)) {
+            // set by the subvisitor because the default returns, *this*, unless MUST_NOT is used which returns
+            // an ideal EMPTY_VISITOR
+            visitor = visitor.getSubVisitor(BooleanClause.Occur.MUST, this);
+            for (int i = 0; i < termArrays.size() - 1; ++i) {
+                if (termArrays.get(i).length == 1) {
+                    visitor.consumeTerms(this, termArrays.get(i)[0]); // use a must if we only have a single phrase
+                } else {
+                    // if more than one sub-phrase
+                    // we should match at least one
+                    QueryVisitor subVisitor = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this);
+                    subVisitor.consumeTerms(this, termArrays.get(i));
+                }
+            }
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java
@@ -32,10 +32,12 @@
 package org.opensearch.index.engine;
 
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafMetaData;
 import org.apache.lucene.index.LeafReader;
@@ -50,7 +52,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -255,8 +256,13 @@ public CacheHelper getReaderCacheHelper() {
     }
 
     @Override
-    public VectorValues getVectorValues(String field) throws IOException {
-        return getVectorValues(field);
+    public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+        return getFloatVectorValues(field);
+    }
+
+    @Override
+    public ByteVectorValues getByteVectorValues(String field) throws IOException {
+        return getByteVectorValues(field);
     }
 
     @Override
@@ -265,7 +271,7 @@ public TopDocs searchNearestVectors(String field, float[] target, int k, Bits ac
     }
 
     @Override
-    public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+    public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
         throw new UnsupportedOperationException();
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
@@ -37,7 +37,7 @@
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PointValues;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.Query;

diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
@@ -42,7 +42,7 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;

diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
@@ -40,7 +40,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.Query;

diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java
@@ -43,7 +43,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;