Upgrade Lucene to 9.8.0

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
opensearch-project · Oct 6, 2023 · 7ae9980 · 7ae9980
1 parent 21398bd
commit 7ae9980
Show file tree

Hide file tree

Showing 55 changed files with 59 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
 - Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))
 - Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210))
+- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
 
 ### Changed
 - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 2.12.0
-lucene            = 9.7.0
+lucene            = 9.8.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.8+7

diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.8.0.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0.jar.sha1
@@ -0,0 +1 @@
+5e8421c5f8573bcf22e9265fc7e19469545a775a
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -57,7 +57,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
      * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99
      * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id
-     *
+     *>>
      * IMPORTANT: Unreleased vs. Released Versions
      *
      * All listed versions MUST be released versions, except the last major, the last minor and the last revison. ONLY those are required
@@ -119,7 +119,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_9_1 = new Version(2090199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0);
-    public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_7_0);
+    public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
     public static final Version CURRENT = V_2_12_0;
 
     public static Version fromId(int id) {

diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0.jar.sha1
@@ -0,0 +1 @@
+7725476acfcb9bdfeff1b813ce15c39c6b857dc2
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0.jar.sha1
@@ -0,0 +1 @@
+7133d34e92770f59eb28686f4d511b9f3f32e970
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0.jar.sha1
@@ -0,0 +1 @@
+be44282e1f6b91a0650fcceb558053d6bdd4863d
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0.jar.sha1
@@ -0,0 +1 @@
+bd1f80d33346f7e588685484ef29a304db5190e4
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0.jar.sha1
@@ -0,0 +1 @@
+b9ffdc7a52d2087ecb03318ec06305b480cdfe82
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0.jar.sha1
@@ -0,0 +1 @@
+f73e2007b133fb699e517ef13b4952844f0150d8
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0.jar.sha1
@@ -0,0 +1 @@
+2c09cbc021a8f81a01600a1d2a999361e70f7aed
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0.jar.sha1
@@ -0,0 +1 @@
+b054f2c7b11fc7c5601b4c3cdf18aa7508612898
diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.8.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0.jar.sha1
@@ -0,0 +1 @@
+36f0363325ca7bf62c180160d1ed5165c7c37795
diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0.jar.sha1
@@ -0,0 +1 @@
+e98fb408028f40170e6d87c16422bfdc0bb2e392
diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-core-9.8.0.jar.sha1 b/server/licenses/lucene-core-9.8.0.jar.sha1
@@ -0,0 +1 @@
+5e8421c5f8573bcf22e9265fc7e19469545a775a
diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.8.0.jar.sha1 b/server/licenses/lucene-grouping-9.8.0.jar.sha1
@@ -0,0 +1 @@
+d39184518351178c404ed9669fc6cb6111f2288d
diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.8.0.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0.jar.sha1
@@ -0,0 +1 @@
+1ac38c8278dbd63dfab30744a41dd955a415a31c
diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-join-9.8.0.jar.sha1 b/server/licenses/lucene-join-9.8.0.jar.sha1
@@ -0,0 +1 @@
+3d64fc57bb6e718d906413a9f73c713e6d4d8bb0
diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-memory-9.8.0.jar.sha1 b/server/licenses/lucene-memory-9.8.0.jar.sha1
@@ -0,0 +1 @@
+5283ac71d6ccecb5e00c7b52df2faec012f2625a
diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-misc-9.8.0.jar.sha1 b/server/licenses/lucene-misc-9.8.0.jar.sha1
@@ -0,0 +1 @@
+9a57b049cf51a5e9c9c1909c420f645f1b6f9a54
diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-queries-9.8.0.jar.sha1 b/server/licenses/lucene-queries-9.8.0.jar.sha1
@@ -0,0 +1 @@
+628db4ef46f1c6a05145bdac1d1bc4ace6341b13
diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.8.0.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0.jar.sha1
@@ -0,0 +1 @@
+982faf2bfa55542bf57fbadef54c19ac00f57cae
diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.8.0.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0.jar.sha1
@@ -0,0 +1 @@
+06493dbd14d02537716822254866a94458f4d842
diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0.jar.sha1
@@ -0,0 +1 @@
+9d9a731822ad6eefa1ba288a0c158d478522f165
diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.8.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0.jar.sha1
@@ -0,0 +1 @@
+ce752a52b2d4eac90633c7df7982e29504f99e76
diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.8.0.jar.sha1 b/server/licenses/lucene-suggest-9.8.0.jar.sha1
@@ -0,0 +1 @@
+f977f96f2093b7fddea6b67caa2e1c5b10edebf6
diff --git a/...sterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/...sterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java
@@ -108,8 +108,9 @@ public void testConcurrentSearchTaskTracking() {
             assertEquals(mainTaskInfo.getTaskId(), taskInfo.getParentTaskId());
 
             Map<Long, List<ThreadResourceInfo>> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId());
-            // Concurrent search forks each slice of 5 segments to different thread
-            assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size());
+            // Concurrent search forks each slice of 5 segments to different thread (see please
+            // https://github.com/apache/lucene/issues/12498)
+            assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0) + 1, threadStats.size());
 
             // assert that all task descriptions have non-zero length
             MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0));

diff --git a/...src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java b/...src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java
@@ -201,8 +201,18 @@ public void testExplainScript() throws InterruptedException, IOException, Execut
         for (SearchHit hit : hits.getHits()) {
             assertThat(hit.getId(), equalTo(Integer.toString(idCounter)));
             assertThat(hit.getExplanation().toString(), containsString(Double.toString(idCounter)));
-            assertThat(hit.getExplanation().toString(), containsString("1 = n"));
-            assertThat(hit.getExplanation().toString(), containsString("1 = N"));
+
+            // Since Apache Lucene 9.8, the scores are not computed because script (see please ExplainableScriptPlugin)
+            // says "needs_score() == false"
+            // 19.0 = min of:
+            // 19.0 = This script returned 19.0
+            // 0.0 = _score:
+            // 0.0 = weight(text:text in 0) [PerFieldSimilarity], result of:
+            // 0.0 = score(freq=1.0), with freq of:
+            // 1.0 = freq, occurrences of term within document
+            // 3.4028235E38 = maxBoost
+
+            assertThat(hit.getExplanation().toString(), containsString("1.0 = freq, occurrences of term within document"));
             assertThat(hit.getExplanation().getDetails().length, equalTo(2));
             idCounter--;
         }

diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java
@@ -77,6 +77,7 @@
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.KnnCollector;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.ScoreMode;
@@ -1152,14 +1153,10 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException {
             }
 
             @Override
-            public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
-                return null;
-            }
+            public void searchNearestVectors(String field, byte[] target, KnnCollector k, Bits acceptDocs) throws IOException {}
 
             @Override
-            public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
-                return null;
-            }
+            public void searchNearestVectors(String field, float[] target, KnnCollector k, Bits acceptDocs) throws IOException {}
         };
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java
@@ -52,7 +52,7 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.KnnCollector;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.common.util.set.Sets;
@@ -271,12 +271,12 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException {
     }
 
     @Override
-    public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+    public void searchNearestVectors(String field, byte[] target, KnnCollector k, Bits acceptDocs) throws IOException {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+    public void searchNearestVectors(String field, float[] target, KnnCollector k, Bits acceptDocs) throws IOException {
         throw new UnsupportedOperationException();
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java b/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java
@@ -86,14 +86,6 @@ public void copy(int slot, int doc) throws IOException {
             super.copy(slot, doc);
         }
 
-        @Override
-        protected boolean isMissingValueCompetitive() {
-            int result = missingValue.compareTo(bottom);
-            // in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom,
-            // in asc sort missingValue is competitive when it's smaller or equal to bottom
-            return reverse ? (result >= 0) : (result <= 0);
-        }
-
         @Override
         protected void encodeBottom(byte[] packedValue) {
             BigIntegerPoint.encodeDimension(bottom, packedValue, 0);
@@ -103,5 +95,15 @@ protected void encodeBottom(byte[] packedValue) {
         protected void encodeTop(byte[] packedValue) {
             BigIntegerPoint.encodeDimension(topValue, packedValue, 0);
         }
+
+        @Override
+        protected int compareMissingValueWithBottomValue() {
+            return missingValue.compareTo(bottom);
+        }
+
+        @Override
+        protected int compareMissingValueWithTopValue() {
+            return missingValue.compareTo(topValue);
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
@@ -116,6 +116,7 @@ private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSu
                 }
             }
         }
+        collector.finish();
     }
 
     @Override
-Original file line number
+Diff line change
@@ Expand Up @@
                     }
                 }
             }
+            collector.finish();
         }
         @Override
@@ Expand Down @@