Update Apache Lucene to 9.8.0-snapshot-4373c3b (opensearch-project#8668)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io> Signed-off-by: Shivansh Arora <hishiv@amazon.com>
Gaurav614 · Apr 25, 2024 · d35501c · d35501c
1 parent 84852f1
commit d35501c
Show file tree

Hide file tree

Showing 50 changed files with 39 additions and 28 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.7.0
+lucene            = 9.8.0-snapshot-4373c3b
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.2+9

diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+30c3afcf058532d3d2b8820375043000e7f34a9b
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -55,7 +55,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
      * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99
      * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id
-     *
+     *>>
      * IMPORTANT: Unreleased vs. Released Versions
      *
      * All listed versions MUST be released versions, except the last major, the last minor and the last revison. ONLY those are required
@@ -91,7 +91,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0);
     public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+9f8a34fc3d450343ab05ccb5af318a836a6a5fb3
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+fde64e3b23bc9a0849b9897febfe9f13c5113143
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+b01a791705fa01fce48dd02ea79fa8045de8dd5e
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+43d19320b1b9cd18638b1602fa87d5f21ee043bc
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+9244dc232f175010b480d4d88e13945c17a0b28b
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+3101a4f79820c1ca3dfb8f49b74c5fb5b32940e1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+f12b2a22cd5ebcd84f40a40e78fdd4e268b3b26d
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+7dbf5cc3dff93cc1ffe45d79b129859590d001dd
diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+1446b7641743a1082b566179d1bf2960f5a0724b
diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+127032ea137d2501b24f0e35e5f9a2e1c7864633
diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+30c3afcf058532d3d2b8820375043000e7f34a9b
diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+e6f742efe0ef3b383468fe38f88ab2dd69ed3d2c
diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+3162856444777130dee2c4cabe1bf6d18710ff63
diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+5fe8383516eca7300f978ce38042e327b0a57877
diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+b3e77970485be6d2dd59b999bbaa65a2cb993744
diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+86d667ea2f7fb2142d2acacf801dcea47d014a5e
diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+930d004de698f374da8ac5530fd80e241edeba45
diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+f62882823d5aa9ed4cf0081a8c18f35e21992080
diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+f1ec1527e283b423b7ff5e12cd8d889e7247199d
diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+de787c052879893e47d21fa161c93413665d55d7
diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+7e541ed960a571f5d9a0ecff5c26fd5ca857581e
diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+4b222ef09a5f20896d031a8322f2e69304c16384
diff --git a/...src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java b/...src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java
@@ -179,8 +179,18 @@ public void testExplainScript() throws InterruptedException, IOException, Execut
         for (SearchHit hit : hits.getHits()) {
             assertThat(hit.getId(), equalTo(Integer.toString(idCounter)));
             assertThat(hit.getExplanation().toString(), containsString(Double.toString(idCounter)));
-            assertThat(hit.getExplanation().toString(), containsString("1 = n"));
-            assertThat(hit.getExplanation().toString(), containsString("1 = N"));
+
+            // Since Apache Lucene 9.8, the scores are not computed because script (see please ExplainableScriptPlugin)
+            // says "needs_score() == false"
+            // 19.0 = min of:
+            // 19.0 = This script returned 19.0
+            // 0.0 = _score:
+            // 0.0 = weight(text:text in 0) [PerFieldSimilarity], result of:
+            // 0.0 = score(freq=1.0), with freq of:
+            // 1.0 = freq, occurrences of term within document
+            // 3.4028235E38 = maxBoost
+
+            assertThat(hit.getExplanation().toString(), containsString("1.0 = freq, occurrences of term within document"));
             assertThat(hit.getExplanation().getDetails().length, equalTo(2));
             idCounter--;
         }

diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
@@ -116,6 +116,7 @@ private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSu
                 }
             }
         }
+        collector.finish();
     }
 
     @Override
-Original file line number
+Diff line change
@@ Expand Up @@
                     }
                 }
             }
+            collector.finish();
         }
         @Override
@@ Expand Down @@