Bump to Lucene99

CHANGELOG.md

@@ -158,6 +158,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.maxmind.geoip2:geoip2` from 4.1.0 to 4.2.0 ([#11559](https://github.com/opensearch-project/OpenSearch/pull/11559))
 - Bump `org.apache.commons:commons-lang3` from 3.13.0 to 3.14.0 ([#11691](https://github.com/opensearch-project/OpenSearch/pull/11691))
 - Bump `com.maxmind.db:maxmind-db` from 3.0.0 to 3.1.0 ([#11693](https://github.com/opensearch-project/OpenSearch/pull/11693))
+- Bump `Lucene` from 9.8.0 to 9.9.1 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.8.0
+lucene            = 9.9.1
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.1+12

libs/core/licenses/lucene-core-9.9.1.jar.sha1

@@ -0,0 +1 @@
+55249fa9a0ed321adcf8283c6f3b649a6812b0a9

libs/core/src/main/java/org/opensearch/Version.java

@@ -99,7 +99,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_1);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1

@@ -0,0 +1 @@
+1782a69d0e83af9cc3c65db0dcd2e7e7c1e5f90e

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java

@@ -39,6 +39,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
@@ -122,35 +123,40 @@ public void testRangeQuery() throws IOException {
     public void testRoundsUpperBoundCorrectly() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
         Query scaledFloatQ = ft.rangeQuery(null, 0.1, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.1, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.095, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.095, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.105, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.105, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 79.99, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 7999]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 7999]", getQueryString(scaledFloatQ));
     }
 
     public void testRoundsLowerBoundCorrectly() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
         Query scaledFloatQ = ft.rangeQuery(-0.1, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.1, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.095, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.095, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.105, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.105, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
+    }
+
+    private String getQueryString(Query query) {
+        assertTrue(query instanceof IndexOrDocValuesQuery);
+        return ((IndexOrDocValuesQuery) query).getIndexQuery().toString();
     }
 
     public void testValueForSearch() {

modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java

@@ -79,7 +79,7 @@ public class Netty4HttpPipeliningHandlerTests extends OpenSearchTestCase {
     @After
     public void tearDown() throws Exception {
         waitingRequests.keySet().forEach(this::finishRequest);
-        shutdownExecutorService();
+        shutdownExecutorServices();
         super.tearDown();
     }
 
@@ -88,7 +88,7 @@ private CountDownLatch finishRequest(String url) {
         return finishingRequests.get(url);
     }
 
-    private void shutdownExecutorService() throws InterruptedException {
+    private void shutdownExecutorServices() throws InterruptedException {
         if (!handlerService.isShutdown()) {
             handlerService.shutdown();
             handlerService.awaitTermination(10, TimeUnit.SECONDS);

plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1

@@ -0,0 +1 @@
+147cb42a90a29501d9ca6094ea0db1d213f3076a

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1

@@ -0,0 +1 @@
+b034dd3a975763e083c7e11b5d0f7d516ab72590

plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java

@@ -38,7 +38,7 @@
 import org.apache.lucene.analysis.ja.JapaneseTokenizer;
 import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
 import org.apache.lucene.analysis.ja.dict.UserDictionary;
-import org.apache.lucene.analysis.ja.util.CSVUtil;
+import org.apache.lucene.analysis.util.CSVUtil;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.Environment;

plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1

@@ -0,0 +1 @@
+c405f2f7d0fc127d88dfbadd753469b2028fdf52

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1

@@ -0,0 +1 @@
+970e5775876c2d7e1b9af7421a4b17d96f63faf4

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1

@@ -0,0 +1 @@
+2421e5238e9b8484929291744d709dd743c01da1

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1

@@ -0,0 +1 @@
+a23e7de4cd9ae7af285c89dc1c55e0ac3f157fd3

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1

@@ -0,0 +1 @@
+8d9bce1ea51db279878c51091dd9aefc7b335da4

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java

@@ -9,7 +9,7 @@
 package org.opensearch.plugin.correlation.core.index.codec;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.correlation950.CorrelationCodec;
 import org.opensearch.plugin.correlation.core.index.codec.correlation950.PerFieldCorrelationVectorsFormat;
@@ -24,15 +24,15 @@
  * @opensearch.internal
  */
 public enum CorrelationCodecVersion {
-    V_9_5_0(
+    V_9_9_0(
         "CorrelationCodec",
-        new Lucene95Codec(),
+        new Lucene99Codec(),
         new PerFieldCorrelationVectorsFormat(Optional.empty()),
         (userCodec, mapperService) -> new CorrelationCodec(userCodec, new PerFieldCorrelationVectorsFormat(Optional.of(mapperService))),
         CorrelationCodec::new
     );
 
-    private static final CorrelationCodecVersion CURRENT = V_9_5_0;
+    private static final CorrelationCodecVersion CURRENT = V_9_9_0;
     private final String codecName;
     private final Codec defaultCodecDelegate;
     private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat;

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java

@@ -19,7 +19,7 @@
  * @opensearch.internal
  */
 public class CorrelationCodec extends FilterCodec {
-    private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_5_0;
+    private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_9_0;
     private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat;
 
     /**

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java

@@ -8,7 +8,7 @@
 
 package org.opensearch.plugin.correlation.core.index.codec.correlation950;
 
-import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat;
 
@@ -26,10 +26,10 @@ public class PerFieldCorrelationVectorsFormat extends BasePerFieldCorrelationVec
     public PerFieldCorrelationVectorsFormat(final Optional<MapperService> mapperService) {
         super(
             mapperService,
-            Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN,
-            Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-            Lucene95HnswVectorsFormat::new,
-            Lucene95HnswVectorsFormat::new
+            Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+            Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+            Lucene99HnswVectorsFormat::new,
+            Lucene99HnswVectorsFormat::new
         );
     }
 }

plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java

@@ -32,7 +32,7 @@
 
 import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_EF_CONSTRUCTION;
 import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_M;
-import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_5_0;
+import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_9_0;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -56,7 +56,7 @@ public void testCorrelationVectorIndex() throws Exception {
         Function<MapperService, PerFieldCorrelationVectorsFormat> perFieldCorrelationVectorsProvider =
             mapperService -> new PerFieldCorrelationVectorsFormat(Optional.of(mapperService));
         Function<PerFieldCorrelationVectorsFormat, Codec> correlationCodecProvider = (correlationVectorsFormat -> new CorrelationCodec(
-            V_9_5_0.getDefaultCodecDelegate(),
+            V_9_9_0.getDefaultCodecDelegate(),
             correlationVectorsFormat
         ));
         testCorrelationVectorIndex(correlationCodecProvider, perFieldCorrelationVectorsProvider);

plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java

@@ -80,15 +80,15 @@ public class NioHttpPipeliningHandlerTests extends OpenSearchTestCase {
     @After
     public void cleanup() throws Exception {
         waitingRequests.keySet().forEach(this::finishRequest);
-        shutdownExecutorService();
+        shutdownExecutorServices();
     }
 
     private CountDownLatch finishRequest(String url) {
         waitingRequests.get(url).countDown();
         return finishingRequests.get(url);
     }
 
-    private void shutdownExecutorService() throws InterruptedException {
+    private void shutdownExecutorServices() throws InterruptedException {
         if (!handlerService.isShutdown()) {
             handlerService.shutdown();
             handlerService.awaitTermination(10, TimeUnit.SECONDS);

server/licenses/lucene-analysis-common-9.9.1.jar.sha1

@@ -0,0 +1 @@
+24c8401b530308f9568eb7b408c2029c63f564c6

server/licenses/lucene-backward-codecs-9.9.1.jar.sha1

@@ -0,0 +1 @@
+11c46007366bb037be7d271ab0a5849b1d544662

server/licenses/lucene-core-9.9.1.jar.sha1

@@ -0,0 +1 @@
+55249fa9a0ed321adcf8283c6f3b649a6812b0a9

server/licenses/lucene-grouping-9.9.1.jar.sha1

@@ -0,0 +1 @@
+2f2785e17c5c823cc8f41a7ddb4647aaca8ee773

server/licenses/lucene-highlighter-9.9.1.jar.sha1

@@ -0,0 +1 @@
+30928513461bf79a5cb057e84da7d34a1e53227d

server/licenses/lucene-join-9.9.1.jar.sha1

@@ -0,0 +1 @@
+b9c8cc99632280148f92b4c0a64111c482d5d0ac

server/licenses/lucene-memory-9.9.1.jar.sha1

@@ -0,0 +1 @@
+49f820b1b321860fa42a4f7583e8ed8f77b9c1c2

server/licenses/lucene-misc-9.9.1.jar.sha1

@@ -0,0 +1 @@
+db7c30217602dfcda394a4d0f0a9e68140d385a6

server/licenses/lucene-queries-9.9.1.jar.sha1

@@ -0,0 +1 @@
+d157547bd24edc8e9d9d59c273107dc3ac5fde5e

server/licenses/lucene-queryparser-9.9.1.jar.sha1

@@ -0,0 +1 @@
+12d844fe224f6f97c510ac20d68903ed7f626f6c

server/licenses/lucene-sandbox-9.9.1.jar.sha1

@@ -0,0 +1 @@
+272e588fd3d8c0a401b28a1ac715f27044bf62ec

server/licenses/lucene-spatial-extras-9.9.1.jar.sha1

@@ -0,0 +1 @@
+e066432e7ab02b2a4914f989bcd8c44adbf340ad

server/licenses/lucene-spatial3d-9.9.1.jar.sha1

@@ -0,0 +1 @@
+fa54c9b962778e28ebc0efb9f75297781350361a

server/licenses/lucene-suggest-9.9.1.jar.sha1

@@ -0,0 +1 @@
+9554de5b22ae7483b344b94a9a956960b7a5d49c

server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java

@@ -710,6 +710,7 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro
 
         final NodeStats primaryNode = dataNodeStats.get(0);
         final NodeStats replicaNode = dataNodeStats.get(1);
+
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder()
@@ -795,6 +796,17 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro
 
         // Assert the cluster returns to green status because the replica will be promoted to primary
         ensureGreen();
+
+        // After Lucene 9.9 check index will flag corruption with old (not the latest) commit points.
+        // For this test our previous corrupt commit "segments_2" will remain on the primary.
+        // This asserts this is the case, and then resets check index status.
+        assertEquals("Check index has a single failure", 1, checkIndexFailures.size());
+        assertTrue(
+            checkIndexFailures.get(0)
+                .getMessage()
+                .contains("could not read old (not latest) commit point segments file \"segments_2\" in directory")
+        );
+        resetCheckIndexStatus();
     }
 
     private int numShards(String... index) {

server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java

@@ -270,7 +270,10 @@ public void testExplainDateRangeInQueryString() {
 
         long twoMonthsAgo = now.minus(2, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000;
         long rangeEnd = (now.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000) - 1;
-        assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]"));
+        assertThat(
+            response.getQueryExplanation().get(0).getExplanation(),
+            containsString("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]")
+        );
         assertThat(response.isValid(), equalTo(true));
     }
 

server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java

@@ -100,11 +100,10 @@ public Query rewrite(IndexSearcher searcher) throws IOException {
             return rewritten;
         }
         IndexReader reader = searcher.getIndexReader();
-        IndexReaderContext context = reader.getContext();
         TermStates[] ctx = new TermStates[terms.length];
         int[] docFreqs = new int[ctx.length];
         for (int i = 0; i < terms.length; i++) {
-            ctx[i] = TermStates.build(context, terms[i], true);
+            ctx[i] = TermStates.build(searcher, terms[i], true);
             docFreqs[i] = ctx[i].docFreq();
         }