Skip to content

Commit

Permalink
[Backport 2.x] Bump to Lucene99 (#11832)
Browse files Browse the repository at this point in the history
* Bump to Lucene99 (#11421)

Upgraded Lucene dependency to 9.9.1.

---------

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>
Signed-off-by: Marc Handalian <handalm@amazon.com>
(cherry picked from commit 4a42150)
Signed-off-by: Marc Handalian <marc.handalian@gmail.com>

* update deprecated CodecService constructor only in 2.x

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>

* Remove unused emptyReader method in Lucene.java

This method has a conflict inside leafReader's getMetaData as the constructor
to LeafMetaData has changed.  However this method is never invoked so removing it.

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>

* Fix Version for 2.12.0 to correctly point to Lucene 9.9.1

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>

---------

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>
Signed-off-by: Marc Handalian <handalm@amazon.com>
  • Loading branch information
mch2 authored Jan 10, 2024
1 parent a41646e commit de26624
Show file tree
Hide file tree
Showing 90 changed files with 204 additions and 242 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `commons-cli:commons-cli` from 1.5.0 to 1.6.0 ([#10996](https://github.com/opensearch-project/OpenSearch/pull/10996))
- Bump `org.apache.commons:commons-lang3` from 3.13.0 to 3.14.0 ([#11691](https://github.com/opensearch-project/OpenSearch/pull/11691))
- Bump `net.java.dev.jna:jna` from 5.13.0 to 5.14.0 ([#11798](https://github.com/opensearch-project/OpenSearch/pull/11798))
- Bump `Lucene` from 9.8.0 to 9.9.1 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421))

### Changed
- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 2.12.0
lucene = 9.8.0
lucene = 9.9.1

bundled_jdk_vendor = adoptium
bundled_jdk = 21.0.1+12
Expand Down
1 change: 0 additions & 1 deletion libs/core/licenses/lucene-core-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions libs/core/licenses/lucene-core-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
55249fa9a0ed321adcf8283c6f3b649a6812b0a9
2 changes: 1 addition & 1 deletion libs/core/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0);
public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0);
public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0);
public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_1);
public static final Version CURRENT = V_2_12_0;

public static Version fromId(int id) {
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1782a69d0e83af9cc3c65db0dcd2e7e7c1e5f90e
Original file line number Diff line number Diff line change
Expand Up @@ -135,35 +135,40 @@ public void testRangeQuery() throws IOException {
public void testRoundsUpperBoundCorrectly() {
ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
Query scaledFloatQ = ft.rangeQuery(null, 0.1, true, false, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(null, 0.1, true, true, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(null, 0.095, true, false, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(null, 0.095, true, true, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(null, 0.105, true, false, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(null, 0.105, true, true, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(null, 79.99, true, true, MOCK_QSC);
assertEquals("scaled_float:[-9223372036854775808 TO 7999]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9223372036854775808 TO 7999]", getQueryString(scaledFloatQ));
}

public void testRoundsLowerBoundCorrectly() {
ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
Query scaledFloatQ = ft.rangeQuery(-0.1, null, false, true, MOCK_QSC);
assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(-0.1, null, true, true, MOCK_QSC);
assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(-0.095, null, false, true, MOCK_QSC);
assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(-0.095, null, true, true, MOCK_QSC);
assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(-0.105, null, false, true, MOCK_QSC);
assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
scaledFloatQ = ft.rangeQuery(-0.105, null, true, true, MOCK_QSC);
assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
}

private String getQueryString(Query query) {
assertTrue(query instanceof IndexOrDocValuesQuery);
return ((IndexOrDocValuesQuery) query).getIndexQuery().toString();
}

public void testValueForSearch() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public class Netty4HttpPipeliningHandlerTests extends OpenSearchTestCase {
@After
public void tearDown() throws Exception {
waitingRequests.keySet().forEach(this::finishRequest);
shutdownExecutorService();
shutdownExecutorServices();
super.tearDown();
}

Expand All @@ -88,7 +88,7 @@ private CountDownLatch finishRequest(String url) {
return finishingRequests.get(url);
}

private void shutdownExecutorService() throws InterruptedException {
private void shutdownExecutorServices() throws InterruptedException {
if (!handlerService.isShutdown()) {
handlerService.shutdown();
handlerService.awaitTermination(10, TimeUnit.SECONDS);
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
147cb42a90a29501d9ca6094ea0db1d213f3076a

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b034dd3a975763e083c7e11b5d0f7d516ab72590
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.ja.util.CSVUtil;
import org.apache.lucene.analysis.util.CSVUtil;
import org.opensearch.OpenSearchException;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.Environment;
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c405f2f7d0fc127d88dfbadd753469b2028fdf52

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
970e5775876c2d7e1b9af7421a4b17d96f63faf4

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2421e5238e9b8484929291744d709dd743c01da1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a23e7de4cd9ae7af285c89dc1c55e0ac3f157fd3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8d9bce1ea51db279878c51091dd9aefc7b335da4
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,15 @@ public class NioHttpPipeliningHandlerTests extends OpenSearchTestCase {
@After
public void cleanup() throws Exception {
waitingRequests.keySet().forEach(this::finishRequest);
shutdownExecutorService();
shutdownExecutorServices();
}

private CountDownLatch finishRequest(String url) {
waitingRequests.get(url).countDown();
return finishingRequests.get(url);
}

private void shutdownExecutorService() throws InterruptedException {
private void shutdownExecutorServices() throws InterruptedException {
if (!handlerService.isShutdown()) {
handlerService.shutdown();
handlerService.awaitTermination(10, TimeUnit.SECONDS);
Expand Down
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-analysis-common-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
24c8401b530308f9568eb7b408c2029c63f564c6
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-backward-codecs-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
11c46007366bb037be7d271ab0a5849b1d544662
1 change: 0 additions & 1 deletion server/licenses/lucene-core-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-core-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
55249fa9a0ed321adcf8283c6f3b649a6812b0a9
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-grouping-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2f2785e17c5c823cc8f41a7ddb4647aaca8ee773
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-highlighter-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
30928513461bf79a5cb057e84da7d34a1e53227d
1 change: 0 additions & 1 deletion server/licenses/lucene-join-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-join-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b9c8cc99632280148f92b4c0a64111c482d5d0ac
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-memory-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
49f820b1b321860fa42a4f7583e8ed8f77b9c1c2
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-misc-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
db7c30217602dfcda394a4d0f0a9e68140d385a6
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queries-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d157547bd24edc8e9d9d59c273107dc3ac5fde5e
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queryparser-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
12d844fe224f6f97c510ac20d68903ed7f626f6c
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-sandbox-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
272e588fd3d8c0a401b28a1ac715f27044bf62ec
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial-extras-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e066432e7ab02b2a4914f989bcd8c44adbf340ad
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial3d-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fa54c9b962778e28ebc0efb9f75297781350361a
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-9.8.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-suggest-9.9.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9554de5b22ae7483b344b94a9a956960b7a5d49c
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,7 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro

final NodeStats primaryNode = dataNodeStats.get(0);
final NodeStats replicaNode = dataNodeStats.get(1);

assertAcked(
prepareCreate("test").setSettings(
Settings.builder()
Expand Down Expand Up @@ -795,6 +796,17 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro

// Assert the cluster returns to green status because the replica will be promoted to primary
ensureGreen();

// After Lucene 9.9 check index will flag corruption with old (not the latest) commit points.
// For this test our previous corrupt commit "segments_2" will remain on the primary.
// This asserts this is the case, and then resets check index status.
assertEquals("Check index has a single failure", 1, checkIndexFailures.size());
assertTrue(
checkIndexFailures.get(0)
.getMessage()
.contains("could not read old (not latest) commit point segments file \"segments_2\" in directory")
);
resetCheckIndexStatus();
}

private int numShards(String... index) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,10 @@ public void testExplainDateRangeInQueryString() {

long twoMonthsAgo = now.minus(2, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000;
long rangeEnd = (now.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000) - 1;
assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]"));
assertThat(
response.getQueryExplanation().get(0).getExplanation(),
containsString("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]")
);
assertThat(response.isValid(), equalTo(true));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,10 @@ public Query rewrite(IndexSearcher searcher) throws IOException {
return rewritten;
}
IndexReader reader = searcher.getIndexReader();
IndexReaderContext context = reader.getContext();
TermStates[] ctx = new TermStates[terms.length];
int[] docFreqs = new int[ctx.length];
for (int i = 0; i < terms.length; i++) {
ctx[i] = TermStates.build(context, terms[i], true);
ctx[i] = TermStates.build(searcher, terms[i], true);
docFreqs[i] = ctx[i].docFreq();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Pruning;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
Expand Down Expand Up @@ -144,7 +145,7 @@ private static class MergeSortQueue extends PriorityQueue<ShardRef> {
reverseMul = new int[sortFields.length];
for (int compIDX = 0; compIDX < sortFields.length; compIDX++) {
final SortField sortField = sortFields[compIDX];
comparators[compIDX] = sortField.getComparator(1, false);
comparators[compIDX] = sortField.getComparator(1, Pruning.NONE);
reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Pruning;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopFieldDocs;
import org.opensearch.search.DocValueFormat;
Expand All @@ -59,7 +60,7 @@ class BottomSortValuesCollector {
this.reverseMuls = new int[sortFields.length];
this.sortFields = sortFields;
for (int i = 0; i < sortFields.length; i++) {
comparators[i] = sortFields[i].getComparator(1, false);
comparators[i] = sortFields[i].getComparator(1, Pruning.NONE);
reverseMuls[i] = sortFields[i].getReverse() ? -1 : 1;
}
}
Expand Down
Loading

0 comments on commit de26624

Please sign in to comment.