From 7f14bc9627718b26fe7ab5ae68416e9589c2fe4b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 3 Oct 2022 14:19:41 -0400 Subject: [PATCH] Update to Apache Lucene 9.4.0 (#4661) Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/lucene-expressions-9.3.0.jar.sha1 | 1 - .../licenses/lucene-expressions-9.4.0.jar.sha1 | 1 + .../lucene-analysis-icu-9.3.0.jar.sha1 | 1 - .../lucene-analysis-icu-9.4.0.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.3.0.jar.sha1 | 1 - .../lucene-analysis-kuromoji-9.4.0.jar.sha1 | 1 + .../lucene-analysis-nori-9.3.0.jar.sha1 | 1 - .../lucene-analysis-nori-9.4.0.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.3.0.jar.sha1 | 1 - .../lucene-analysis-phonetic-9.4.0.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.3.0.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.4.0.jar.sha1 | 1 + .../lucene-analysis-stempel-9.3.0.jar.sha1 | 1 - .../lucene-analysis-stempel-9.4.0.jar.sha1 | 1 + .../lucene-analysis-morfologik-9.3.0.jar.sha1 | 1 - .../lucene-analysis-morfologik-9.4.0.jar.sha1 | 1 + .../lucene-analysis-common-9.3.0.jar.sha1 | 1 - .../lucene-analysis-common-9.4.0.jar.sha1 | 1 + .../lucene-backward-codecs-9.3.0.jar.sha1 | 1 - .../lucene-backward-codecs-9.4.0.jar.sha1 | 1 + server/licenses/lucene-core-9.3.0.jar.sha1 | 1 - server/licenses/lucene-core-9.4.0.jar.sha1 | 1 + server/licenses/lucene-grouping-9.3.0.jar.sha1 | 1 - server/licenses/lucene-grouping-9.4.0.jar.sha1 | 1 + .../licenses/lucene-highlighter-9.3.0.jar.sha1 | 1 - .../licenses/lucene-highlighter-9.4.0.jar.sha1 | 1 + server/licenses/lucene-join-9.3.0.jar.sha1 | 1 - server/licenses/lucene-join-9.4.0.jar.sha1 | 1 + server/licenses/lucene-memory-9.3.0.jar.sha1 | 1 - server/licenses/lucene-memory-9.4.0.jar.sha1 | 1 + server/licenses/lucene-misc-9.3.0.jar.sha1 | 1 - server/licenses/lucene-misc-9.4.0.jar.sha1 | 1 + server/licenses/lucene-queries-9.3.0.jar.sha1 | 1 - server/licenses/lucene-queries-9.4.0.jar.sha1 | 1 + .../licenses/lucene-queryparser-9.3.0.jar.sha1 | 1 - .../licenses/lucene-queryparser-9.4.0.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.3.0.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.4.0.jar.sha1 | 1 + .../lucene-spatial-extras-9.3.0.jar.sha1 | 1 - .../lucene-spatial-extras-9.4.0.jar.sha1 | 1 + .../licenses/lucene-spatial3d-9.3.0.jar.sha1 | 1 - .../licenses/lucene-spatial3d-9.4.0.jar.sha1 | 1 + server/licenses/lucene-suggest-9.3.0.jar.sha1 | 1 - server/licenses/lucene-suggest-9.4.0.jar.sha1 | 1 + .../src/main/java/org/opensearch/Version.java | 2 +- .../opensearch/index/codec/CodecService.java | 8 ++++---- .../PerFieldMappingPostingFormatCodec.java | 4 ++-- .../index/engine/TranslogLeafReader.java | 4 ++++ .../BytesRefFieldComparatorSource.java | 18 ++++++++++-------- .../opensearch/index/get/ShardGetService.java | 2 ++ .../org/opensearch/index/codec/CodecTests.java | 12 ++++++------ .../engine/CompletionStatsCacheTests.java | 4 ++-- .../search/lookup/LeafFieldsLookupTests.java | 2 ++ 55 files changed, 57 insertions(+), 46 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.4.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.4.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd8e7e78031a..cb658910db475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253)) - [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402)) - Change the version to remove deprecated code of adding node name into log pattern of log4j property file ([#4569](https://github.com/opensearch-project/OpenSearch/pull/4569)) +- Update to Apache Lucene 9.4.0 ([#4661](https://github.com/opensearch-project/OpenSearch/pull/4661)) ### Deprecated diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 5c6c633ba193c..caa25550790fa 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 2.4.0 -lucene = 9.3.0 +lucene = 9.4.0 bundled_jdk_vendor = adoptium bundled_jdk = 17.0.4+8 diff --git a/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 deleted file mode 100644 index 2d216277b3a8e..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5583bcd3a24d3aae40b0a3152458021844ac09aa \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.4.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..2b647c1270e14 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.4.0.jar.sha1 @@ -0,0 +1 @@ +19749e264805171009836cbedecc5494b13cd920 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 deleted file mode 100644 index df4ae8d72dd2b..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11dd9be0448fe594cf918f5260e193b3ab4e07a0 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..0038e3153b139 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0.jar.sha1 @@ -0,0 +1 @@ +aa0f250558375922f3091820361156e514fe1842 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 deleted file mode 100644 index 675bf726d2a65..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87c1357612f2f483174d1a63ea8c6680a1696bac \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..ec8c78a580543 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0.jar.sha1 @@ -0,0 +1 @@ +32eb1ad367ab1289804aeed95ea7216711a7764d \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 deleted file mode 100644 index 8987f89c913df..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d032dbeb3f4015741336a877dd4b0e62099246c \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..438585ee3afa8 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0.jar.sha1 @@ -0,0 +1 @@ +63661714be65f882a921d281965b0779fd487b90 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 deleted file mode 100644 index 00d66c733c548..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fe6ac8772b545e0abd0c755cd4bd07caad58edb9 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..019a98dc594b3 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0.jar.sha1 @@ -0,0 +1 @@ +1034d876551fc21f7835b456dab01db21b9a4af6 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 deleted file mode 100644 index 0c521b5f5ef6a..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -288726e13b598c341e81aef8b5c9ce53f51889d0 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..cc8e31b7a248f --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0.jar.sha1 @@ -0,0 +1 @@ +f704ee4b14e2fe2622bb983f04b36a32df8fd4a7 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 deleted file mode 100644 index ba98dd7e06f71..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -166d02f7f98f18c6607335030a404fcad8f57cd6 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..25e115b84308d --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0.jar.sha1 @@ -0,0 +1 @@ +a95ff17b51da6b3da641fa4053e5ee9ea2ff5daf \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 deleted file mode 100644 index 88ac9a13e8ce3..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3c0e4177aa87a4be2826a360f656f3559ea3f997 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..7cc4a7131f866 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0.jar.sha1 @@ -0,0 +1 @@ +13e1ae2c760d8c0d7990ffe3296e46d9d8e6f842 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 deleted file mode 100644 index 2e260eb028f4c..0000000000000 --- a/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03496708a19a8a55a0dc4f61f8aa2febc6e8977c \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.4.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..d4db2877c486b --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.4.0.jar.sha1 @@ -0,0 +1 @@ +02fbd4e87241411fcf5d34e92a50bee46ab164dc \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 deleted file mode 100644 index 1dda17ee92fdb..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95ea01ee0d1e543e18e3cf58d8a6a27a587a7239 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.4.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..1b7b53ef9fe70 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.4.0.jar.sha1 @@ -0,0 +1 @@ +259863dfd107645de6146b3c87b4ecee66a4d43d \ No newline at end of file diff --git a/server/licenses/lucene-core-9.3.0.jar.sha1 b/server/licenses/lucene-core-9.3.0.jar.sha1 deleted file mode 100644 index fd870008c5bd4..0000000000000 --- a/server/licenses/lucene-core-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a030180999bc3f1a65f23f53b38098ca9daeee79 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.4.0.jar.sha1 b/server/licenses/lucene-core-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..66f7f24485172 --- /dev/null +++ b/server/licenses/lucene-core-9.4.0.jar.sha1 @@ -0,0 +1 @@ +cca1116f813c0f0c63acfac4c952baf29d46d76b \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.3.0.jar.sha1 b/server/licenses/lucene-grouping-9.3.0.jar.sha1 deleted file mode 100644 index 6f63ca177d3c3..0000000000000 --- a/server/licenses/lucene-grouping-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -883071196e53ec93d2a53dcc8211ee30be6c00dc \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.4.0.jar.sha1 b/server/licenses/lucene-grouping-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..fe79c0efd34e4 --- /dev/null +++ b/server/licenses/lucene-grouping-9.4.0.jar.sha1 @@ -0,0 +1 @@ +51bec1d5acc8ecaf9f50e047d3f86d60c7a958f4 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.3.0.jar.sha1 b/server/licenses/lucene-highlighter-9.3.0.jar.sha1 deleted file mode 100644 index 78264d8ee3713..0000000000000 --- a/server/licenses/lucene-highlighter-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e895c49b9991ea2ec08855c425b9eae44a08764 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.4.0.jar.sha1 b/server/licenses/lucene-highlighter-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..54700f08a3fdb --- /dev/null +++ b/server/licenses/lucene-highlighter-9.4.0.jar.sha1 @@ -0,0 +1 @@ +c8cf8c9308d8fb18a927c7ed267a14ace3990a5f \ No newline at end of file diff --git a/server/licenses/lucene-join-9.3.0.jar.sha1 b/server/licenses/lucene-join-9.3.0.jar.sha1 deleted file mode 100644 index 5e641f5f01075..0000000000000 --- a/server/licenses/lucene-join-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04baaae4ce4a35ae919150dd17cd1e63b0da9d24 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.4.0.jar.sha1 b/server/licenses/lucene-join-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..752006d3a66dd --- /dev/null +++ b/server/licenses/lucene-join-9.4.0.jar.sha1 @@ -0,0 +1 @@ +99b2d3c8e137a6853a2503456897d47d4f18974b \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.3.0.jar.sha1 b/server/licenses/lucene-memory-9.3.0.jar.sha1 deleted file mode 100644 index c8e86c7674ede..0000000000000 --- a/server/licenses/lucene-memory-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1a2203b332edc1366b9789f5286296e109dbc8c4 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.4.0.jar.sha1 b/server/licenses/lucene-memory-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..27b488699968e --- /dev/null +++ b/server/licenses/lucene-memory-9.4.0.jar.sha1 @@ -0,0 +1 @@ +881cb214e79da14de35cb0e8e6779d2722828a96 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.3.0.jar.sha1 b/server/licenses/lucene-misc-9.3.0.jar.sha1 deleted file mode 100644 index 11a459a9f52ba..0000000000000 --- a/server/licenses/lucene-misc-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61b502c9557247b6803a346c0bab20c9dc89d125 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.4.0.jar.sha1 b/server/licenses/lucene-misc-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..f9924475b9acd --- /dev/null +++ b/server/licenses/lucene-misc-9.4.0.jar.sha1 @@ -0,0 +1 @@ +a126123e482e6bf2e7aea670d221a2a39d3277dc \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.3.0.jar.sha1 b/server/licenses/lucene-queries-9.3.0.jar.sha1 deleted file mode 100644 index 2b577bd33b46a..0000000000000 --- a/server/licenses/lucene-queries-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8fe3bce3c05015c5fdb78279f36b9f1a75b98d8 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.4.0.jar.sha1 b/server/licenses/lucene-queries-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..65e441bfdaf90 --- /dev/null +++ b/server/licenses/lucene-queries-9.4.0.jar.sha1 @@ -0,0 +1 @@ +fe74dbfe9dba9ee9ee2cb80f151fde97fb4efd12 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.3.0.jar.sha1 b/server/licenses/lucene-queryparser-9.3.0.jar.sha1 deleted file mode 100644 index b106860bf9f3e..0000000000000 --- a/server/licenses/lucene-queryparser-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78f259a66d48f77a2d2b96a0a858efa08eba72dc \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.4.0.jar.sha1 b/server/licenses/lucene-queryparser-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..2d454942d52e1 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.4.0.jar.sha1 @@ -0,0 +1 @@ +13f108a8572fcf0670c7df3ba8dbe1076d0e0dbe \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.3.0.jar.sha1 b/server/licenses/lucene-sandbox-9.3.0.jar.sha1 deleted file mode 100644 index 82c2c6d85ca4c..0000000000000 --- a/server/licenses/lucene-sandbox-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5ee318cf8e9a70c2c99e03e157465316a3d4a17a \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.4.0.jar.sha1 b/server/licenses/lucene-sandbox-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..4ebcf3f6edc8f --- /dev/null +++ b/server/licenses/lucene-sandbox-9.4.0.jar.sha1 @@ -0,0 +1 @@ +e7a676a12ea50dcbf64564f4e4022f939f0a627d \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 deleted file mode 100644 index 8bbc5359487ff..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c9b226b49ae987a4226791f023562187583eb9ad \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.4.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..c0f181ad19eb6 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.4.0.jar.sha1 @@ -0,0 +1 @@ +84d956d1cb1458c51967af1c4acadd2a1f92634d \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 deleted file mode 100644 index 31132ef0ad6df..0000000000000 --- a/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -201aa61856ae44fa494504591aed54fd9b75af16 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.4.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..3414f36b02bef --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.4.0.jar.sha1 @@ -0,0 +1 @@ +76887ca708f23b13613e45fb9e307c548b22c6da \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.3.0.jar.sha1 b/server/licenses/lucene-suggest-9.3.0.jar.sha1 deleted file mode 100644 index 71a263aa163f8..0000000000000 --- a/server/licenses/lucene-suggest-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb5d7243ba67616edbda1ecf421c615dd595752d \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.4.0.jar.sha1 b/server/licenses/lucene-suggest-9.4.0.jar.sha1 new file mode 100644 index 0000000000000..563a0b5ad966b --- /dev/null +++ b/server/licenses/lucene-suggest-9.4.0.jar.sha1 @@ -0,0 +1 @@ +406c9c539f262449d3b1e57e7bc4302efeecaf6c \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index b8789b060e534..598f9b15de0fb 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -100,7 +100,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_2_2 = new Version(2020299, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_3_1 = new Version(2030199, org.apache.lucene.util.Version.LUCENE_9_3_0); - public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_3_0); + public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_4_0); public static final Version CURRENT = V_2_4_0; public static Version readVersion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index ff254a63fadb6..b1e73b3855759 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.mapper.MapperService; @@ -62,8 +62,8 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene92Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene94Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index fd0c66983208a..c101321e47350 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -36,7 +36,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.mapper.CompletionFieldMapper; @@ -53,7 +53,7 @@ * * @opensearch.internal */ -public class PerFieldMappingPostingFormatCodec extends Lucene92Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene94Codec { private final Logger logger; private final MapperService mapperService; private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java index 224c1dcab08ab..3a198743c3d8a 100644 --- a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java +++ b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java @@ -46,6 +46,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.index.VectorValues; import org.apache.lucene.search.TopDocs; @@ -84,6 +85,7 @@ public final class TranslogLeafReader extends LeafReader { 0, 0, 0, + VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, false ); @@ -101,6 +103,7 @@ public final class TranslogLeafReader extends LeafReader { 0, 0, 0, + VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, false ); @@ -118,6 +121,7 @@ public final class TranslogLeafReader extends LeafReader { 0, 0, 0, + VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, false ); diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index e816b366c3153..12c1d1ab72519 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.comparators.TermOrdValComparator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.opensearch.common.util.BigArrays; @@ -99,8 +100,13 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed; final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed); if (indexFieldData instanceof IndexOrdinalsFieldData) { - return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) { - + FieldComparator cmp = new TermOrdValComparator( + numHits, + indexFieldData.getFieldName(), + sortMissingLast, + reversed, + enableSkipping + ) { @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { final SortedSetDocValues values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); @@ -121,13 +127,9 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f return new ReplaceMissing(selectedValues, missingBytes); } } - - @Override - public void setScorer(Scorable scorer) { - BytesRefFieldComparatorSource.this.setScorer(scorer); - } - }; + cmp.disableSkipping(); + return cmp; } return new FieldComparator.TermValComparator(numHits, null, sortMissingLast) { diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java index 7b34ab5f0d5da..08e2b32bded0e 100644 --- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; @@ -326,6 +327,7 @@ private GetResult innerGetLoadFromStoredFields( 0, 0, 0, + VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, false ); diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 0275066f9af1b..0a6338333bffc 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -91,7 +91,7 @@ private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Co SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene92Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene94Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index 340811352a203..575997dc2609e 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -70,7 +70,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); final PostingsFormat postingsFormat = new Completion90PostingsFormat(); - indexWriterConfig.setCodec(new Lucene92Codec() { + indexWriterConfig.setCodec(new Lucene94Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java index 7deb6845af607..0155e288a96fd 100644 --- a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java +++ b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java @@ -36,6 +36,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; @@ -81,6 +82,7 @@ public void setUp() throws Exception { 0, 0, 0, + VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN, false );