Added support for jdk-21 (#500)

* Added support for jdk-21 Signed-off-by: Martin Gaievski <gaievski@amazon.com>
opensearch-project · Dec 6, 2023 · 5daddfd · 5daddfd
1 parent c338b94
commit 5daddfd
Show file tree

Hide file tree

Showing 61 changed files with 173 additions and 175 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -21,7 +21,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [ubuntu-latest]
 
     name: Gradle Check Linux
@@ -55,7 +55,7 @@ jobs:
   Check-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [windows-latest]
 
     name: Gradle Check Windows
@@ -83,7 +83,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [ubuntu-latest]
 
     name: Pre-commit Linux
@@ -117,7 +117,7 @@ jobs:
   Precommit-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [windows-latest]
 
     name: Pre-commit Windows

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Infrastructure
 ### Documentation
 ### Maintenance
+- Added support for jdk-21 ([#500](https://github.com/opensearch-project/neural-search/pull/500)))
 ### Refactoring
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/neural-search/compare/2.11...2.x)

diff --git a/build.gradle b/build.gradle
@@ -109,8 +109,8 @@ buildscript {
 
     dependencies {
         classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
-        classpath "com.diffplug.spotless:spotless-plugin-gradle:5.6.1"
-        classpath "io.freefair.gradle:lombok-plugin:8.0.1"
+        classpath "com.diffplug.spotless:spotless-plugin-gradle:6.23.2"
+        classpath "io.freefair.gradle:lombok-plugin:8.4"
     }
 }
 
@@ -144,7 +144,7 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
     compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
-    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
+    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
     // ml-common excluded reflection for runtime so we need to add it by ourselves.
     // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9
     // TODO: Remove following three lines of dependencies if ml-common include them in their jar
@@ -255,7 +255,7 @@ testClusters.integTest {
 
     // Increase heap size from default of 512mb to 1gb. When heap size is 512mb, our integ tests sporadically fail due
     // to ml-commons memory circuit breaker exception
-    jvmArgs("-Xms1g", "-Xmx1g")
+    jvmArgs("-Xms1g", "-Xmx4g")
 }
 
 // Remote Integration Tests

diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionSha256Sum=3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=e111cb9948407e26351227dabce49822fb88c37ee72f1d1582a69c68af2e702f
diff --git a/gradlew b/gradlew
@@ -1,19 +1,8 @@
 #!/bin/sh
 
 #
-# Copyright © 2015-2021 the original authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
 #
 
 ##############################################################################
@@ -83,7 +72,8 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -130,26 +120,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -198,11 +191,11 @@ fi
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \

diff --git a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java
@@ -14,10 +14,6 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.ml.client.MachineLearningNodeClient;
@@ -32,6 +28,10 @@
 import org.opensearch.ml.common.output.model.ModelTensors;
 import org.opensearch.neuralsearch.util.RetryUtil;
 
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This class will act as an abstraction on the MLCommons client for accessing the ML Capabilities
  */

diff --git a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java
@@ -14,8 +14,6 @@
 import java.util.Optional;
 import java.util.function.Supplier;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
@@ -61,6 +59,8 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.watcher.ResourceWatcherService;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Neural Search plugin class
  */

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java
@@ -14,16 +14,16 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
 import lombok.extern.log4j.Log4j2;
 
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TotalHits;
-
 /**
  * Class stores collection of TopDocs for each sub query from hybrid query. Collection of results is at shard level. We do store
  * list of TopDocs and list of ScoreDoc as well as total hits for the shard.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java
@@ -14,8 +14,6 @@
 import java.util.function.Supplier;
 import java.util.stream.IntStream;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.commons.lang3.StringUtils;
 import org.opensearch.env.Environment;
 import org.opensearch.index.mapper.MapperService;
@@ -26,6 +24,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * The abstract class for text processing use cases. Users provide a field name map and a model id.
  * During ingestion, the processor will use the corresponding model to inference the input texts,

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java
@@ -10,9 +10,6 @@
 
 import java.util.Map;
 
-import lombok.Getter;
-import lombok.Setter;
-
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.Nullable;
 import org.opensearch.index.query.QueryBuilder;
@@ -22,6 +19,9 @@
 import org.opensearch.search.pipeline.Processor;
 import org.opensearch.search.pipeline.SearchRequestProcessor;
 
+import lombok.Getter;
+import lombok.Setter;
+
 /**
  * Neural Search Query Request Processor, It modifies the search request with neural query clause
  * and adds model Id if not present in the search query.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java
@@ -12,9 +12,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import lombok.AllArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.action.search.QueryPhaseResultConsumer;
 import org.opensearch.action.search.SearchPhaseContext;
 import org.opensearch.action.search.SearchPhaseName;
@@ -27,6 +24,9 @@
 import org.opensearch.search.pipeline.SearchPhaseResultsProcessor;
 import org.opensearch.search.query.QuerySearchResult;
 
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Processor for score normalization and combination on post query search results. Updates query results with
  * normalized and combined scores for next phase (typically it's FETCH)

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java
@@ -14,9 +14,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import lombok.AllArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
@@ -29,6 +26,9 @@
 import org.opensearch.search.fetch.FetchSearchResult;
 import org.opensearch.search.query.QuerySearchResult;
 
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Class abstracts steps required for score normalization and combination, this includes pre-processing of incoming data
  * and post-processing of final results

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java
@@ -9,14 +9,14 @@
 import java.util.Map;
 import java.util.function.BiConsumer;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.env.Environment;
 import org.opensearch.ingest.IngestDocument;
 import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor;
 import org.opensearch.neuralsearch.util.TokenWeightUtil;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This processor is used for user input data text sparse encoding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs text embedding and the corresponding keys for the sparse encoding results.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java
@@ -9,13 +9,13 @@
 import java.util.Map;
 import java.util.function.BiConsumer;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.env.Environment;
 import org.opensearch.ingest.IngestDocument;
 import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This processor is used for user input data text embedding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs text embedding and the corresponding keys for the text embedding results.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java
@@ -15,8 +15,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.commons.lang3.StringUtils;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
@@ -30,6 +28,8 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * This processor is used for user input data text and image embedding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs embedding and the corresponding keys for the embedding results.

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java
@@ -14,12 +14,12 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.commons.lang3.Range;
 
 import com.google.common.math.DoubleMath;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Collection of utility methods for score combination technique classes
  */

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java
@@ -13,13 +13,13 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-import lombok.extern.log4j.Log4j2;
-
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHits;
 import org.opensearch.neuralsearch.processor.CompoundTopDocs;
 
+import lombok.extern.log4j.Log4j2;
+
 /**
  * Abstracts combination of scores in query search results.
  */