Merge branch 'main' into simd/rounding

.github/ISSUE_TEMPLATE/bug_template.yml

@@ -15,7 +15,7 @@ body:
       description: Choose a specific OpenSearch component your bug belongs to.  If you are unsure which to select or if the component is not present, select "Other".
       multiple: false
       options:
-        - Other
+        - # Empty first option to force selection
         - Build
         - Clients
         - Cluster Manager
@@ -24,6 +24,7 @@ body:
         - Indexing:Replication
         - Indexing
         - Libraries
+        - Other
         - Plugins
         - Search:Aggregations
         - Search:Performance

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -22,7 +22,7 @@ body:
       description: Choose a specific OpenSearch component your feature request belongs to.  If you are unsure of which component to select or if the component is not present, select "Other".
       multiple: false
       options:
-        - Other
+        - # Empty first option to force selection
         - Build
         - Clients
         - Cluster Manager
@@ -31,6 +31,7 @@ body:
         - Indexing:Replication
         - Indexing
         - Libraries
+        - Other
         - Plugins
         - Search:Aggregations
         - Search:Performance

.github/workflows/check-compatibility.yml

@@ -36,7 +36,7 @@ jobs:
           echo "### Compatible components"   >> "${{ github.workspace }}/results.txt" && grep -e 'Compatible component'   $HOME/gradlew-check.out | sed -e 's/Compatible component: \[\(.*\)\]/- \1/'   >> "${{ github.workspace }}/results.txt"
 
       - name: Upload results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: results.txt
           path: ${{ github.workspace }}/results.txt
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download results
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: results.txt
 

.github/workflows/links.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.8.0
+        uses: lycheeverse/lychee-action@v1.9.1
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true

.github/workflows/maintainer-approval.yml

@@ -2,7 +2,6 @@ name: Maintainers approval
 
 on:
   pull_request_review:
-    types: [submitted]
 
 jobs:
   maintainer-approved-check:
@@ -26,7 +25,7 @@ jobs:
 
             return maintainersResponse.data.map(item => item.login).join(', ');
 
-      - uses: peternied/required-approval@v1.2
+      - uses: peternied/required-approval@v1.3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           min-required: 1

DEVELOPER_GUIDE.md

@@ -183,6 +183,12 @@ Run OpenSearch using `gradlew run`.
 ./gradlew run
 ```
 
+[Plugins](plugins/) may be installed by passing a `-PinstalledPlugins` property:
+
+```bash
+./gradlew run -PinstalledPlugins="['plugin1', 'plugin2']"
+```
+
 That will build OpenSearch and start it, writing its log above Gradle's status message. We log a lot of stuff on startup, specifically these lines tell you that OpenSearch is ready.
 
 ```
@@ -578,7 +584,7 @@ explicitly marked by an annotation should not be extended by external implementa
 any time. The `@DeprecatedApi` annotation could also be added to any classes annotated with `@PublicApi` (or documented as `@opensearch.api`) or their methods that
 are either changed (with replacement) or planned to be removed across major versions.
 
-The APIs which are designated to be public but have not been stabilized yet should be marked with `@ExperimentalApi` (or documented as `@opensearch.experimental`) 
+The APIs which are designated to be public but have not been stabilized yet should be marked with `@ExperimentalApi` (or documented as `@opensearch.experimental`)
 annotation. The presence of this annotation signals that API may change at any time (major, minor or even patch releases). In general, the classes annotated with
 `@PublicApi` may expose other classes or methods annotated with `@ExperimentalApi`, in such cases the backward compatibility guarantees would not apply to latter
 (see please [Experimental Development](#experimental-development) for more details).

TRIAGING.md

@@ -0,0 +1,83 @@
+<img src="https://opensearch.org/assets/img/opensearch-logo-themed.svg" height="64px">
+
+The maintainers of the OpenSearch Repo seek to promote an inclusive and engaged community of contributors. In order to facilitate this, weekly triage meetings are open-to-all and attendance is encouraged for anyone who hopes to contribute, discuss an issue, or learn more about the project. To learn more about contributing to the OpenSearch Repo visit the [Contributing](./CONTRIBUTING.md) documentation.
+
+### Do I need to attend for my issue to be addressed/triaged?
+
+Attendance is not required for your issue to be triaged or addressed.  If not accepted the issue will be updated with a comment for next steps.  All new issues are triaged weekly.
+
+You can track if your issue was triaged by watching your GitHub notifications for updates.
+
+### What happens if my issue does not get covered this time?
+
+Each meeting we seek to address all new issues. However, should we run out of time before your issue is discussed, you are always welcome to attend the next meeting or to follow up on the issue post itself.
+
+### How do I join the Triage meeting?
+
+Meetings are hosted regularly at 10:00a - 10:55a Central Time every Wednesday and can be joined via [Chime](https://aws.amazon.com/chime/), with this [meeting link](https://chime.aws/1988437365).
+
+After joining the Chime meeting, you can enable your video / voice to join the discussion.  If you do not have a webcam or microphone available, you can still join in via the text chat.
+
+If you have an issue you'd like to bring forth please prepare a link to the issue so it can be presented and viewed by everyone in the meeting.
+
+### Is there an agenda for each week?
+
+Meetings are 55 minutes and follows this structure:
+
+Yes, each 55-minute meeting follows this structure:
+1. **Initial Gathering:** Feel free to turn on your video and engage in informal conversation. Shortly, a volunteer triage [facilitator](#what-is-the-role-of-the-facilitator) will begin the meeting and share their screen.
+2. **Record Attendees:** The facilitator will request attendees to share their GitHub profile links. These links will be collected and assembled into a [tag](#how-do-triage-facilitator-tag-comments-during-the-triage-meeting) to annotate comments during the meeting.
+3. **Announcements:** Any announcements will be made at the beginning of the meeting.
+4. **Review of New Issues:** We start by reviewing all untriaged [issues](https://github.com/search?q=label%3Auntriaged+is%3Aopen++repo%3Aopensearch-project%2FOpenSearch+&type=issues&ref=advsearch&s=created&o=desc) for the OpenSearch repo.
+5. **Attendee Requests:** An opportunity for any meeting member to request consideration of an issue or pull request.
+6. **Open Discussion:** Attendees can bring up any topics not already covered by filed issues or pull requests.
+
+### What is the role of the facilitator?
+
+The facilitator is crucial in driving the meeting, ensuring a smooth flow of issues into OpenSearch for future contributions. They maintain the meeting's agenda, solicit input from attendees, and record outcomes using the triage tag as items are discussed.
+
+### Do I need to have already contributed to the project to attend a triage meeting?
+
+No prior contributions are required. All interested individuals are welcome and encouraged to attend. Triage meetings offer a fantastic opportunity for new contributors to understand the project and explore various contribution avenues.
+
+### What if I have an issue that is almost a duplicate, should I open a new one to be triaged?
+
+You can always open an [issue](https://github.com/opensearch-project/OpenSearch/issues/new/choose) including one that you think may be a duplicate. If you believe your issue is similar but distinct from an existing one, you are encouraged to file it and explain the differences during the triage meeting.
+
+### What if I have follow-up questions on an issue?
+
+If you have an existing issue you would like to discuss, you can always comment on the issue itself. Alternatively, you are welcome to come to the triage meeting to discuss.
+
+### Is this meeting a good place to get help setting up features on my OpenSearch instance?
+
+While we are always happy to help the community, the best resource for implementation questions is [the OpenSearch forum](https://forum.opensearch.org/).
+
+There you can find answers to many common questions as well as speak with implementation experts.
+
+### What are the issue labels associated with triaging?
+
+Yes, there are several labels that are used to identify the 'state' of issues filed in OpenSearch .
+| Label         | When Applied         | Meaning                                                                                                                                 |
+|---------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------|
+| `Untriaged` | When issues are created or re-opened. | Issues labeled as 'Untriaged' require the attention of the repository maintainers and may need to be prioritized for quicker resolution. It's crucial to keep the count of 'Untriaged' labels low to ensure all potential security issues are addressed in a timely manner. See [SECURITY.md](https://github.com/opensearch-project/OpenSearch/blob/main/SECURITY.md) for more details on handling these issues. |
+| `Help Wanted` | Anytime. | Issues marked as 'Help Wanted' signal that they are actionable and not the current focus of the project maintainers. Community contributions are especially encouraged for these issues. |
+| `Good First Issue` | Anytime. | Issues labeled as 'Good First Issue' are small in scope and can be resolved with a single pull request. These are recommended starting points for newcomers looking to make their first contributions. |
+
+### What are the typical outcomes of a triaged issue?
+
+| Outcome      | Label            | Description                                                                                                                                                                                      | Canned Response                                                                                                                                                           |
+|--------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Accepted     | `-untriaged`     | The issue has the details needed to be directed towards area owners.                                                                                                                            | "Thanks for filing this issue, please feel free to submit a pull request."                                                                                                 |
+| Rejected     | N/A              | The issue will be closed with a reason for why it was rejected. Reasons might include lack of details, or being outside the scope of the project.                                               | "Thanks for creating this issue; however, it isn't being accepted due to {REASON}. Please feel free to re-open after addressing the reason."                                |
+| Area Triage  | `+{AREALABEL}`  | OpenSearch has many different areas. If it's unclear whether an issue should be accepted, it will be labeled with the area and an owner will be @mentioned for follow-up.                        | "Thanks for creating this issue; the triage meeting was unsure if this issue should be accepted, @{PERSON} or someone from the area please review and then accept or reject this issue?" |
+| Transfer     | N/A              | If the issue applies to another repository within the OpenSearch Project, it will be transferred accordingly.                                                                                    | "@opensearch-project/triage, can you please transfer this issue to project {REPOSITORY}." Or, if someone at the meeting has permissions, they can start the transfer.        |
+
+### Is this where I should bring up potential security vulnerabilities?
+
+Due to the sensitive nature of security vulnerabilities, please report all potential vulnerabilities directly by following the steps outlined on the [SECURITY.md](https://github.com/opensearch-project/OpenSearch/blob/main/SECURITY.md) document.
+
+### How do triage facilitator tag comments during the triage meeting?
+
+During the triage meeting, facilitators should use the tag _[Triage - attendees [1](#Profile_link) [2](#Profile_link)]_ to indicate a collective decision. This ensures contributors know the decision came from the meeting rather than an individual and identifies participants for any follow-up queries.
+
+This tag should not be used outside triage meetings.

build.gradle

@@ -54,7 +54,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.20.0" apply false
+  id "com.diffplug.spotless" version "6.23.2" apply false
   id "org.gradle.test-retry" version "1.5.4" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/build.gradle

@@ -106,19 +106,19 @@ dependencies {
   api 'org.apache.commons:commons-compress:1.25.0'
   api 'org.apache.ant:ant:1.10.14'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
-  api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
+  api 'com.netflix.nebula:nebula-publishing-plugin:21.0.0'
   api 'com.netflix.nebula:gradle-info-plugin:12.1.6'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.15.1'
-  api "net.java.dev.jna:jna:5.13.0"
+  api "net.java.dev.jna:jna:5.14.0"
   api 'com.github.johnrengelman:shadow:8.1.1'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.6'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.5'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'
-  api 'com.networknt:json-schema-validator:1.0.86'
+  api 'com.networknt:json-schema-validator:1.1.0'
   api 'org.jruby.jcodings:jcodings:1.0.58'
   api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"

buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java

@@ -94,7 +94,7 @@ public static void configureRepositories(Project project) {
             String revision = matcher.group(1);
             MavenArtifactRepository luceneRepo = repos.maven(repo -> {
                 repo.setName("lucene-snapshots");
-                repo.setUrl("https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/");
+                repo.setUrl("https://artifacts.opensearch.org/snapshots/lucene/");
             });
             repos.exclusiveContent(exclusiveRepo -> {
                 exclusiveRepo.filter(

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.22.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.22.1"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

buildSrc/version.properties

@@ -1,14 +1,14 @@
 opensearch        = 3.0.0
-lucene            = 9.8.0
+lucene            = 9.9.1
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.1+12
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.16.0
-jackson_databind  = 2.16.0
+jackson           = 2.16.1
+jackson_databind  = 2.16.1
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -26,7 +26,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.101.Final
+netty             = 4.1.104.Final
 joda              = 2.12.2
 
 # project reactor
@@ -70,5 +70,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.32.0
+opentelemetry         = 1.34.1
 opentelemetrysemconv  = 1.23.1-alpha

client/sniffer/licenses/jackson-core-2.16.1.jar.sha1

@@ -0,0 +1 @@
+9456bb3cdd0f79f91a5f730a1b1bb041a380c91f

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.5.0"
+  id "com.netflix.nebula.ospackage-base" version "11.6.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

distribution/src/config/jvm.options

@@ -38,12 +38,12 @@
 8-10:-XX:+UseCMSInitiatingOccupancyOnly
 
 ## G1GC Configuration
-# NOTE: G1 GC is only supported on JDK version 10 or later
-# to use G1GC, uncomment the next two lines and update the version on the
-# following three lines to your version of the JDK
-# 10:-XX:-UseConcMarkSweepGC
-# 10:-XX:-UseCMSInitiatingOccupancyOnly
+# NOTE: G1GC is the default GC for all JDKs 11 and newer
 11-:-XX:+UseG1GC
+# See https://github.com/elastic/elasticsearch/pull/46169 for the history
+# behind these settings, but the tl;dr is that default values can lead
+# to situations where heap usage grows enough to trigger a circuit breaker
+# before GC kicks in.
 11-:-XX:G1ReservePercent=25
 11-:-XX:InitiatingHeapOccupancyPercent=30
 

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.1.jar.sha1

@@ -0,0 +1 @@
+fd441d574a71e7d10a4f73de6609f881d8cdfeec

distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.1.jar.sha1

@@ -0,0 +1 @@
+02a16efeb840c45af1e2f31753dfe76795278b73

gradle/code-coverage.gradle

@@ -13,7 +13,7 @@ repositories {
   gradlePluginPortal()
   // TODO: Find the way to use the repositories from RepositoriesSetupPlugin
   maven {
-    url = "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/"
+    url = "https://artifacts.opensearch.org/snapshots/lucene/"
   }
 }
 

gradle/run.gradle

@@ -39,6 +39,12 @@ testClusters {
     testDistribution = 'archive'
     if (numZones > 1) numberOfZones = numZones
     if (numNodes > 1) numberOfNodes = numNodes
+    if (findProperty("installedPlugins")) {
+      installedPlugins = Eval.me(installedPlugins)
+      for (String p : installedPlugins) {
+        plugin('plugins:'.concat(p))
+      }
+    }
   }
 }
 

libs/common/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java

@@ -113,7 +113,7 @@ private void process(ExecutableElement executable, Element enclosing) {
         // The executable element should not be internal (unless constructor for injectable core component)
         checkNotInternal(enclosing, executable);
 
-        // Check this elements annotations
+        // Check this element's annotations
         for (final AnnotationMirror annotation : executable.getAnnotationMirrors()) {
             final Element element = annotation.getAnnotationType().asElement();
             if (inspectable(element)) {
@@ -210,7 +210,7 @@ private void process(ExecutableElement executable, ReferenceType ref) {
             }
         }
 
-        // Check this elements annotations
+        // Check this element's annotations
         for (final AnnotationMirror annotation : ref.getAnnotationMirrors()) {
             final Element element = annotation.getAnnotationType().asElement();
             if (inspectable(element)) {
@@ -316,7 +316,7 @@ private void checkPublic(@Nullable Element referencedBy, final Element element)
                     reportFailureAs,
                     "The element "
                         + element
-                        + " is part of the public APIs but is not maked as @PublicApi, @ExperimentalApi or @DeprecatedApi"
+                        + " is part of the public APIs but is not marked as @PublicApi, @ExperimentalApi or @DeprecatedApi"
                         + ((referencedBy != null) ? " (referenced by " + referencedBy + ") " : "")
                 );
         }