Merge branch 'opensearch-project:main' into main

.github/workflows/assemble.yml

@@ -19,6 +19,8 @@ jobs:
       - name: Setup docker (missing on MacOS)
         if: runner.os == 'macos'
         uses: douglascamata/setup-docker-macos-action@main
+        with:
+          upgrade-qemu: true
       - name: Run Gradle (assemble)
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE

.github/workflows/gradle-check.yml

@@ -12,13 +12,28 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
+  check-files:
+    runs-on: ubuntu-latest
+    outputs: 
+      RUN_GRADLE_CHECK: ${{ steps.changed-files-specific.outputs.any_changed }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get changed files
+      id: changed-files-specific
+      uses: tj-actions/changed-files@v44
+      with:
+        files_ignore: |
+          release-notes/*.md
+          .github/**
+          *.md
+    
   gradle-check:
-    if: github.repository == 'opensearch-project/OpenSearch'
+    needs: check-files
+    if: github.repository == 'opensearch-project/OpenSearch' && needs.check-files.outputs.RUN_GRADLE_CHECK == 'true'
     permissions:
       contents: read # to fetch code (actions/checkout)
       pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
       issues: write # To create an issue if check fails on push.
-
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:
@@ -151,3 +166,12 @@ jobs:
         with:
           assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
           filename: .github/ISSUE_TEMPLATE/failed_check.md
+
+  check-result:
+    needs: [check-files, gradle-check]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if gradle-check fails
+        if: ${{ needs.check-files.outputs.RUN_GRADLE_CHECK && needs.gradle-check.result == 'failure' }}
+        run: exit 1

CHANGELOG.md

@@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
 - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679))
 - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637))
+- Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819))
+- [Remote State] Add async remote state deletion task running on an interval, configurable by a setting ([#13131](https://github.com/opensearch-project/OpenSearch/pull/13131))
+- Allow setting query parameters on requests ([#13776](https://github.com/opensearch-project/OpenSearch/issues/13776))
 
 ### Dependencies
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
@@ -23,6 +26,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#13752](https://github.com/opensearch-project/OpenSearch/pull/13752))
 - Bump `ch.qos.logback:logback-core` from 1.5.3 to 1.5.6 ([#13756](https://github.com/opensearch-project/OpenSearch/pull/13756))
 - Bump `netty` from 4.1.109.Final to 4.1.110.Final ([#13802](https://github.com/opensearch-project/OpenSearch/pull/13802))
+- Bump `jackson` from 2.17.0 to 2.17.1 ([#13817](https://github.com/opensearch-project/OpenSearch/pull/13817))
+- Bump `reactor` from 3.5.15 to 3.5.17 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `reactor-netty` from 1.1.17 to 1.1.19 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `commons-cli:commons-cli` from 1.7.0 to 1.8.0 ([#13840](https://github.com/opensearch-project/OpenSearch/pull/13840))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.0 to 5.2.1 ([#13839](https://github.com/opensearch-project/OpenSearch/pull/13839))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
@@ -35,10 +43,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove handling of index.mapper.dynamic in AutoCreateIndex([#13067](https://github.com/opensearch-project/OpenSearch/pull/13067))
 
 ### Fixed
-- Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
 - Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
 - Allow clearing `remote_store.compatibility_mode` setting ([#13646](https://github.com/opensearch-project/OpenSearch/pull/13646))
 - Fix ReplicaShardBatchAllocator to batch shards without duplicates ([#13710](https://github.com/opensearch-project/OpenSearch/pull/13710))
+- Don't return negative scores from `multi_match` query with `cross_fields` type  ([#13829](https://github.com/opensearch-project/OpenSearch/pull/13829))
+- Pass parent filter to inner hit query ([#13903](https://github.com/opensearch-project/OpenSearch/pull/13903))
+- Fix NPE on restore searchable snapshot ([#13911](https://github.com/opensearch-project/OpenSearch/pull/13911))
 
 ### Security
 

DEVELOPER_GUIDE.md

@@ -62,6 +62,7 @@
     - [LineLint](#linelint)
     - [Lucene Snapshots](#lucene-snapshots)
     - [Flaky Tests](#flaky-tests)
+    - [Gradle Check Metrics Dashboard](#gradle-check-metrics-dashboard)
 
 # Developer Guide
 
@@ -660,3 +661,7 @@ If you encounter a build/test failure in CI that is unrelated to the change in y
 4. If an existing issue is found, paste a link to the known issue in a comment to your PR.
 5. If no existing issue is found, open one.
 6. Retry CI via the GitHub UX or by pushing an update to your PR.
+
+### Gradle Check Metrics Dashboard
+
+To get the comprehensive insights and analysis of the Gradle Check test failures, visit the [OpenSearch Gradle Check Metrics Dashboard](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083). This dashboard is part of the [OpenSearch Metrics Project](https://github.com/opensearch-project/opensearch-metrics) initiative. The dashboard contains multiple data points that can help investigate and resolve flaky failures. Additionally, this dashboard can be used to drill down, slice, and dice the data using multiple supported filters, which further aids in troubleshooting and resolving issues.

buildSrc/version.properties

@@ -1,14 +1,14 @@
 opensearch        = 3.0.0
-lucene            = 9.11.0-snapshot-fb97840
+lucene            = 9.11.0-snapshot-4be6531
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.17.0
-jackson_databind  = 2.17.0
+jackson           = 2.17.1
+jackson_databind  = 2.17.1
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -31,8 +31,8 @@ netty             = 4.1.110.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.17
-reactor           = 3.5.15
+reactor_netty     = 1.1.19
+reactor           = 3.5.17
 
 # client dependencies
 httpclient5       = 5.2.1

client/rest/src/main/java/org/opensearch/client/Request.java

@@ -110,7 +110,13 @@ public void addParameters(Map<String, String> paramSource) {
      * will change it.
      */
     public Map<String, String> getParameters() {
-        return unmodifiableMap(parameters);
+        if (options.getParameters().isEmpty()) {
+            return unmodifiableMap(parameters);
+        } else {
+            Map<String, String> combinedParameters = new HashMap<>(parameters);
+            combinedParameters.putAll(options.getParameters());
+            return unmodifiableMap(combinedParameters);
+        }
     }
 
     /**

client/rest/src/main/java/org/opensearch/client/RequestOptions.java

@@ -40,8 +40,11 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * The portion of an HTTP request to OpenSearch that can be
@@ -53,18 +56,21 @@ public final class RequestOptions {
      */
     public static final RequestOptions DEFAULT = new Builder(
         Collections.emptyList(),
+        Collections.emptyMap(),
         HeapBufferedResponseConsumerFactory.DEFAULT,
         null,
         null
     ).build();
 
     private final List<Header> headers;
+    private final Map<String, String> parameters;
     private final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory;
     private final WarningsHandler warningsHandler;
     private final RequestConfig requestConfig;
 
     private RequestOptions(Builder builder) {
         this.headers = Collections.unmodifiableList(new ArrayList<>(builder.headers));
+        this.parameters = Collections.unmodifiableMap(new HashMap<>(builder.parameters));
         this.httpAsyncResponseConsumerFactory = builder.httpAsyncResponseConsumerFactory;
         this.warningsHandler = builder.warningsHandler;
         this.requestConfig = builder.requestConfig;
@@ -74,7 +80,7 @@ private RequestOptions(Builder builder) {
      * Create a builder that contains these options but can be modified.
      */
     public Builder toBuilder() {
-        return new Builder(headers, httpAsyncResponseConsumerFactory, warningsHandler, requestConfig);
+        return new Builder(headers, parameters, httpAsyncResponseConsumerFactory, warningsHandler, requestConfig);
     }
 
     /**
@@ -84,6 +90,14 @@ public List<Header> getHeaders() {
         return headers;
     }
 
+    /**
+     * Query parameters to attach to the request. Any parameters present here
+     * will override matching parameters in the {@link Request}, if they exist.
+     */
+    public Map<String, String> getParameters() {
+        return parameters;
+    }
+
     /**
      * The {@link HttpAsyncResponseConsumerFactory} used to create one
      * {@link AsyncResponseConsumer} callback per retry. Controls how the
@@ -142,6 +156,12 @@ public String toString() {
                 b.append(headers.get(h).toString());
             }
         }
+        if (parameters.size() > 0) {
+            if (comma) b.append(", ");
+            comma = true;
+            b.append("parameters=");
+            b.append(parameters.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")));
+        }
         if (httpAsyncResponseConsumerFactory != HttpAsyncResponseConsumerFactory.DEFAULT) {
             if (comma) b.append(", ");
             comma = true;
@@ -170,6 +190,7 @@ public boolean equals(Object obj) {
 
         RequestOptions other = (RequestOptions) obj;
         return headers.equals(other.headers)
+            && parameters.equals(other.parameters)
             && httpAsyncResponseConsumerFactory.equals(other.httpAsyncResponseConsumerFactory)
             && Objects.equals(warningsHandler, other.warningsHandler);
     }
@@ -179,7 +200,7 @@ public boolean equals(Object obj) {
      */
     @Override
     public int hashCode() {
-        return Objects.hash(headers, httpAsyncResponseConsumerFactory, warningsHandler);
+        return Objects.hash(headers, parameters, httpAsyncResponseConsumerFactory, warningsHandler);
     }
 
     /**
@@ -189,17 +210,20 @@ public int hashCode() {
      */
     public static class Builder {
         private final List<Header> headers;
+        private final Map<String, String> parameters;
         private HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory;
         private WarningsHandler warningsHandler;
         private RequestConfig requestConfig;
 
         private Builder(
             List<Header> headers,
+            Map<String, String> parameters,
             HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
             WarningsHandler warningsHandler,
             RequestConfig requestConfig
         ) {
             this.headers = new ArrayList<>(headers);
+            this.parameters = new HashMap<>(parameters);
             this.httpAsyncResponseConsumerFactory = httpAsyncResponseConsumerFactory;
             this.warningsHandler = warningsHandler;
             this.requestConfig = requestConfig;
@@ -226,6 +250,21 @@ public Builder addHeader(String name, String value) {
             return this;
         }
 
+        /**
+         * Add the provided query parameter to the request. Any parameters added here
+         * will override matching parameters in the {@link Request}, if they exist.
+         *
+         * @param name  the query parameter name
+         * @param value the query parameter value
+         * @throws NullPointerException if {@code name} or {@code value} is null.
+         */
+        public Builder addParameter(String name, String value) {
+            Objects.requireNonNull(name, "query parameter name cannot be null");
+            Objects.requireNonNull(value, "query parameter value cannot be null");
+            this.parameters.put(name, value);
+            return this;
+        }
+
         /**
          * Set the {@link HttpAsyncResponseConsumerFactory} used to create one
          * {@link AsyncResponseConsumer} callback per retry. Controls how the

client/rest/src/test/java/org/opensearch/client/RequestOptionsTests.java

@@ -39,12 +39,15 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 
@@ -90,6 +93,39 @@ public void testAddHeader() {
         }
     }
 
+    public void testAddParameter() {
+        assertThrows(
+            "query parameter name cannot be null",
+            NullPointerException.class,
+            () -> randomBuilder().addParameter(null, randomAsciiLettersOfLengthBetween(3, 10))
+        );
+
+        assertThrows(
+            "query parameter value cannot be null",
+            NullPointerException.class,
+            () -> randomBuilder().addParameter(randomAsciiLettersOfLengthBetween(3, 10), null)
+        );
+
+        RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+        int numParameters = between(0, 5);
+        Map<String, String> parameters = new HashMap<>();
+        for (int i = 0; i < numParameters; i++) {
+            String name = randomAsciiAlphanumOfLengthBetween(5, 10);
+            String value = randomAsciiAlphanumOfLength(3);
+            parameters.put(name, value);
+            builder.addParameter(name, value);
+        }
+        RequestOptions options = builder.build();
+        assertEquals(parameters, options.getParameters());
+
+        try {
+            options.getParameters().put(randomAsciiAlphanumOfLengthBetween(5, 10), randomAsciiAlphanumOfLength(3));
+            fail("expected failure");
+        } catch (UnsupportedOperationException e) {
+            assertNull(e.getMessage());
+        }
+    }
+
     public void testSetHttpAsyncResponseConsumerFactory() {
         try {
             RequestOptions.DEFAULT.toBuilder().setHttpAsyncResponseConsumerFactory(null);
@@ -145,6 +181,13 @@ static RequestOptions.Builder randomBuilder() {
             }
         }
 
+        if (randomBoolean()) {
+            int queryParamCount = between(1, 5);
+            for (int i = 0; i < queryParamCount; i++) {
+                builder.addParameter(randomAsciiAlphanumOfLength(3), randomAsciiAlphanumOfLength(3));
+            }
+        }
+
         if (randomBoolean()) {
             builder.setHttpAsyncResponseConsumerFactory(new HeapBufferedResponseConsumerFactory(1));
         }

client/sniffer/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1

@@ -0,0 +1 @@
+fca7ef6192c9ad05d07bc50da991bf937a84af3a

distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1

@@ -0,0 +1 @@
+0524dcbcccdde7d45a679dfc333e4763feb09079

libs/core/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

libs/core/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+3c2361bd633374ae3814b175cc25ccf773f67026

libs/x-content/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1

@@ -0,0 +1 @@
+ba5d8e6ecc62aa0e49c0ce935b8696352dbebc71

libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1

@@ -0,0 +1 @@
+89683ac4f0a0c2c4f69ea56b90480ed40266dac8