Skip to content

Commit

Permalink
Merge branch 'main' into main-simplify-rrt
Browse files Browse the repository at this point in the history
Signed-off-by: Shailendra Singh <singhlhs@amazon.com>
  • Loading branch information
Shailendra Singh committed Sep 2, 2024
2 parents cfde447 + ef47252 commit f9567e1
Show file tree
Hide file tree
Showing 375 changed files with 16,651 additions and 2,342 deletions.
54 changes: 54 additions & 0 deletions .github/benchmark-configs.json
Original file line number Diff line number Diff line change
Expand Up @@ -167,5 +167,59 @@
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
},
"baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
},
"id_11": {
"description": "Search only test-procedure for big5 with concurrent segment search setting enabled",
"supported_major_versions": ["3"],
"cluster-benchmark-configs": {
"SINGLE_NODE_CLUSTER": "true",
"MIN_DISTRIBUTION": "true",
"TEST_WORKLOAD": "big5",
"ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true",
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
"CAPTURE_NODE_STAT": "true",
"TEST_PROCEDURE": "restore-from-snapshot"
},
"cluster_configuration": {
"size": "Single-Node",
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
},
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
},
"id_12": {
"description": "Search only test-procedure for big5 with concurrent segment search mode as all",
"supported_major_versions": ["3"],
"cluster-benchmark-configs": {
"SINGLE_NODE_CLUSTER": "true",
"MIN_DISTRIBUTION": "true",
"TEST_WORKLOAD": "big5",
"ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all",
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
"CAPTURE_NODE_STAT": "true",
"TEST_PROCEDURE": "restore-from-snapshot"
},
"cluster_configuration": {
"size": "Single-Node",
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
},
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
},
"id_13": {
"description": "Search only test-procedure for big5 with concurrent segment search mode as auto",
"supported_major_versions": ["3"],
"cluster-benchmark-configs": {
"SINGLE_NODE_CLUSTER": "true",
"MIN_DISTRIBUTION": "true",
"TEST_WORKLOAD": "big5",
"ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto",
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
"CAPTURE_NODE_STAT": "true",
"TEST_PROCEDURE": "restore-from-snapshot"
},
"cluster_configuration": {
"size": "Single-Node",
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
},
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
}
}
2 changes: 1 addition & 1 deletion .github/workflows/gradle-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
- uses: actions/checkout@v4
- name: Get changed files
id: changed-files-specific
uses: tj-actions/changed-files@v44
uses: tj-actions/changed-files@v45
with:
files_ignore: |
release-notes/*.md
Expand Down
26 changes: 25 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
- Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774))
- Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153))
- Make balanced shards allocator timebound ([#15239](https://github.com/opensearch-project/OpenSearch/pull/15239))
- Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325))
- Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895))
- Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336))
- [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381))
- Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124))
- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326))
- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343)))
- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630))
- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454))
- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))
- Add fieldType to AbstractQueryBuilder and FieldSortBuilder ([#15328](https://github.com/opensearch-project/OpenSearch/pull/15328)))
- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410))
- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426))

### Dependencies
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.16.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205))
- OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998))
- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.16.2 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995))
- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420))
- Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997))
- Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110))
- Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104))
Expand All @@ -48,15 +62,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298))
- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361))
- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361))
- Bump `tj-actions/changed-files` from 44 to 45 ([#15422](https://github.com/opensearch-project/OpenSearch/pull/15422))
- Bump `dnsjava:dnsjava` from 3.6.0 to 3.6.1 ([#15418](https://github.com/opensearch-project/OpenSearch/pull/15418))
- Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419))
- Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423))
- Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469))

### Changed
- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
- Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371))
- Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238))
- Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216))


### Deprecated

### Removed
- Remove some unused code in the search backpressure package ([#15518](https://github.com/opensearch-project/OpenSearch/pull/15518))

### Fixed
- Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908))
Expand All @@ -67,8 +89,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620))
- Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194))
- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233))
- Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375))
- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393))
- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394))
- Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069))

### Security

Expand Down
4 changes: 2 additions & 2 deletions buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jts = 1.15.0
jackson = 2.17.2
jackson_databind = 2.17.2
snakeyaml = 2.1
icu4j = 70.1
icu4j = 75.1
supercsv = 2.4.0
log4j = 2.21.0
slf4j = 1.7.36
Expand All @@ -37,7 +37,7 @@ reactor_netty = 1.1.22
reactor = 3.5.20

# client dependencies
httpclient5 = 5.2.3
httpclient5 = 5.3.1
httpcore5 = 5.2.5
httpclient = 4.5.14
httpcore = 4.4.16
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import org.apache.lucene.util.CollectionUtil;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.XContentParser;
Expand Down Expand Up @@ -61,6 +62,7 @@ public class GetIndexResponse {
private Map<String, Settings> settings;
private Map<String, Settings> defaultSettings;
private Map<String, String> dataStreams;
private Map<String, Context> contexts;
private String[] indices;

GetIndexResponse(
Expand All @@ -69,7 +71,8 @@ public class GetIndexResponse {
Map<String, List<AliasMetadata>> aliases,
Map<String, Settings> settings,
Map<String, Settings> defaultSettings,
Map<String, String> dataStreams
Map<String, String> dataStreams,
Map<String, Context> contexts
) {
this.indices = indices;
// to have deterministic order
Expand All @@ -89,6 +92,9 @@ public class GetIndexResponse {
if (dataStreams != null) {
this.dataStreams = dataStreams;
}
if (contexts != null) {
this.contexts = contexts;
}
}

public String[] getIndices() {
Expand Down Expand Up @@ -123,6 +129,10 @@ public Map<String, String> getDataStreams() {
return dataStreams;
}

public Map<String, Context> contexts() {
return contexts;
}

/**
* Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to
* false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set
Expand Down Expand Up @@ -167,6 +177,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
Settings indexSettings = null;
Settings indexDefaultSettings = null;
String dataStream = null;
Context context = null;
// We start at START_OBJECT since fromXContent ensures that
while (parser.nextToken() != Token.END_OBJECT) {
ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser);
Expand All @@ -185,6 +196,9 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
case "defaults":
indexDefaultSettings = Settings.fromXContent(parser);
break;
case "context":
context = Context.fromXContent(parser);
break;
default:
parser.skipChildren();
}
Expand All @@ -197,7 +211,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
parser.skipChildren();
}
}
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream);
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream, context);
}

// This is just an internal container to make stuff easier for returning
Expand All @@ -207,19 +221,22 @@ private static class IndexEntry {
Settings indexSettings = Settings.EMPTY;
Settings indexDefaultSettings = Settings.EMPTY;
String dataStream;
Context context;

IndexEntry(
List<AliasMetadata> indexAliases,
MappingMetadata indexMappings,
Settings indexSettings,
Settings indexDefaultSettings,
String dataStream
String dataStream,
Context context
) {
if (indexAliases != null) this.indexAliases = indexAliases;
if (indexMappings != null) this.indexMappings = indexMappings;
if (indexSettings != null) this.indexSettings = indexSettings;
if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings;
if (dataStream != null) this.dataStream = dataStream;
if (context != null) this.context = context;
}
}

Expand All @@ -229,6 +246,7 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
Map<String, Settings> settings = new HashMap<>();
Map<String, Settings> defaultSettings = new HashMap<>();
Map<String, String> dataStreams = new HashMap<>();
Map<String, Context> contexts = new HashMap<>();
List<String> indices = new ArrayList<>();

if (parser.currentToken() == null) {
Expand All @@ -254,12 +272,15 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
if (indexEntry.dataStream != null) {
dataStreams.put(indexName, indexEntry.dataStream);
}
if (indexEntry.context != null) {
contexts.put(indexName, indexEntry.context);
}
} else if (parser.currentToken() == Token.START_ARRAY) {
parser.skipChildren();
} else {
parser.nextToken();
}
}
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams);
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams, contexts);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.opensearch.client.AbstractResponseTestCase;
import org.opensearch.client.GetAliasesResponseTests;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.settings.IndexScopedSettings;
import org.opensearch.common.settings.Settings;
Expand Down Expand Up @@ -66,6 +67,7 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
final Map<String, Settings> settings = new HashMap<>();
final Map<String, Settings> defaultSettings = new HashMap<>();
final Map<String, String> dataStreams = new HashMap<>();
final Map<String, Context> contexts = new HashMap<>();
IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
boolean includeDefaults = randomBoolean();
for (String index : indices) {
Expand All @@ -90,14 +92,19 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
if (randomBoolean()) {
dataStreams.put(index, randomAlphaOfLength(5).toLowerCase(Locale.ROOT));
}

if (randomBoolean()) {
contexts.put(index, new Context(randomAlphaOfLength(5).toLowerCase(Locale.ROOT)));
}
}
return new org.opensearch.action.admin.indices.get.GetIndexResponse(
indices,
mappings,
aliases,
settings,
defaultSettings,
dataStreams
dataStreams,
null
);
}

Expand All @@ -116,6 +123,7 @@ protected void assertInstances(
assertEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
assertEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
assertEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
assertEquals(serverTestInstance.contexts(), clientInstance.contexts());
}

private static MappingMetadata createMappingsForIndex() {
Expand Down
1 change: 0 additions & 1 deletion client/rest/licenses/httpclient5-5.2.3.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions client/rest/licenses/httpclient5-5.3.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
56b53c8f4bcdaada801d311cf2ff8a24d6d96883
Loading

0 comments on commit f9567e1

Please sign in to comment.