Skip to content

Commit

Permalink
Merge branch 'main' into feature/sandbox-rejection
Browse files Browse the repository at this point in the history
Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>
  • Loading branch information
kaushalmahi12 committed Aug 28, 2024
2 parents d51b251 + b30df02 commit e29269e
Show file tree
Hide file tree
Showing 25 changed files with 529 additions and 34 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381))
- Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124))
- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326))
- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343)))
- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))

### Dependencies
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.opensearch.core.rest.RestStatus;
import org.opensearch.plugin.wlm.action.CreateQueryGroupResponse;
import org.opensearch.plugin.wlm.action.DeleteQueryGroupRequest;
import org.opensearch.search.ResourceType;
import org.opensearch.wlm.ResourceType;

import java.util.Collection;
import java.util.EnumMap;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import java.util.Set;

import static org.opensearch.cluster.metadata.QueryGroup.builder;
import static org.opensearch.search.ResourceType.fromName;
import static org.opensearch.wlm.ResourceType.fromName;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
import org.opensearch.plugin.wlm.QueryGroupTestUtils;
import org.opensearch.plugin.wlm.action.CreateQueryGroupResponse;
import org.opensearch.plugin.wlm.action.DeleteQueryGroupRequest;
import org.opensearch.search.ResourceType;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.wlm.ResourceType;

import java.util.ArrayList;
import java.util.Collection;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ teardown:
# and no dynamic fields were created.
"Mappings":
- skip:
version: " - 2.99.99"
reason: "flat_object is introduced in 3.0.0 in main branch"
version: " - 2.16.99"
reason: "flat_object field with null (doc 4) throws exception before 2.17"

- do:
indices.get_mapping:
Expand All @@ -131,8 +131,8 @@ teardown:
---
"Supported queries":
- skip:
version: " - 2.99.99"
reason: "flat_object is introduced in 3.0.0 in main branch"
version: " - 2.16.99"
reason: "flat_object field with null (doc 4) throws exception before 2.17"


# Verify Document Count
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ teardown:
# and no dynamic fields were created.
"Mappings":
- skip:
version: " - 2.99.99"
reason: "flat_object is introduced in 3.0.0 in main branch"
version: " - 2.6.99"
reason: "flat_object is introduced in 2.7"

- do:
indices.get_mapping:
Expand All @@ -106,8 +106,8 @@ teardown:
---
"Supported queries":
- skip:
version: " - 2.99.99"
reason: "flat_object is introduced in 3.0.0 in main branch"
version: " - 2.6.99"
reason: "flat_object is introduced in 2.7"


# Verify Document Count
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ teardown:
---
"Supported queries":
- skip:
version: " - 2.99.99"
reason: "flat_object is introduced in 3.0.0 in main branch"
version: " - 2.6.99"
reason: "flat_object is introduced in 2.7"

# Verify Document Count
- do:
Expand Down Expand Up @@ -607,8 +607,8 @@ teardown:
---
"Unsupported":
- skip:
version: " - 2.99.99"
reason: "flat_object is introduced in 3.0.0 in main branch"
version: " - 2.6.99"
reason: "flat_object is introduced in 2.7"

# Mapping parameters (such as index/search analyzers) are currently not supported
# The plan is to support them in the next version
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.search.ResourceType;
import org.opensearch.wlm.ResourceType;
import org.joda.time.Instant;

import java.io.IOException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.monitor.jvm.JvmStats;
import org.opensearch.monitor.process.ProcessProbe;
import org.opensearch.search.ResourceType;
import org.opensearch.search.backpressure.settings.SearchBackpressureMode;
import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
import org.opensearch.search.backpressure.settings.SearchShardTaskSettings;
Expand All @@ -43,6 +42,7 @@
import org.opensearch.tasks.TaskResourceTrackingService.TaskCompletionListener;
import org.opensearch.threadpool.Scheduler;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.wlm.ResourceType;

import java.io.IOException;
import java.util.ArrayList;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
package org.opensearch.search.backpressure.trackers;

import org.opensearch.common.util.Streak;
import org.opensearch.search.ResourceType;
import org.opensearch.wlm.ResourceType;

import java.util.Map;
import java.util.function.BooleanSupplier;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

package org.opensearch.wlm;

import org.opensearch.search.ResourceType;
import org.opensearch.tasks.Task;

import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* compatible open source license.
*/

package org.opensearch.search;
package org.opensearch.wlm;

import org.opensearch.common.annotation.PublicApi;
import org.opensearch.core.common.io.stream.StreamOutput;
Expand All @@ -21,15 +21,17 @@
*/
@PublicApi(since = "2.x")
public enum ResourceType {
CPU("cpu", task -> task.getTotalResourceUtilization(ResourceStats.CPU)),
MEMORY("memory", task -> task.getTotalResourceUtilization(ResourceStats.MEMORY));
CPU("cpu", task -> task.getTotalResourceUtilization(ResourceStats.CPU), true),
MEMORY("memory", task -> task.getTotalResourceUtilization(ResourceStats.MEMORY), true);

private final String name;
private final Function<Task, Long> getResourceUsage;
private final boolean statsEnabled;

ResourceType(String name, Function<Task, Long> getResourceUsage) {
ResourceType(String name, Function<Task, Long> getResourceUsage, boolean statsEnabled) {
this.name = name;
this.getResourceUsage = getResourceUsage;
this.statsEnabled = statsEnabled;
}

/**
Expand Down Expand Up @@ -63,4 +65,8 @@ public String getName() {
public long getResourceUsage(Task task) {
return getResourceUsage.apply(task);
}

public boolean hasStatsEnabled() {
return statsEnabled;
}
}
103 changes: 103 additions & 0 deletions server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.wlm.stats;

import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.wlm.ResourceType;

import java.util.EnumMap;
import java.util.Map;

/**
* This class will keep the point in time view of the query group stats
*/
public class QueryGroupState {
/**
* completions at the query group level, this is a cumulative counter since the Opensearch start time
*/
final CounterMetric completions = new CounterMetric();

/**
* rejections at the query group level, this is a cumulative counter since the OpenSearch start time
*/
final CounterMetric totalRejections = new CounterMetric();

/**
* this will track the cumulative failures in a query group
*/
final CounterMetric failures = new CounterMetric();

/**
* This will track total number of cancellations in the query group due to all resource type breaches
*/
final CounterMetric totalCancellations = new CounterMetric();

/**
* This is used to store the resource type state both for CPU and MEMORY
*/
private final Map<ResourceType, ResourceTypeState> resourceState;

public QueryGroupState() {
resourceState = new EnumMap<>(ResourceType.class);
for (ResourceType resourceType : ResourceType.values()) {
if (resourceType.hasStatsEnabled()) {
resourceState.put(resourceType, new ResourceTypeState(resourceType));
}
}
}

/**
*
* @return completions in the query group
*/
public long getCompletions() {
return completions.count();
}

/**
*
* @return rejections in the query group
*/
public long getTotalRejections() {
return totalRejections.count();
}

/**
*
* @return failures in the query group
*/
public long getFailures() {
return failures.count();
}

public long getTotalCancellations() {
return totalCancellations.count();
}

/**
* getter for query group resource state
* @return the query group resource state
*/
public Map<ResourceType, ResourceTypeState> getResourceState() {
return resourceState;
}

/**
* This class holds the resource level stats for the query group
*/
public static class ResourceTypeState {
final ResourceType resourceType;
final CounterMetric cancellations = new CounterMetric();
final CounterMetric rejections = new CounterMetric();

public ResourceTypeState(ResourceType resourceType) {
this.resourceType = resourceType;
}
}
}
Loading

0 comments on commit e29269e

Please sign in to comment.