Skip to content

Commit

Permalink
Include file cache stats in node stats response (#6333)
Browse files Browse the repository at this point in the history
This change exposes the file cache stats as part the node stats API response.

Signed-off-by: Rabi Panda <adnapibar@gmail.com>
  • Loading branch information
adnapibar authored Feb 23, 2023
1 parent 8c1e19c commit 0f3b870
Show file tree
Hide file tree
Showing 15 changed files with 382 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.hamcrest.MatcherAssert;
import org.opensearch.action.admin.cluster.node.stats.NodeStats;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
Expand All @@ -29,6 +31,7 @@
import org.opensearch.index.Index;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
import org.opensearch.monitor.fs.FsInfo;
import org.opensearch.repositories.fs.FsRepository;

Expand Down Expand Up @@ -413,4 +416,49 @@ private void assertIndexDirectoryDoesNotExist(String... indexNames) {
}
}
}

public void testFileCacheStats() throws Exception {
final String snapshotName = "test-snap";
final String repoName = "test-repo";
final String indexName1 = "test-idx-1";
final Client client = client();
final int numNodes = 2;

internalCluster().ensureAtLeastNumSearchNodes(numNodes);
createIndexWithDocsAndEnsureGreen(1, 100, indexName1);

createRepositoryWithSettings(null, repoName);
takeSnapshot(client, snapshotName, repoName, indexName1);
deleteIndicesAndEnsureGreen(client, indexName1);
assertAllNodesFileCacheEmpty();

restoreSnapshotAndEnsureGreen(client, snapshotName, repoName);
assertNodesFileCacheNonEmpty(numNodes);
}

private void assertAllNodesFileCacheEmpty() {
NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().all()).actionGet();
for (NodeStats stats : response.getNodes()) {
FileCacheStats fcstats = stats.getFileCacheStats();
assertNotNull(fcstats);
assertTrue(isFileCacheEmpty(fcstats));
}
}

private void assertNodesFileCacheNonEmpty(int numNodes) {
NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().all()).actionGet();
int nonEmptyFileCacheNodes = 0;
for (NodeStats stats : response.getNodes()) {
FileCacheStats fcstats = stats.getFileCacheStats();
assertNotNull(fcstats);
if (!isFileCacheEmpty(fcstats)) {
nonEmptyFileCacheNodes++;
}
}
assertEquals(numNodes, nonEmptyFileCacheNodes);
}

private boolean isFileCacheEmpty(FileCacheStats stats) {
return stats.getUsed().getBytes() == 0L && stats.getActive().getBytes() == 0L;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.discovery.DiscoveryStats;
import org.opensearch.http.HttpStats;
import org.opensearch.index.stats.IndexingPressureStats;
import org.opensearch.index.stats.ShardIndexingPressureStats;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
import org.opensearch.indices.NodeIndicesStats;
import org.opensearch.indices.breaker.AllCircuitBreakerStats;
import org.opensearch.ingest.IngestStats;
Expand Down Expand Up @@ -130,6 +132,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
@Nullable
private WeightedRoutingStats weightedRoutingStats;

@Nullable
private FileCacheStats fileCacheStats;

public NodeStats(StreamInput in) throws IOException {
super(in);
timestamp = in.readVLong();
Expand Down Expand Up @@ -171,6 +176,11 @@ public NodeStats(StreamInput in) throws IOException {
} else {
weightedRoutingStats = null;
}
if (in.getVersion().onOrAfter(Version.V_3_0_0) && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
fileCacheStats = in.readOptionalWriteable(FileCacheStats::new);
} else {
fileCacheStats = null;
}
}

public NodeStats(
Expand All @@ -194,7 +204,8 @@ public NodeStats(
@Nullable ShardIndexingPressureStats shardIndexingPressureStats,
@Nullable SearchBackpressureStats searchBackpressureStats,
@Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats,
@Nullable WeightedRoutingStats weightedRoutingStats
@Nullable WeightedRoutingStats weightedRoutingStats,
@Nullable FileCacheStats fileCacheStats
) {
super(node);
this.timestamp = timestamp;
Expand All @@ -217,6 +228,7 @@ public NodeStats(
this.searchBackpressureStats = searchBackpressureStats;
this.clusterManagerThrottlingStats = clusterManagerThrottlingStats;
this.weightedRoutingStats = weightedRoutingStats;
this.fileCacheStats = fileCacheStats;
}

public long getTimestamp() {
Expand Down Expand Up @@ -340,6 +352,10 @@ public WeightedRoutingStats getWeightedRoutingStats() {
return weightedRoutingStats;
}

public FileCacheStats getFileCacheStats() {
return fileCacheStats;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
Expand Down Expand Up @@ -374,6 +390,9 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_2_6_0)) {
out.writeOptionalWriteable(weightedRoutingStats);
}
if (out.getVersion().onOrAfter(Version.V_3_0_0) && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
out.writeOptionalWriteable(fileCacheStats);
}
}

@Override
Expand Down Expand Up @@ -455,6 +474,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (getWeightedRoutingStats() != null) {
getWeightedRoutingStats().toXContent(builder, params);
}
if (getFileCacheStats() != null && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
getFileCacheStats().toXContent(builder, params);
}

return builder;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,8 @@ public enum Metric {
SHARD_INDEXING_PRESSURE("shard_indexing_pressure"),
SEARCH_BACKPRESSURE("search_backpressure"),
CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling"),
WEIGHTED_ROUTING_STATS("weighted_routing");
WEIGHTED_ROUTING_STATS("weighted_routing"),
FILE_CACHE_STATS("file_cache");

private String metricName;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
NodesStatsRequest.Metric.SHARD_INDEXING_PRESSURE.containedIn(metrics),
NodesStatsRequest.Metric.SEARCH_BACKPRESSURE.containedIn(metrics),
NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics),
NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics)
NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics),
NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics)
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
false,
false,
false,
false,
false
);
List<ShardStats> shardsStats = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.store.remote.filecache;

import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
import org.opensearch.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;

import java.io.IOException;

/**
* Statistics on file cache
*
* @opensearch.internal
*/
public class FileCacheStats implements Writeable, ToXContentFragment {

private final long timestamp;
private final long active;
private final long total;
private final long used;
private final long evicted;
private final long removed;
private final long replaced;
private final long hits;
private final long miss;

public FileCacheStats(
final long timestamp,
final long active,
final long total,
final long used,
final long evicted,
final long removed,
final long replaced,
final long hits,
final long miss
) {
this.timestamp = timestamp;
this.active = active;
this.total = total;
this.used = used;
this.evicted = evicted;
this.removed = removed;
this.replaced = replaced;
this.hits = hits;
this.miss = miss;
}

public FileCacheStats(final StreamInput in) throws IOException {
this.timestamp = in.readLong();
this.active = in.readLong();
this.total = in.readLong();
this.used = in.readLong();
this.evicted = in.readLong();
this.removed = in.readLong();
this.replaced = in.readLong();
this.hits = in.readLong();
this.miss = in.readLong();
}

public static short calculatePercentage(long used, long max) {
return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
}

@Override
public void writeTo(final StreamOutput out) throws IOException {
out.writeLong(timestamp);
out.writeLong(active);
out.writeLong(total);
out.writeLong(used);
out.writeLong(evicted);
out.writeLong(removed);
out.writeLong(replaced);
out.writeLong(hits);
out.writeLong(miss);
}

public long getTimestamp() {
return timestamp;
}

public ByteSizeValue getTotal() {
return new ByteSizeValue(total);
}

public ByteSizeValue getActive() {
return new ByteSizeValue(active);
}

public short getActivePercent() {
return calculatePercentage(active, used);
}

public ByteSizeValue getUsed() {
return new ByteSizeValue(used);
}

public short getUsedPercent() {
return calculatePercentage(getUsed().getBytes(), total);
}

public ByteSizeValue getEvicted() {
return new ByteSizeValue(evicted);
}

public ByteSizeValue getRemoved() {
return new ByteSizeValue(removed);
}

public long getReplacedCount() {
return replaced;
}

public long getCacheHits() {
return hits;
}

public long getCacheMiss() {
return miss;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.FILE_CACHE);
builder.field(Fields.TIMESTAMP, getTimestamp());
builder.humanReadableField(Fields.ACTIVE_IN_BYTES, Fields.ACTIVE, getActive());
builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
builder.humanReadableField(Fields.EVICTED_IN_BYTES, Fields.EVICTED, getEvicted());
builder.humanReadableField(Fields.REMOVED_IN_BYTES, Fields.REMOVED, getRemoved());
builder.field(Fields.REPLACED_COUNT, getReplacedCount());
builder.field(Fields.ACTIVE_PERCENT, getActivePercent());
builder.field(Fields.USED_PERCENT, getUsedPercent());
builder.field(Fields.CACHE_HITS, getCacheHits());
builder.field(Fields.CACHE_MISS, getCacheMiss());
builder.endObject();
return builder;
}

static final class Fields {
static final String FILE_CACHE = "file_cache";
static final String TIMESTAMP = "timestamp";
static final String ACTIVE = "active";
static final String ACTIVE_IN_BYTES = "active_in_bytes";
static final String USED = "used";
static final String USED_IN_BYTES = "used_in_bytes";
static final String EVICTED = "evicted";
static final String EVICTED_IN_BYTES = "evicted_in_bytes";
static final String REMOVED = "removed";
static final String REMOVED_IN_BYTES = "removed_in_bytes";
static final String REPLACED_COUNT = "replaced_count";
static final String TOTAL = "total";
static final String TOTAL_IN_BYTES = "total_in_bytes";

static final String ACTIVE_PERCENT = "active_percent";
static final String USED_PERCENT = "used_percent";

static final String CACHE_HITS = "cache_hits";
static final String CACHE_MISS = "cache_miss";
}
}
Loading

0 comments on commit 0f3b870

Please sign in to comment.