Skip to content

Commit

Permalink
Add telemetry for data tiers (#63031)
Browse files Browse the repository at this point in the history
This commit adds telemetry for our data tier formalization. This telemetry helps determine the
topology of the cluster with regard to the content, hot, warm, & cold tiers/roles.

An example of the telemetry looks like:

```
GET /_xpack/usage?human
{
  ...
  "data_tiers" : {
    "available" : true,
    "enabled" : true,
    "data_warm" : {
      ...
    },
    "data_cold" : {
      ...
    },
    "data_content" : {
      "node_count" : 1,
      "index_count" : 6,
      "total_shard_count" : 6,
      "primary_shard_count" : 6,
      "doc_count" : 71,
      "total_size" : "59.6kb",
      "total_size_bytes" : 61110,
      "primary_size" : "59.6kb",
      "primary_size_bytes" : 61110,
      "primary_shard_size_avg" : "9.9kb",
      "primary_shard_size_avg_bytes" : 10185,
      "primary_shard_size_median" : "8kb",
      "primary_shard_size_median_bytes" : 8254,
      "primary_shard_size_mad" : "7.2kb",
      "primary_shard_size_mad_bytes" : 7391
    },
    "data_hot" : {
       ...
    }
  }
}
```

The fields are as follows:

- node_count :: number of nodes with this tier/role
- index_count :: number of indices on this tier
- total_shard_count :: total number of shards for all nodes in this tier
- primary_shard_count :: number of primary shards for all nodes in this tier
- doc_count :: number of documents for all nodes in this tier
- total_size_bytes :: total number of bytes for all shards for all nodes in this tier
- primary_size_bytes :: number of bytes for all primary shards on all nodes in this tier
- primary_shard_size_avg_bytes :: average shard size for primary shard in this tier
- primary_shard_size_median_bytes :: median shard size for primary shard in this tier
- primary_shard_size_mad_bytes :: [median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) of shard size for primary shard in this tier

Relates to #60848
  • Loading branch information
dakrone authored Oct 1, 2020

Verified

This commit was signed with the committer’s verified signature.
targos Michaël Zasso
1 parent 8eb83d3 commit 5fca68a
Showing 15 changed files with 703 additions and 5 deletions.
4 changes: 4 additions & 0 deletions docs/reference/rest-api/info.asciidoc
Original file line number Diff line number Diff line change
@@ -146,6 +146,10 @@ Example response:
"data_streams" : {
"available" : true,
"enabled" : true,
},
"data_tiers" : {
"available" : true,
"enabled" : true,
}
},
"tagline" : "You know, for X"
52 changes: 52 additions & 0 deletions docs/reference/rest-api/usage.asciidoc
Original file line number Diff line number Diff line change
@@ -285,6 +285,58 @@ GET /_xpack/usage
"enabled" : true,
"data_streams" : 0,
"indices_count" : 0
},
"data_tiers" : {
"available" : true,
"enabled" : true,
"data_warm" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
"data_cold" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
"data_content" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
},
"data_hot" : {
"node_count" : 0,
"index_count" : 0,
"total_shard_count" : 0,
"primary_shard_count" : 0,
"doc_count" : 0,
"total_size_bytes" : 0,
"primary_size_bytes" : 0,
"primary_shard_size_avg_bytes" : 0,
"primary_shard_size_median_bytes" : 0,
"primary_shard_size_mad_bytes" : 0
}
}
}
------------------------------------------------------------
Original file line number Diff line number Diff line change
@@ -16,21 +16,24 @@
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.core.DataTier;
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
import org.elasticsearch.xpack.core.DataTiersFeatureSetUsage;
import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
import org.elasticsearch.xpack.core.action.XPackUsageResponse;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;

import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;

@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
public class DataTierIT extends ESIntegTestCase {
private static final String index = "myindex";

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Collections.singleton(LocalStateCompositeXPackPlugin.class);
return Collections.singleton(DataTierTelemetryPlugin.class);
}

public void testDefaultIndexAllocateToContent() {
@@ -194,6 +197,61 @@ public void testTemplateOverridesDefaults() {
ensureYellow(index);
}

public void testDataTierTelemetry() {
startContentOnlyNode();
startContentOnlyNode();
startHotOnlyNode();

client().admin().indices().prepareCreate(index)
.setSettings(Settings.builder()
.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, "data_hot")
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 0))
.setWaitForActiveShards(0)
.get();

client().admin().indices().prepareCreate(index + "2")
.setSettings(Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1))
.setWaitForActiveShards(0)
.get();

ensureGreen();
client().prepareIndex(index).setSource("foo", "bar").get();
client().prepareIndex(index + "2").setSource("foo", "bar").get();
client().prepareIndex(index + "2").setSource("foo", "bar").get();
refresh(index, index + "2");

DataTiersFeatureSetUsage usage = getUsage();
// We can't guarantee that internal indices aren't created, so some of these are >= checks
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).nodeCount, equalTo(2));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).indexCount, greaterThanOrEqualTo(1));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).totalShardCount, greaterThanOrEqualTo(2));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryShardCount, greaterThanOrEqualTo(1));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).docCount, greaterThanOrEqualTo(2L));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryByteCount, greaterThanOrEqualTo(1L));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryByteCountMedian, greaterThanOrEqualTo(1L));
assertThat(usage.getTierStats().get(DataTier.DATA_CONTENT).primaryShardBytesMAD, greaterThanOrEqualTo(0L));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).nodeCount, equalTo(1));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).indexCount, greaterThanOrEqualTo(1));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).totalShardCount, greaterThanOrEqualTo(2));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryShardCount, greaterThanOrEqualTo(2));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).docCount, greaterThanOrEqualTo(1L));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryByteCount, greaterThanOrEqualTo(1L));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryByteCountMedian, greaterThanOrEqualTo(1L));
assertThat(usage.getTierStats().get(DataTier.DATA_HOT).primaryShardBytesMAD, greaterThanOrEqualTo(0L));
}

private DataTiersFeatureSetUsage getUsage() {
XPackUsageResponse usages = new XPackUsageRequestBuilder(client()).execute().actionGet();
return usages.getUsages().stream()
.filter(u -> u instanceof DataTiersFeatureSetUsage)
.findFirst()
.map(u -> (DataTiersFeatureSetUsage) u)
.orElseThrow();
}

public void startDataNode() {
Settings nodeSettings = Settings.builder()
.putList("node.roles", Arrays.asList("master", "data", "ingest"))
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.cluster.routing.allocation;

import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.TransportAction;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.protocol.xpack.XPackInfoRequest;
import org.elasticsearch.protocol.xpack.XPackInfoResponse;
import org.elasticsearch.protocol.xpack.XPackUsageRequest;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
import org.elasticsearch.xpack.core.action.XPackUsageResponse;

import java.nio.file.Path;
import java.util.Collections;
import java.util.List;

/**
* This plugin extends {@link LocalStateCompositeXPackPlugin} to only make the data tier telemetry
* available. This allows telemetry to be retrieved in integration tests where it would otherwise
* throw errors trying to retrieve all of the different telemetry types.
*/
public class DataTierTelemetryPlugin extends LocalStateCompositeXPackPlugin {

public static class DataTiersTransportXPackUsageAction extends TransportXPackUsageAction {
@Inject
public DataTiersTransportXPackUsageAction(ThreadPool threadPool, TransportService transportService,
ClusterService clusterService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, NodeClient client) {
super(threadPool, transportService, clusterService, actionFilters, indexNameExpressionResolver, client);
}
@Override
protected List<XPackUsageFeatureAction> usageActions() {
return Collections.singletonList(XPackUsageFeatureAction.DATA_TIERS);
}
}

public static class DataTiersTransportXPackInfoAction extends TransportXPackInfoAction {
@Inject
public DataTiersTransportXPackInfoAction(TransportService transportService, ActionFilters actionFilters,
LicenseService licenseService, NodeClient client) {
super(transportService, actionFilters, licenseService, client);
}

@Override
protected List<XPackInfoFeatureAction> infoActions() {
return Collections.singletonList(XPackInfoFeatureAction.DATA_TIERS);
}
}

public DataTierTelemetryPlugin(final Settings settings, final Path configPath) {
super(settings, configPath);
}

@Override
protected Class<? extends TransportAction<XPackUsageRequest, XPackUsageResponse>> getUsageAction() {
return DataTiersTransportXPackUsageAction.class;
}

@Override
protected Class<? extends TransportAction<XPackInfoRequest, XPackInfoResponse>> getInfoAction() {
return DataTiersTransportXPackInfoAction.class;
}
}
Original file line number Diff line number Diff line change
@@ -16,6 +16,8 @@
import org.elasticsearch.index.shard.IndexSettingProvider;
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;

import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

/**
@@ -34,6 +36,8 @@ public class DataTier {
public static final String DATA_WARM = "data_warm";
public static final String DATA_COLD = "data_cold";

public static final Set<String> ALL_DATA_TIERS = new HashSet<>(Arrays.asList(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD));

/**
* Returns true if the given tier name is a valid tier
*/
Loading

0 comments on commit 5fca68a

Please sign in to comment.