Include min/max/average file size in Searchable Snapshots Stats API

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -126,6 +127,8 @@ public static class CacheIndexInputStats implements Writeable, ToXContentObject
         private final String fileExt;
         private final long numFiles;
         private final long totalSize;
+        private final ByteSizeValue minSize;
+        private final ByteSizeValue maxSize;
 
         private final long openCount;
         private final long closeCount;
@@ -144,7 +147,8 @@ public static class CacheIndexInputStats implements Writeable, ToXContentObject
         private final Counter blobStoreBytesRequested;
         private final long currentIndexCacheFills;
 
-        public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, long openCount, long closeCount,
+        public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, ByteSizeValue minSize, ByteSizeValue maxSize,
+                                    long openCount, long closeCount,
                                     Counter forwardSmallSeeks, Counter backwardSmallSeeks,
                                     Counter forwardLargeSeeks, Counter backwardLargeSeeks,
                                     Counter contiguousReads, Counter nonContiguousReads,
@@ -154,6 +158,8 @@ public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, long
             this.fileExt = fileExt;
             this.numFiles = numFiles;
             this.totalSize = totalSize;
+            this.minSize = minSize;
+            this.maxSize = maxSize;
             this.openCount = openCount;
             this.closeCount = closeCount;
             this.forwardSmallSeeks = forwardSmallSeeks;
@@ -181,6 +187,13 @@ public CacheIndexInputStats(String fileExt, long numFiles, long totalSize, long
             this.fileExt = in.readString();
             this.numFiles = in.readVLong();
             this.totalSize = in.readVLong();
+            if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO adapt BWC version after backport
+                this.minSize = new ByteSizeValue(in);
+                this.maxSize = new ByteSizeValue(in);
+            } else {
+                this.minSize = ByteSizeValue.ZERO;
+                this.maxSize = ByteSizeValue.ZERO;
+            }
             this.openCount = in.readVLong();
             this.closeCount = in.readVLong();
             this.forwardSmallSeeks = new Counter(in);
@@ -208,6 +221,8 @@ public static CacheIndexInputStats combine(CacheIndexInputStats cis1, CacheIndex
                 cis1.fileExt,
                 cis1.numFiles + cis2.numFiles,
                 cis1.totalSize + cis2.totalSize,
+                new ByteSizeValue(Math.min(cis1.minSize.getBytes(), cis2.minSize.getBytes())),
+                new ByteSizeValue(Math.max(cis1.maxSize.getBytes(), cis2.maxSize.getBytes())),
                 cis1.openCount + cis2.openCount,
                 cis1.closeCount + cis2.closeCount,
                 cis1.forwardSmallSeeks.add(cis2.forwardSmallSeeks),
@@ -237,6 +252,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(fileExt);
             out.writeVLong(numFiles);
             out.writeVLong(totalSize);
+            if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO adapt BWC version after backport
+                minSize.writeTo(out);
+                maxSize.writeTo(out);
+            }
             out.writeVLong(openCount);
             out.writeVLong(closeCount);
 
@@ -267,6 +286,19 @@ public long getTotalSize() {
             return totalSize;
         }
 
+        public ByteSizeValue getMinSize() {
+            return minSize;
+        }
+
+        public ByteSizeValue getMaxSize() {
+            return maxSize;
+        }
+
+        public ByteSizeValue getAverageSize() {
+            final double average = (double) totalSize / (double) numFiles;
+            return new ByteSizeValue(Math.round(average));
+        }
+
         public long getOpenCount() {
             return openCount;
         }
@@ -334,6 +366,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field("file_ext", getFileExt());
                 builder.field("num_files", getNumFiles());
                 builder.field("total_size", getTotalSize());
+                builder.humanReadableField("min_size_in_bytes", "min_size", getMinSize());
+                builder.humanReadableField("max_size_in_bytes", "max_size", getMaxSize());
+                builder.humanReadableField("avg_size_in_bytes", "avg_size", getAverageSize());
                 builder.field("open_count", getOpenCount());
                 builder.field("close_count", getCloseCount());
                 builder.field("contiguous_bytes_read", getContiguousReads(), params);
@@ -375,6 +410,8 @@ public boolean equals(Object other) {
                 && openCount == stats.openCount
                 && closeCount == stats.closeCount
                 && Objects.equals(fileExt, stats.fileExt)
+                && Objects.equals(minSize, stats.minSize)
+                && Objects.equals(maxSize, stats.maxSize)
                 && Objects.equals(forwardSmallSeeks, stats.forwardSmallSeeks)
                 && Objects.equals(backwardSmallSeeks, stats.backwardSmallSeeks)
                 && Objects.equals(forwardLargeSeeks, stats.forwardLargeSeeks)
@@ -392,7 +429,9 @@ public boolean equals(Object other) {
 
         @Override
         public int hashCode() {
-            return Objects.hash(fileExt, numFiles, totalSize, openCount, closeCount,
+            return Objects.hash(fileExt, numFiles, totalSize,
+                minSize, maxSize,
+                openCount, closeCount,
                 forwardSmallSeeks, backwardSmallSeeks,
                 forwardLargeSeeks, backwardLargeSeeks,
                 contiguousReads, nonContiguousReads,

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
@@ -43,6 +44,7 @@ protected SearchableSnapshotShardStats createTestInstance() {
 
     private CacheIndexInputStats randomCacheIndexInputStats() {
         return new CacheIndexInputStats(randomAlphaOfLength(10), randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(randomNonNegativeLong()), new ByteSizeValue(randomNonNegativeLong()),
             randomNonNegativeLong(), randomNonNegativeLong(),
             randomCounter(), randomCounter(),
             randomCounter(), randomCounter(),

x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml

@@ -390,3 +390,78 @@ teardown:
   - is_true: total.0.file_ext
   - is_true: indices.docs.total.0.file_ext
   - is_false: indices.docs.shards.0.0.files.0.file_ext
+
+
+---
+"Tests searchable snapshots stats with min/max/average sizes":
+  - skip:
+      version: " - 7.99.99"
+      reason:  min/max/average sizes introduced in 8.0.0
+
+  - do:
+      searchable_snapshots.mount:
+        repository: repository-fs
+        snapshot: snapshot
+        wait_for_completion: true
+        storage: shared_cache
+        body:
+          index: docs
+
+  - match: { snapshot.snapshot: snapshot }
+  - match: { snapshot.shards.failed: 0 }
+  - match: { snapshot.shards.successful: 1 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: docs
+        body:
+          query:
+            match_all: {}
+
+  - match: { hits.total: 3 }
+
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      searchable_snapshots.stats:
+        index: "d*"
+        level: "shards"
+
+  - match:  { _shards.total: 1 }
+  - match:  { _shards.failed: 0 }
+
+  - gt:      { total.0.min_size_in_bytes: 0 }
+  - gt:      { total.0.max_size_in_bytes: 0 }
+  - gt:      { total.0.avg_size_in_bytes: 0 }
+
+  - do:
+      searchable_snapshots.stats:
+        index: "d*"
+        level: "shards"
+        human: true
+
+  - is_true:   indices.docs.shards.0.0.files.0.min_size
+  - is_true:   indices.docs.shards.0.0.files.0.max_size
+  - is_true:   indices.docs.shards.0.0.files.0.avg_size
+
+  - do:
+      searchable_snapshots.stats:
+        index: "d*"
+        level: "cluster"
+
+  - gt:      { total.0.min_size_in_bytes: 0 }
+  - gt:      { total.0.max_size_in_bytes: 0 }
+  - gt:      { total.0.avg_size_in_bytes: 0 }
+
+  - do:
+      searchable_snapshots.stats:
+        index: "d*"
+
+  - gt:      { total.0.min_size_in_bytes: 0 }
+  - gt:      { total.0.max_size_in_bytes: 0 }
+  - gt:      { total.0.avg_size_in_bytes: 0 }
+

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

@@ -1475,6 +1475,21 @@ private void assertSearchableSnapshotStats(String indexName, boolean cacheEnable
                         indexInputStats.getTotalSize(),
                         greaterThan(0L)
                     );
+                    assertThat(
+                        "Unexpected min. file size for " + fileExt + " of shard " + shardRouting,
+                        indexInputStats.getMinSize().getBytes(),
+                        greaterThan(0L)
+                    );
+                    assertThat(
+                        "Unexpected max. file size for " + fileExt + " of shard " + shardRouting,
+                        indexInputStats.getMaxSize().getBytes(),
+                        greaterThan(0L)
+                    );
+                    assertThat(
+                        "Unexpected average file size for " + fileExt + " of shard " + shardRouting,
+                        indexInputStats.getAverageSize().getBytes(),
+                        greaterThan(0L)
+                    );
 
                     if (cacheEnabled == false || nonCachedExtensions.contains(fileExt)) {
                         assertThat(

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/IndexInputStats.java

@@ -25,8 +25,11 @@ public class IndexInputStats {
     /* A threshold beyond which an index input seeking is counted as "large" */
     static final ByteSizeValue SEEKING_THRESHOLD = new ByteSizeValue(8, ByteSizeUnit.MB);
 
-    private final int numFiles;
+    private final long numFiles;
     private final long totalSize;
+    private final ByteSizeValue minSize;
+    private final ByteSizeValue maxSize;
+
     private final long seekingThreshold;
     private final LongSupplier currentTimeNanos;
 
@@ -52,13 +55,22 @@ public class IndexInputStats {
     private final Counter blobStoreBytesRequested = new Counter();
     private final AtomicLong currentIndexCacheFills = new AtomicLong();
 
-    public IndexInputStats(int numFiles, long totalSize, LongSupplier currentTimeNanos) {
-        this(numFiles, totalSize, SEEKING_THRESHOLD.getBytes(), currentTimeNanos);
+    public IndexInputStats(long numFiles, long totalSize, long minSize, long maxSize, LongSupplier currentTimeNanos) {
+        this(numFiles, totalSize, minSize, maxSize, SEEKING_THRESHOLD.getBytes(), currentTimeNanos);
     }
 
-    public IndexInputStats(int numFiles, long totalSize, long seekingThreshold, LongSupplier currentTimeNanos) {
+    public IndexInputStats(
+        long numFiles,
+        long totalSize,
+        long minSize,
+        long maxSize,
+        long seekingThreshold,
+        LongSupplier currentTimeNanos
+    ) {
         this.numFiles = numFiles;
         this.totalSize = totalSize;
+        this.minSize = new ByteSizeValue(minSize);
+        this.maxSize = new ByteSizeValue(maxSize);
         this.seekingThreshold = seekingThreshold;
         this.currentTimeNanos = currentTimeNanos;
     }
@@ -137,14 +149,22 @@ public Releasable addIndexCacheFill() {
         };
     }
 
-    public int getNumFiles() {
+    public long getNumFiles() {
         return numFiles;
     }
 
     public long getTotalSize() {
         return totalSize;
     }
 
+    public ByteSizeValue getMinSize() {
+        return minSize;
+    }
+
+    public ByteSizeValue getMaxSize() {
+        return maxSize;
+    }
+
     public LongAdder getOpened() {
         return opened;
     }

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java

@@ -360,8 +360,8 @@ public void clearCache() {
         }
     }
 
-    protected IndexInputStats createIndexInputStats(final int numFiles, final long totalSize) {
-        return new IndexInputStats(numFiles, totalSize, statsCurrentTimeNanosSupplier);
+    protected IndexInputStats createIndexInputStats(long numFiles, long totalSize, long minSize, long maxSize) {
+        return new IndexInputStats(numFiles, totalSize, minSize, maxSize, statsCurrentTimeNanosSupplier);
     }
 
     public CacheKey createCacheKey(String fileName) {
@@ -395,13 +395,13 @@ public IndexInput openInput(final String name, final IOContext context) throws I
 
         final String ext = getNonNullFileExt(name);
         final IndexInputStats inputStats = stats.computeIfAbsent(ext, n -> {
-            // get all fileInfo with same extension
-            final Tuple<Integer, Long> fileExtCompoundStats = files().stream()
-                .filter(fi -> ext.equals(getNonNullFileExt(fi.physicalName())))
-                .map(fi -> Tuple.tuple(1, fi.length()))
-                .reduce((t1, t2) -> Tuple.tuple(t1.v1() + t2.v1(), t1.v2() + t2.v2()))
-                .get();
-            return createIndexInputStats(fileExtCompoundStats.v1(), fileExtCompoundStats.v2());
+            final IndexInputStats.Counter counter = new IndexInputStats.Counter();
+            for (BlobStoreIndexShardSnapshot.FileInfo file : files()) {
+                if (ext.equals(getNonNullFileExt(file.physicalName()))) {
+                    counter.add(file.length());
+                }
+            }
+            return createIndexInputStats(counter.count(), counter.total(), counter.min(), counter.max());
         });
         if (useCache && isExcludedFromCache(name) == false) {
             if (partial) {

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.store.IndexInputStats;
 import org.elasticsearch.index.store.SearchableSnapshotDirectory;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.license.XPackLicenseState;
@@ -23,7 +24,6 @@
 import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotShardStats.CacheIndexInputStats;
 import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotShardStats.Counter;
 import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotShardStats.TimedCounter;
-import org.elasticsearch.index.store.IndexInputStats;
 
 import java.io.IOException;
 import java.util.List;
@@ -101,6 +101,8 @@ private static CacheIndexInputStats toCacheIndexInputStats(final String fileExt,
             fileExt,
             inputStats.getNumFiles(),
             inputStats.getTotalSize(),
+            inputStats.getMinSize(),
+            inputStats.getMaxSize(),
             inputStats.getOpened().sum(),
             inputStats.getClosed().sum(),
             toCounter(inputStats.getForwardSmallSeeks()),

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/IndexInputStatsTests.java

@@ -23,7 +23,7 @@ public class IndexInputStatsTests extends ESTestCase {
 
     public void testReads() {
         final long fileLength = randomLongBetween(1L, 1_000L);
-        final IndexInputStats inputStats = new IndexInputStats(1, fileLength, FAKE_CLOCK);
+        final IndexInputStats inputStats = new IndexInputStats(1, fileLength, fileLength, fileLength, FAKE_CLOCK);
 
         assertCounter(inputStats.getContiguousReads(), 0L, 0L, 0L, 0L);
         assertCounter(inputStats.getNonContiguousReads(), 0L, 0L, 0L, 0L);
@@ -58,7 +58,7 @@ public void testReads() {
     public void testSeeks() {
         final long fileLength = randomLongBetween(1L, 1_000L);
         final long seekingThreshold = randomBoolean() ? randomLongBetween(1L, fileLength) : SEEKING_THRESHOLD.getBytes();
-        final IndexInputStats inputStats = new IndexInputStats(1, fileLength, seekingThreshold, FAKE_CLOCK);
+        final IndexInputStats inputStats = new IndexInputStats(1, fileLength, fileLength, fileLength, seekingThreshold, FAKE_CLOCK);
 
         assertCounter(inputStats.getForwardSmallSeeks(), 0L, 0L, 0L, 0L);
         assertCounter(inputStats.getForwardLargeSeeks(), 0L, 0L, 0L, 0L);
@@ -117,7 +117,7 @@ public void testSeeks() {
     }
 
     public void testSeekToSamePosition() {
-        final IndexInputStats inputStats = new IndexInputStats(1, randomLongBetween(1L, 1_000L), FAKE_CLOCK);
+        final IndexInputStats inputStats = new IndexInputStats(1, 123L, 123L, 123L, FAKE_CLOCK);
         final long position = randomLongBetween(0L, inputStats.getTotalSize());
 
         inputStats.incrementSeeks(position, position);

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java

@@ -655,11 +655,11 @@ private void executeTestCase(
                 frozenCacheService
             ) {
                 @Override
-                protected IndexInputStats createIndexInputStats(final int numFiles, final long totalSize) {
+                protected IndexInputStats createIndexInputStats(long numFiles, long totalSize, long minSize, long maxSize) {
                     if (seekingThreshold == null) {
-                        return super.createIndexInputStats(numFiles, totalSize);
+                        return super.createIndexInputStats(numFiles, totalSize, minSize, maxSize);
                     }
-                    return new IndexInputStats(numFiles, totalSize, seekingThreshold, statsCurrentTimeNanos);
+                    return new IndexInputStats(numFiles, totalSize, minSize, maxSize, seekingThreshold, statsCurrentTimeNanos);
                 }
             }
         ) {