feat: track content length histogram

packages/gateway/src/durable-objects/summary-metrics.js

@@ -3,6 +3,16 @@
  * @property {number} totalWinnerResponseTime total response time of the requests
  * @property {number} totalWinnerSuccessfulRequests total number of successful requests
  * @property {number} totalCachedResponses total number of cached responses
+ * @property {BigInt} totalContentLengthBytes total content length of responses
+ * @property {BigInt} totalCachedContentLengthBytes total content length of cached responses
+ * @property {Record<string, number>} contentLengthHistogram
+ *
+ * @typedef {Object} ResponseWinnerStats
+ * @property {number} responseTime number of milliseconds to get response
+ * @property {number} contentLength content length header content
+ *
+ * @typedef {Object} ContentLengthStats
+ * @property {number} contentLength content length header content
  */
 
 // Key to track total time for winner gateway to respond
@@ -11,11 +21,17 @@ const TOTAL_WINNER_RESPONSE_TIME_ID = 'totalWinnerResponseTime'
 const TOTAL_WINNER_SUCCESSFUL_REQUESTS_ID = 'totalWinnerSuccessfulRequests'
 // Key to track total cached requests
 const TOTAL_CACHED_RESPONSES_ID = 'totalCachedResponses'
+// Key to track total content length of responses
+const TOTAL_CONTENT_LENGTH_BYTES_ID = 'totalContentLengthBytes'
+// Key to track total cached content length of responses
+const TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID = 'totalCachedContentLengthBytes'
+// Key to track content size histogram
+const CONTENT_LENGTH_HISTOGRAM_ID = 'contentLengthHistogram'
 
 /**
  * Durable Object for keeping generic Metrics of gateway.nft.storage
  */
-export class SummaryMetrics0 {
+export class SummaryMetrics1 {
   constructor(state) {
     this.state = state
 
@@ -30,6 +46,18 @@ export class SummaryMetrics0 {
       // Total cached requests
       this.totalCachedResponses =
         (await this.state.storage.get(TOTAL_CACHED_RESPONSES_ID)) || 0
+      // Total content length responses
+      this.totalContentLengthBytes =
+        (await this.state.storage.get(TOTAL_CONTENT_LENGTH_BYTES_ID)) ||
+        BigInt(0)
+      // Total cached content length responses
+      this.totalCachedContentLengthBytes =
+        (await this.state.storage.get(TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID)) ||
+        BigInt(0)
+      // Content length histogram
+      this.contentLengthHistogram =
+        (await this.state.storage.get(CONTENT_LENGTH_HISTOGRAM_ID)) ||
+        createHistogramObject()
     })
   }
 
@@ -47,6 +75,10 @@ export class SummaryMetrics0 {
               totalWinnerResponseTime: this.totalWinnerResponseTime,
               totalWinnerSuccessfulRequests: this.totalWinnerSuccessfulRequests,
               totalCachedResponses: this.totalCachedResponses,
+              totalContentLengthBytes: this.totalContentLengthBytes.toString(),
+              totalCachedContentLengthBytes:
+                this.totalCachedContentLengthBytes.toString(),
+              contentLengthHistogram: this.contentLengthHistogram,
             })
           )
         default:
@@ -55,35 +87,110 @@ export class SummaryMetrics0 {
     }
 
     // POST
+    let data
     switch (url.pathname) {
       case '/metrics/winner':
-        const data = await request.json()
-        // Updated Metrics
-        this.totalWinnerResponseTime += data.responseTime
-        this.totalWinnerSuccessfulRequests += 1
-        // Save updated Metrics
-        await Promise.all([
-          this.state.storage.put(
-            TOTAL_WINNER_RESPONSE_TIME_ID,
-            this.totalWinnerResponseTime
-          ),
-          this.state.storage.put(
-            TOTAL_WINNER_SUCCESSFUL_REQUESTS_ID,
-            this.totalWinnerSuccessfulRequests
-          ),
-        ])
+        /** @type {ResponseWinnerStats} */
+        data = await request.json()
+        await this._updateWinnerMetrics(data)
         return new Response()
       case '/metrics/cache':
-        // Update metrics
-        this.totalCachedResponses += 1
-        // Sabe updated metrics
-        await this.state.storage.put(
-          TOTAL_CACHED_RESPONSES_ID,
-          this.totalCachedResponses
-        )
+        /** @type {ContentLengthStats} */
+        data = await request.json()
+        await this._updateWinnerMetrics(data)
         return new Response()
       default:
         return new Response('Not found', { status: 404 })
     }
   }
+
+  /**
+   * @param {ContentLengthStats} stats
+   */
+  async _updatedCacheMetrics(stats) {
+    // Update metrics
+    this.totalCachedResponses += 1
+    this.totalCachedContentLengthBytes += BigInt(stats.contentLength)
+    this._updateContentLengthMetrics(stats)
+    // Sabe updated metrics
+    await Promise.all([
+      this.state.storage.put(
+        TOTAL_CACHED_RESPONSES_ID,
+        this.totalCachedResponses
+      ),
+      this.state.storage.put(
+        TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID,
+        this.totalCachedContentLengthBytes
+      ),
+      this.state.storage.put(
+        TOTAL_CONTENT_LENGTH_BYTES_ID,
+        this.totalContentLengthBytes
+      ),
+      this.state.storage.put(
+        CONTENT_LENGTH_HISTOGRAM_ID,
+        this.contentLengthHistogram
+      ),
+    ])
+  }
+
+  /**
+   * @param {ResponseWinnerStats} stats
+   */
+  async _updateWinnerMetrics(stats) {
+    // Updated Metrics
+    this.totalWinnerResponseTime += stats.responseTime
+    this.totalWinnerSuccessfulRequests += 1
+    this._updateContentLengthMetrics(stats)
+    // Save updated Metrics
+    await Promise.all([
+      this.state.storage.put(
+        TOTAL_WINNER_RESPONSE_TIME_ID,
+        this.totalWinnerResponseTime
+      ),
+      this.state.storage.put(
+        TOTAL_WINNER_SUCCESSFUL_REQUESTS_ID,
+        this.totalWinnerSuccessfulRequests
+      ),
+      this.state.storage.put(
+        TOTAL_CONTENT_LENGTH_BYTES_ID,
+        this.totalContentLengthBytes
+      ),
+      this.state.storage.put(
+        CONTENT_LENGTH_HISTOGRAM_ID,
+        this.contentLengthHistogram
+      ),
+    ])
+  }
+
+  /**
+   * @param {ContentLengthStats} stats
+   */
+  _updateContentLengthMetrics(stats) {
+    this.totalContentLengthBytes += BigInt(stats.contentLength)
+
+    // Update histogram
+    const tmpHistogram = {
+      ...this.contentLengthHistogram,
+    }
+
+    // Get all the histogram buckets where the content size is smaller
+    const histogramCandidates = contentLengthHistogram.filter(
+      (h) => stats.contentLength < h
+    )
+    histogramCandidates.forEach((candidate) => {
+      tmpHistogram[candidate] += 1
+    })
+
+    this.contentLengthHistogram = tmpHistogram
+  }
 }
+
+function createHistogramObject() {
+  const h = contentLengthHistogram.map((h) => [h, 0])
+  return Object.fromEntries(h)
+}
+
+// We will count occurences per bucket where content size is less or equal than bucket value
+export const contentLengthHistogram = [
+  0.5, 1, 2, 5, 25, 50, 100, 500, 1000, 5000, 10000, 15000, 20000, 30000, 32000,
+].map((v) => v * Math.pow(1024, 2))

packages/gateway/src/gateway.js

@@ -27,7 +27,7 @@ export async function gatewayGet(request, env, ctx) {
 
   if (res) {
     // Update cache metrics in background
-    ctx.waitUntil(updateSummaryCacheMetrics(request, env))
+    ctx.waitUntil(updateSummaryCacheMetrics(request, env, res))
     return res
   }
 
@@ -150,13 +150,21 @@ async function _gatewayFetch(
 /**
  * @param {Request} request
  * @param {import('./env').Env} env
+ * @param {Response} response
  */
-async function updateSummaryCacheMetrics(request, env) {
+async function updateSummaryCacheMetrics(request, env, response) {
   // Get durable object for gateway
   const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID)
   const stub = env.summaryMetricsDurable.get(id)
 
-  await stub.fetch(_getDurableRequestUrl(request, 'metrics/cache'))
+  /** @type {import('./durable-objects/summary-metrics').ContentLengthStats} */
+  const contentLengthStats = {
+    contentLength: Number(response.headers.get('content-length')),
+  }
+
+  await stub.fetch(
+    _getDurableRequestUrl(request, 'metrics/cache', contentLengthStats)
+  )
 }
 
 /**
@@ -169,10 +177,10 @@ async function updateSummaryWinnerMetrics(request, env, gwResponse) {
   const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID)
   const stub = env.summaryMetricsDurable.get(id)
 
-  /** @type {import('./durable-objects/gateway-metrics').ResponseStats} */
+  /** @type {import('./durable-objects/summary-metrics').ResponseWinnerStats} */
   const responseStats = {
-    ok: gwResponse.response.ok,
     responseTime: gwResponse.responseTime,
+    contentLength: Number(gwResponse.response.headers.get('content-length')),
   }
 
   await stub.fetch(

packages/gateway/src/index.js

@@ -7,7 +7,7 @@ import { metricsGet } from './metrics.js'
 
 // Export Durable Object namespace from the root module.
 export { GatewayMetrics1 } from './durable-objects/gateway-metrics.js'
-export { SummaryMetrics0 } from './durable-objects/summary-metrics.js'
+export { SummaryMetrics1 } from './durable-objects/summary-metrics.js'
 export { CidsTracker0 } from './durable-objects/cids.js'
 
 import { addCorsHeaders, withCorsHeaders } from './cors.js'

packages/gateway/src/metrics.js

@@ -4,6 +4,7 @@ import pMap from 'p-map'
 
 import { METRICS_CACHE_MAX_AGE, SUMMARY_METRICS_ID } from './constants.js'
 import { histogram } from './durable-objects/gateway-metrics.js'
+import { contentLengthHistogram } from './durable-objects/summary-metrics.js'
 
 /**
  * @typedef {import('./durable-objects/gateway-metrics').GatewayMetrics} GatewayMetrics
@@ -128,6 +129,19 @@ export async function metricsGet(request, env, ctx) {
       (gw) =>
         `nftgateway_requests_per_time_total{gateway="${gw}",le="+Inf",env="${env.ENV}"} ${metricsCollected.ipfsGateways[gw].totalSuccessfulRequests}`
     ),
+    `# HELP nftgateway_responses_content_length_total`,
+    `# TYPE nftgateway_responses_content_length_total content length delivered histogram`,
+    ...contentLengthHistogram.map(
+      (t) =>
+        `nftgateway_responses_content_length_total{le="${t}",env="${env.ENV}"} ${metricsCollected.summaryMetrics.contentLengthHistogram[t]}`
+    ),
+    `nftgateway_responses_content_length_total{le="+Inf",env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalWinnerSuccessfulRequests}`,
+    `# HELP nftgateway_responses_content_length_bytes_total`,
+    `# TYPE nftgateway_responses_content_length_bytes_total content length of delivered cached responses`,
+    `nftgateway_responses_content_length_bytes_total{env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalContentLengthBytes}`,
+    `# HELP nftgateway_cached_responses_content_length_bytes_total`,
+    `# TYPE nftgateway_cached_responses_content_length_bytes_total content length of delivered cached responses`,
+    `nftgateway_cached_responses_content_length_bytes_total{env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalCachedContentLengthBytes}`,
   ].join('\n')
 
   res = new Response(metrics, {

packages/gateway/test/metrics.spec.js

@@ -21,6 +21,13 @@ test('Gets Metrics content when empty state', async (t) => {
     true
   )
   t.is(metricsResponse.includes('nftgateway_winner_requests_total'), true)
+  t.is(metricsResponse.includes(`_responses_content_length_total{le=`), true)
+  t.is(
+    metricsResponse.includes(
+      `_responses_content_length_bytes_total{env="test"} 0`
+    ),
+    true
+  )
   gateways.forEach((gw) => {
     t.is(
       metricsResponse.includes(`_requests_total{gateway="${gw}",env="test"} 0`),
@@ -54,7 +61,23 @@ test('Gets Metrics content when empty state', async (t) => {
 test('Gets Metrics content', async (t) => {
   const { mf } = t.context
 
-  // Trigger two requests
+  let response = await mf.dispatchFetch('http://localhost:8787/metrics')
+  let metricsResponse = await response.text()
+
+  t.is(
+    metricsResponse.includes(
+      `_responses_content_length_total{le="524288",env="test"} 0`
+    ),
+    true
+  )
+  t.is(
+    metricsResponse.includes(
+      `_responses_content_length_bytes_total{env="test"} 0`
+    ),
+    true
+  )
+
+  // Trigger two requests with content length of 23 each
   const p = await Promise.all([
     mf.dispatchFetch(
       'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:8787'
@@ -67,8 +90,21 @@ test('Gets Metrics content', async (t) => {
   // Wait for waitUntil
   await Promise.all(p.map((p) => p.waitUntil()))
 
-  const response = await mf.dispatchFetch('http://localhost:8787/metrics')
-  const metricsResponse = await response.text()
+  response = await mf.dispatchFetch('http://localhost:8787/metrics')
+  metricsResponse = await response.text()
+
+  t.is(
+    metricsResponse.includes(
+      `_responses_content_length_total{le="524288",env="test"} 2`
+    ),
+    true
+  )
+  t.is(
+    metricsResponse.includes(
+      `_responses_content_length_bytes_total{env="test"} 46`
+    ),
+    true
+  )
 
   gateways.forEach((gw) => {
     t.is(

packages/gateway/wrangler.toml

@@ -19,7 +19,7 @@ main = "index.mjs"
 [durable_objects]
 bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics1"},
-  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
+  {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"}
 ]
 
@@ -129,3 +129,7 @@ deleted_classes = ["GatewayMetrics0"]
 tag = "v17" # Should be unique for each entry
 new_classes = ["SummaryMetrics0"]
 deleted_classes = ["GenericMetrics1"]
+[[migrations]]
+tag = "v18" # Should be unique for each entry
+new_classes = ["SummaryMetrics1"]
+deleted_classes = ["SummaryMetrics0"]