diff --git a/packages/gateway/src/durable-objects/summary-metrics.js b/packages/gateway/src/durable-objects/summary-metrics.js index e1456bfa18..a967f3d322 100644 --- a/packages/gateway/src/durable-objects/summary-metrics.js +++ b/packages/gateway/src/durable-objects/summary-metrics.js @@ -3,6 +3,16 @@ * @property {number} totalWinnerResponseTime total response time of the requests * @property {number} totalWinnerSuccessfulRequests total number of successful requests * @property {number} totalCachedResponses total number of cached responses + * @property {BigInt} totalContentLengthBytes total content length of responses + * @property {BigInt} totalCachedContentLengthBytes total content length of cached responses + * @property {Record} contentLengthHistogram + * + * @typedef {Object} ResponseWinnerStats + * @property {number} responseTime number of milliseconds to get response + * @property {number} contentLength content length header content + * + * @typedef {Object} ContentLengthStats + * @property {number} contentLength content length header content */ // Key to track total time for winner gateway to respond @@ -11,11 +21,17 @@ const TOTAL_WINNER_RESPONSE_TIME_ID = 'totalWinnerResponseTime' const TOTAL_WINNER_SUCCESSFUL_REQUESTS_ID = 'totalWinnerSuccessfulRequests' // Key to track total cached requests const TOTAL_CACHED_RESPONSES_ID = 'totalCachedResponses' +// Key to track total content length of responses +const TOTAL_CONTENT_LENGTH_BYTES_ID = 'totalContentLengthBytes' +// Key to track total cached content length of responses +const TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID = 'totalCachedContentLengthBytes' +// Key to track content size histogram +const CONTENT_LENGTH_HISTOGRAM_ID = 'contentLengthHistogram' /** * Durable Object for keeping generic Metrics of gateway.nft.storage */ -export class SummaryMetrics0 { +export class SummaryMetrics1 { constructor(state) { this.state = state @@ -30,6 +46,18 @@ export class SummaryMetrics0 { // Total cached requests this.totalCachedResponses = (await this.state.storage.get(TOTAL_CACHED_RESPONSES_ID)) || 0 + // Total content length responses + this.totalContentLengthBytes = + (await this.state.storage.get(TOTAL_CONTENT_LENGTH_BYTES_ID)) || + BigInt(0) + // Total cached content length responses + this.totalCachedContentLengthBytes = + (await this.state.storage.get(TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID)) || + BigInt(0) + // Content length histogram + this.contentLengthHistogram = + (await this.state.storage.get(CONTENT_LENGTH_HISTOGRAM_ID)) || + createHistogramObject() }) } @@ -47,6 +75,10 @@ export class SummaryMetrics0 { totalWinnerResponseTime: this.totalWinnerResponseTime, totalWinnerSuccessfulRequests: this.totalWinnerSuccessfulRequests, totalCachedResponses: this.totalCachedResponses, + totalContentLengthBytes: this.totalContentLengthBytes.toString(), + totalCachedContentLengthBytes: + this.totalCachedContentLengthBytes.toString(), + contentLengthHistogram: this.contentLengthHistogram, }) ) default: @@ -55,35 +87,110 @@ export class SummaryMetrics0 { } // POST + let data switch (url.pathname) { case '/metrics/winner': - const data = await request.json() - // Updated Metrics - this.totalWinnerResponseTime += data.responseTime - this.totalWinnerSuccessfulRequests += 1 - // Save updated Metrics - await Promise.all([ - this.state.storage.put( - TOTAL_WINNER_RESPONSE_TIME_ID, - this.totalWinnerResponseTime - ), - this.state.storage.put( - TOTAL_WINNER_SUCCESSFUL_REQUESTS_ID, - this.totalWinnerSuccessfulRequests - ), - ]) + /** @type {ResponseWinnerStats} */ + data = await request.json() + await this._updateWinnerMetrics(data) return new Response() case '/metrics/cache': - // Update metrics - this.totalCachedResponses += 1 - // Sabe updated metrics - await this.state.storage.put( - TOTAL_CACHED_RESPONSES_ID, - this.totalCachedResponses - ) + /** @type {ContentLengthStats} */ + data = await request.json() + await this._updateWinnerMetrics(data) return new Response() default: return new Response('Not found', { status: 404 }) } } + + /** + * @param {ContentLengthStats} stats + */ + async _updatedCacheMetrics(stats) { + // Update metrics + this.totalCachedResponses += 1 + this.totalCachedContentLengthBytes += BigInt(stats.contentLength) + this._updateContentLengthMetrics(stats) + // Sabe updated metrics + await Promise.all([ + this.state.storage.put( + TOTAL_CACHED_RESPONSES_ID, + this.totalCachedResponses + ), + this.state.storage.put( + TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID, + this.totalCachedContentLengthBytes + ), + this.state.storage.put( + TOTAL_CONTENT_LENGTH_BYTES_ID, + this.totalContentLengthBytes + ), + this.state.storage.put( + CONTENT_LENGTH_HISTOGRAM_ID, + this.contentLengthHistogram + ), + ]) + } + + /** + * @param {ResponseWinnerStats} stats + */ + async _updateWinnerMetrics(stats) { + // Updated Metrics + this.totalWinnerResponseTime += stats.responseTime + this.totalWinnerSuccessfulRequests += 1 + this._updateContentLengthMetrics(stats) + // Save updated Metrics + await Promise.all([ + this.state.storage.put( + TOTAL_WINNER_RESPONSE_TIME_ID, + this.totalWinnerResponseTime + ), + this.state.storage.put( + TOTAL_WINNER_SUCCESSFUL_REQUESTS_ID, + this.totalWinnerSuccessfulRequests + ), + this.state.storage.put( + TOTAL_CONTENT_LENGTH_BYTES_ID, + this.totalContentLengthBytes + ), + this.state.storage.put( + CONTENT_LENGTH_HISTOGRAM_ID, + this.contentLengthHistogram + ), + ]) + } + + /** + * @param {ContentLengthStats} stats + */ + _updateContentLengthMetrics(stats) { + this.totalContentLengthBytes += BigInt(stats.contentLength) + + // Update histogram + const tmpHistogram = { + ...this.contentLengthHistogram, + } + + // Get all the histogram buckets where the content size is smaller + const histogramCandidates = contentLengthHistogram.filter( + (h) => stats.contentLength < h + ) + histogramCandidates.forEach((candidate) => { + tmpHistogram[candidate] += 1 + }) + + this.contentLengthHistogram = tmpHistogram + } } + +function createHistogramObject() { + const h = contentLengthHistogram.map((h) => [h, 0]) + return Object.fromEntries(h) +} + +// We will count occurences per bucket where content size is less or equal than bucket value +export const contentLengthHistogram = [ + 0.5, 1, 2, 5, 25, 50, 100, 500, 1000, 5000, 10000, 15000, 20000, 30000, 32000, +].map((v) => v * Math.pow(1024, 2)) diff --git a/packages/gateway/src/gateway.js b/packages/gateway/src/gateway.js index 6112c7eb49..5dce35f94f 100644 --- a/packages/gateway/src/gateway.js +++ b/packages/gateway/src/gateway.js @@ -27,7 +27,7 @@ export async function gatewayGet(request, env, ctx) { if (res) { // Update cache metrics in background - ctx.waitUntil(updateSummaryCacheMetrics(request, env)) + ctx.waitUntil(updateSummaryCacheMetrics(request, env, res)) return res } @@ -150,13 +150,21 @@ async function _gatewayFetch( /** * @param {Request} request * @param {import('./env').Env} env + * @param {Response} response */ -async function updateSummaryCacheMetrics(request, env) { +async function updateSummaryCacheMetrics(request, env, response) { // Get durable object for gateway const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID) const stub = env.summaryMetricsDurable.get(id) - await stub.fetch(_getDurableRequestUrl(request, 'metrics/cache')) + /** @type {import('./durable-objects/summary-metrics').ContentLengthStats} */ + const contentLengthStats = { + contentLength: Number(response.headers.get('content-length')), + } + + await stub.fetch( + _getDurableRequestUrl(request, 'metrics/cache', contentLengthStats) + ) } /** @@ -169,10 +177,10 @@ async function updateSummaryWinnerMetrics(request, env, gwResponse) { const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID) const stub = env.summaryMetricsDurable.get(id) - /** @type {import('./durable-objects/gateway-metrics').ResponseStats} */ + /** @type {import('./durable-objects/summary-metrics').ResponseWinnerStats} */ const responseStats = { - ok: gwResponse.response.ok, responseTime: gwResponse.responseTime, + contentLength: Number(gwResponse.response.headers.get('content-length')), } await stub.fetch( diff --git a/packages/gateway/src/index.js b/packages/gateway/src/index.js index e3e7f56934..034df30c13 100644 --- a/packages/gateway/src/index.js +++ b/packages/gateway/src/index.js @@ -7,7 +7,7 @@ import { metricsGet } from './metrics.js' // Export Durable Object namespace from the root module. export { GatewayMetrics1 } from './durable-objects/gateway-metrics.js' -export { SummaryMetrics0 } from './durable-objects/summary-metrics.js' +export { SummaryMetrics1 } from './durable-objects/summary-metrics.js' export { CidsTracker0 } from './durable-objects/cids.js' import { addCorsHeaders, withCorsHeaders } from './cors.js' diff --git a/packages/gateway/src/metrics.js b/packages/gateway/src/metrics.js index d0cf98e0cf..45b47e1b4f 100644 --- a/packages/gateway/src/metrics.js +++ b/packages/gateway/src/metrics.js @@ -4,6 +4,7 @@ import pMap from 'p-map' import { METRICS_CACHE_MAX_AGE, SUMMARY_METRICS_ID } from './constants.js' import { histogram } from './durable-objects/gateway-metrics.js' +import { contentLengthHistogram } from './durable-objects/summary-metrics.js' /** * @typedef {import('./durable-objects/gateway-metrics').GatewayMetrics} GatewayMetrics @@ -128,6 +129,19 @@ export async function metricsGet(request, env, ctx) { (gw) => `nftgateway_requests_per_time_total{gateway="${gw}",le="+Inf",env="${env.ENV}"} ${metricsCollected.ipfsGateways[gw].totalSuccessfulRequests}` ), + `# HELP nftgateway_responses_content_length_total`, + `# TYPE nftgateway_responses_content_length_total content length delivered histogram`, + ...contentLengthHistogram.map( + (t) => + `nftgateway_responses_content_length_total{le="${t}",env="${env.ENV}"} ${metricsCollected.summaryMetrics.contentLengthHistogram[t]}` + ), + `nftgateway_responses_content_length_total{le="+Inf",env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalWinnerSuccessfulRequests}`, + `# HELP nftgateway_responses_content_length_bytes_total`, + `# TYPE nftgateway_responses_content_length_bytes_total content length of delivered cached responses`, + `nftgateway_responses_content_length_bytes_total{env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalContentLengthBytes}`, + `# HELP nftgateway_cached_responses_content_length_bytes_total`, + `# TYPE nftgateway_cached_responses_content_length_bytes_total content length of delivered cached responses`, + `nftgateway_cached_responses_content_length_bytes_total{env="${env.ENV}"} ${metricsCollected.summaryMetrics.totalCachedContentLengthBytes}`, ].join('\n') res = new Response(metrics, { diff --git a/packages/gateway/test/metrics.spec.js b/packages/gateway/test/metrics.spec.js index e5d0cb301e..489539b778 100644 --- a/packages/gateway/test/metrics.spec.js +++ b/packages/gateway/test/metrics.spec.js @@ -21,6 +21,13 @@ test('Gets Metrics content when empty state', async (t) => { true ) t.is(metricsResponse.includes('nftgateway_winner_requests_total'), true) + t.is(metricsResponse.includes(`_responses_content_length_total{le=`), true) + t.is( + metricsResponse.includes( + `_responses_content_length_bytes_total{env="test"} 0` + ), + true + ) gateways.forEach((gw) => { t.is( metricsResponse.includes(`_requests_total{gateway="${gw}",env="test"} 0`), @@ -54,7 +61,23 @@ test('Gets Metrics content when empty state', async (t) => { test('Gets Metrics content', async (t) => { const { mf } = t.context - // Trigger two requests + let response = await mf.dispatchFetch('http://localhost:8787/metrics') + let metricsResponse = await response.text() + + t.is( + metricsResponse.includes( + `_responses_content_length_total{le="524288",env="test"} 0` + ), + true + ) + t.is( + metricsResponse.includes( + `_responses_content_length_bytes_total{env="test"} 0` + ), + true + ) + + // Trigger two requests with content length of 23 each const p = await Promise.all([ mf.dispatchFetch( 'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:8787' @@ -67,8 +90,21 @@ test('Gets Metrics content', async (t) => { // Wait for waitUntil await Promise.all(p.map((p) => p.waitUntil())) - const response = await mf.dispatchFetch('http://localhost:8787/metrics') - const metricsResponse = await response.text() + response = await mf.dispatchFetch('http://localhost:8787/metrics') + metricsResponse = await response.text() + + t.is( + metricsResponse.includes( + `_responses_content_length_total{le="524288",env="test"} 2` + ), + true + ) + t.is( + metricsResponse.includes( + `_responses_content_length_bytes_total{env="test"} 46` + ), + true + ) gateways.forEach((gw) => { t.is( diff --git a/packages/gateway/wrangler.toml b/packages/gateway/wrangler.toml index ff0e98b288..2735aa0020 100644 --- a/packages/gateway/wrangler.toml +++ b/packages/gateway/wrangler.toml @@ -19,7 +19,7 @@ main = "index.mjs" [durable_objects] bindings = [ {name = "GATEWAYMETRICS", class_name = "GatewayMetrics1"}, - {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"}, + {name = "SUMMARYMETRICS", class_name = "SummaryMetrics1"}, {name = "CIDSTRACKER", class_name = "CidsTracker0"} ] @@ -129,3 +129,7 @@ deleted_classes = ["GatewayMetrics0"] tag = "v17" # Should be unique for each entry new_classes = ["SummaryMetrics0"] deleted_classes = ["GenericMetrics1"] +[[migrations]] +tag = "v18" # Should be unique for each entry +new_classes = ["SummaryMetrics1"] +deleted_classes = ["SummaryMetrics0"] \ No newline at end of file