From 98c8a87c52ef88da728225259e77f65733d2d7e6 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Thu, 24 Oct 2024 10:31:21 -0300 Subject: [PATCH] feat: usage/record capability definition (#1562) ### Add `usage/record` Capability definition for Egress Traffic Tracking **Summary:** - Introduced a new capability `usage/record` definition to track egress traffic metrics. - The `w3upInfra` upload-api will be updated to implement this new capability by leveraging Stripe's Usage Records API for accurate tracking and billing. RFC: https://github.com/storacha/RFC/pull/37 --- packages/capabilities/src/types.ts | 20 ++++++ packages/capabilities/src/usage.js | 17 +++++ packages/upload-api/src/types/usage.ts | 18 +++++- packages/upload-api/src/usage.js | 8 ++- packages/upload-api/src/usage/record.js | 41 ++++++++++++ .../test/storage/provisions-storage.js | 1 + .../upload-api/test/storage/usage-storage.js | 33 ++++++++++ packages/upload-client/src/types.ts | 8 +++ packages/w3up-client/src/capability/usage.js | 63 +++++++++++++++++++ packages/w3up-client/src/types.ts | 4 ++ .../w3up-client/test/capability/usage.test.js | 41 ++++++++++++ 11 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 packages/upload-api/src/usage/record.js diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index ba28eeb05..7704922e3 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -131,6 +131,10 @@ export type UsageReport = InferInvokedCapability export type UsageReportSuccess = Record export type UsageReportFailure = Ucanto.Failure +export type EgressRecord = InferInvokedCapability +export type EgressRecordSuccess = Unit +export type EgressRecordFailure = ConsumerNotFound | Ucanto.Failure + export interface UsageData { /** Provider the report concerns, e.g. `did:web:web3.storage` */ provider: ProviderDID @@ -161,6 +165,21 @@ export interface UsageData { }> } +export interface EgressData { + /** The space which contains the resource that was served. */ + space: SpaceDID + /** The customer that is being billed for the egress traffic. */ + customer: AccountDID + /** CID of the resource that was served it's the CID of some gateway accessible content. It is not the CID of a blob/shard.*/ + resource: UnknownLink + /** Amount of bytes served. */ + bytes: number + /** ISO datetime that the bytes were served at. */ + servedAt: ISO8601Date + /** Identifier of the invocation that caused the egress traffic. */ + cause: UnknownLink +} + // Provider export type ProviderAdd = InferInvokedCapability // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -193,6 +212,7 @@ export interface ConsumerGetSuccess { allocated: number limit: number subscription: string + customer: AccountDID } export interface ConsumerNotFound extends Ucanto.Failure { name: 'ConsumerNotFound' diff --git a/packages/capabilities/src/usage.js b/packages/capabilities/src/usage.js index d80fb212d..bde471236 100644 --- a/packages/capabilities/src/usage.js +++ b/packages/capabilities/src/usage.js @@ -40,3 +40,20 @@ export const report = capability({ ) }, }) + +/** + * Capability can be invoked by an agent to record usage data for a given resource. + */ +export const record = capability({ + can: 'usage/record', + with: SpaceDID, + nb: Schema.struct({ + /** CID of the resource that was served. */ + resource: Schema.link(), + /** Amount of bytes served. */ + bytes: Schema.integer().greaterThan(0), + /** Timestamp of the event in seconds after Unix epoch. */ + servedAt: Schema.integer().greaterThan(-1), + }), + derives: equalWith, +}) diff --git a/packages/upload-api/src/types/usage.ts b/packages/upload-api/src/types/usage.ts index bed076201..bd08a48cd 100644 --- a/packages/upload-api/src/types/usage.ts +++ b/packages/upload-api/src/types/usage.ts @@ -1,8 +1,10 @@ -import { Failure, Result } from '@ucanto/interface' +import { Failure, Result, UnknownLink } from '@ucanto/interface' import { + AccountDID, ProviderDID, SpaceDID, UsageData, + EgressData, } from '@web3-storage/capabilities/types' export type { UsageData } @@ -13,4 +15,18 @@ export interface UsageStorage { space: SpaceDID, period: { from: Date; to: Date } ) => Promise> + record: ( + /** The space which contains the resource that was served. */ + space: SpaceDID, + /** The customer that is being billed for the egress traffic. */ + customer: AccountDID, + /** The resource that was served to the customer through the gateway. */ + resource: UnknownLink, + /** The number of bytes that were served. */ + bytes: number, + /** The date and time when the resource was served. */ + servedAt: Date, + /** Identifier of the invocation that caused the egress traffic. */ + cause: UnknownLink, + ) => Promise> } diff --git a/packages/upload-api/src/usage.js b/packages/upload-api/src/usage.js index 04076baca..8cea68010 100644 --- a/packages/upload-api/src/usage.js +++ b/packages/upload-api/src/usage.js @@ -1,4 +1,8 @@ -import { provide } from './usage/report.js' +import { provide as provideReport } from './usage/report.js' +import { provide as provideRecord } from './usage/record.js' /** @param {import('./types.js').UsageServiceContext} context */ -export const createService = (context) => ({ report: provide(context) }) +export const createService = (context) => ({ + report: provideReport(context), + record: provideRecord(context), +}) diff --git a/packages/upload-api/src/usage/record.js b/packages/upload-api/src/usage/record.js new file mode 100644 index 000000000..f1d5b56bd --- /dev/null +++ b/packages/upload-api/src/usage/record.js @@ -0,0 +1,41 @@ +import * as API from '../types.js' +import * as Provider from '@ucanto/server' +import { Usage } from '@web3-storage/capabilities' + +/** @param {API.UsageServiceContext} context */ +export const provide = (context) => + Provider.provide(Usage.record, (input) => record(input, context)) + +/** + * @param {API.Input} input + * @param {API.UsageServiceContext} context + * @returns {Promise>} + */ +const record = async ({ capability, invocation }, context) => { + const provider = /** @type {`did:web:${string}`} */ ( + invocation.audience.did() + ) + const consumerResponse = await context.provisionsStorage.getConsumer( + provider, + capability.with + ) + if (consumerResponse.error) { + return consumerResponse + } + const consumer = consumerResponse.ok + const res = await context.usageStorage.record( + // The space which contains the resource that was served. + capability.with, + // The customer that is being billed for the egress traffic. + consumer.customer, + // CID of the resource that was served. + capability.nb.resource, + // Number of bytes that were served. + capability.nb.bytes, + // Date and time when the resource was served. + new Date(capability.nb.servedAt * 1000), + // Link to the invocation that caused the egress traffic. + invocation.cid + ) + return res +} diff --git a/packages/upload-api/test/storage/provisions-storage.js b/packages/upload-api/test/storage/provisions-storage.js index 272c44a7a..78f38426e 100644 --- a/packages/upload-api/test/storage/provisions-storage.js +++ b/packages/upload-api/test/storage/provisions-storage.js @@ -152,6 +152,7 @@ export class ProvisionsStorage { allocated: 0, limit: 100, subscription: itemKey(provision), + customer: provision.customer, }, } } else { diff --git a/packages/upload-api/test/storage/usage-storage.js b/packages/upload-api/test/storage/usage-storage.js index 706bc3d5b..8e571f87b 100644 --- a/packages/upload-api/test/storage/usage-storage.js +++ b/packages/upload-api/test/storage/usage-storage.js @@ -9,6 +9,10 @@ export class UsageStorage { constructor(storeTable, allocationsStorage) { this.storeTable = storeTable this.allocationsStorage = allocationsStorage + /** + * @type {Record} + */ + this._egressRecords = {} } get items() { @@ -64,4 +68,33 @@ export class UsageStorage { }, } } + + /** + * Simulate a record of egress data for a customer. + * + * @param {import('../types.js').SpaceDID} space + * @param {import('../types.js').AccountDID} customer + * @param {import('../types.js').UnknownLink} resource + * @param {number} bytes + * @param {Date} servedAt + * @param {import('../types.js').UnknownLink} cause + */ + async record(space, customer, resource, bytes, servedAt, cause) { + const egressData = { + space, + customer, + resource, + bytes, + servedAt: servedAt.toISOString(), + cause, + } + this._egressRecords[customer] = egressData + return Promise.resolve({ + ok: egressData, + }) + } + + get egressRecords() { + return this._egressRecords + } } diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 38a41f2f0..8d95612d6 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -69,6 +69,10 @@ import { UsageReport, UsageReportSuccess, UsageReportFailure, + EgressData, + EgressRecord, + EgressRecordSuccess, + EgressRecordFailure, ServiceAbility, } from '@web3-storage/capabilities/types' import { StorefrontService } from '@web3-storage/filecoin-client/storefront' @@ -135,6 +139,10 @@ export type { UsageReport, UsageReportSuccess, UsageReportFailure, + EgressData, + EgressRecord, + EgressRecordSuccess, + EgressRecordFailure, ListResponse, CARLink, PieceLink, diff --git a/packages/w3up-client/src/capability/usage.js b/packages/w3up-client/src/capability/usage.js index 19b53ba7c..5685dd772 100644 --- a/packages/w3up-client/src/capability/usage.js +++ b/packages/w3up-client/src/capability/usage.js @@ -31,6 +31,37 @@ export class UsageClient extends Base { return out.ok } + + /** + * Record egress data for a served resource. + * It will execute the capability invocation to find the customer and then record the egress data for the resource. + * + * Required delegated capabilities: + * - `usage/record` + * + * @param {import('../types.js').SpaceDID} space + * @param {object} egressData + * @param {API.UnknownLink} egressData.resource + * @param {number} egressData.bytes + * @param {string} egressData.servedAt + * @param {object} [options] + * @param {string} [options.nonce] + */ + async record(space, egressData, options) { + const out = await record( + { agent: this.agent }, + { space, ...egressData }, + { ...options } + ) + /* c8 ignore next 5 */ + if (!out.ok) { + throw new Error(`failed ${UsageCapabilities.record.can} invocation`, { + cause: out.error, + }) + } + + return out.ok + } } /** @@ -61,3 +92,35 @@ export const report = async ( }) return receipt.out } + +/** + * Record egress data for a resource from a given space. + * + * @param {{agent: API.Agent}} client + * @param {object} egressData + * @param {API.SpaceDID} egressData.space + * @param {API.UnknownLink} egressData.resource + * @param {number} egressData.bytes + * @param {string} egressData.servedAt + * @param {object} options + * @param {string} [options.nonce] + * @param {API.Delegation[]} [options.proofs] + * @returns {Promise>} + */ +export const record = async ( + { agent }, + { space, resource, bytes, servedAt }, + { nonce, proofs = [] } +) => { + const receipt = await agent.invokeAndExecute(UsageCapabilities.record, { + with: space, + proofs, + nonce, + nb: { + resource, + bytes, + servedAt: Math.floor(new Date(servedAt).getTime() / 1000), + }, + }) + return receipt.out +} diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 1480715fa..553c55e93 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -141,6 +141,10 @@ export type { UploadListItem, UsageReportSuccess, UsageReportFailure, + EgressData, + EgressRecord, + EgressRecordSuccess, + EgressRecordFailure, ListResponse, AnyLink, CARLink, diff --git a/packages/w3up-client/test/capability/usage.test.js b/packages/w3up-client/test/capability/usage.test.js index 13d598dd3..4ca1553ff 100644 --- a/packages/w3up-client/test/capability/usage.test.js +++ b/packages/w3up-client/test/capability/usage.test.js @@ -2,6 +2,7 @@ import { AgentData } from '@web3-storage/access/agent' import { Client } from '../../src/client.js' import * as Test from '../test.js' import { receiptsEndpoint } from '../helpers/utils.js' +import { randomCAR } from '../helpers/random.js' export const UsageClient = Test.withContext({ report: { @@ -68,6 +69,46 @@ export const UsageClient = Test.withContext({ assert.deepEqual(report, {}) }, }, + record: { + 'should record egress': async ( + assert, + { connection, provisionsStorage } + ) => { + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: { + access: connection, + upload: connection, + }, + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + await alice.addSpace(auth) + + // Then we setup a billing for this account + await provisionsStorage.put({ + // @ts-expect-error + provider: connection.id.did(), + account: alice.agent.did(), + consumer: space.did(), + }) + + const car = await randomCAR(128) + const resource = car.cid + await alice.capability.upload.add(car.roots[0], [resource]) + + const result = await alice.capability.upload.get(car.roots[0]) + assert.ok(result) + + const record = await alice.capability.usage.record(space.did(), { + resource: resource.link(), + bytes: car.size, + servedAt: new Date().toISOString(), + }) + assert.ok(record) + }, + }, }) Test.test({ UsageClient })