From adb24424d1faf50daf2339b77c22fdd44faa236a Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 8 Nov 2023 17:20:31 +0100 Subject: [PATCH] feat: filecoin info (#1091) Goal of the PR is to make filecoin deals information available for end users via client (and later on CLI + console). For this: - adds `filecoin/info` capability to `capabilities`, `filecoin-api` and `filecoin-client` - user can give the `piece` that was computed while uploading the file, or that was published via content claims - exposes via `w3up-client` some filecoin sugar `*.filecoin.offer` and `*.filecoin.info` Note that `filecoin.info` only returns effective deals. Alternatively, user could ask directly dealTracker, even though user would need to know the aggregate where their piece was included. So, better make storefront facilitate user life with this shortcut Would be great to actually give more information of the on going state, but I am failing to find a nice way of giving this info here. However, probably should be a separate capability (e.g. `track`, so that we avoid reading from so many stores and follow the chain, unless user specifically looks for it) Note: I will add this to spec later on --- packages/capabilities/src/filecoin/index.js | 1 + .../capabilities/src/filecoin/storefront.js | 27 +++ packages/capabilities/src/index.js | 1 + packages/capabilities/src/types.ts | 19 ++ packages/filecoin-api/src/storefront/api.ts | 5 + .../filecoin-api/src/storefront/service.js | 99 +++++++++- .../filecoin-api/test/context/receipts.js | 94 +++++++++ packages/filecoin-api/test/context/store.js | 8 +- .../filecoin-api/test/events/storefront.js | 1 + .../filecoin-api/test/services/storefront.js | 184 +++++++++++++++++- packages/filecoin-api/test/storefront.spec.js | 14 ++ packages/filecoin-client/src/storefront.js | 30 +++ packages/filecoin-client/src/types.ts | 4 + .../filecoin-client/test/helpers/mocks.js | 1 + .../filecoin-client/test/storefront.test.js | 50 +++++ packages/upload-api/test/helpers/context.js | 17 ++ packages/w3up-client/README.md | 23 +++ packages/w3up-client/package.json | 4 +- .../w3up-client/src/capability/filecoin.js | 33 ++++ packages/w3up-client/src/client.js | 2 + packages/w3up-client/src/service.js | 13 ++ packages/w3up-client/src/types.ts | 2 + .../test/capability/filecoin.test.js | 162 +++++++++++++++ packages/w3up-client/test/helpers/car.js | 2 +- packages/w3up-client/test/helpers/mocks.js | 7 +- packages/w3up-client/test/helpers/random.js | 40 ++++ packages/w3up-client/test/test.js | 1 + pnpm-lock.yaml | 16 +- 28 files changed, 844 insertions(+), 16 deletions(-) create mode 100644 packages/w3up-client/src/capability/filecoin.js create mode 100644 packages/w3up-client/test/capability/filecoin.test.js diff --git a/packages/capabilities/src/filecoin/index.js b/packages/capabilities/src/filecoin/index.js index d7675e0fd..aee774398 100644 --- a/packages/capabilities/src/filecoin/index.js +++ b/packages/capabilities/src/filecoin/index.js @@ -16,4 +16,5 @@ export { filecoinOffer as offer, filecoinSubmit as submit, filecoinAccept as accept, + filecoinInfo as info, } from './storefront.js' diff --git a/packages/capabilities/src/filecoin/storefront.js b/packages/capabilities/src/filecoin/storefront.js index 88b103690..53f0cc4b3 100644 --- a/packages/capabilities/src/filecoin/storefront.js +++ b/packages/capabilities/src/filecoin/storefront.js @@ -106,3 +106,30 @@ export const filecoinAccept = capability({ ) }, }) + +/** + * Capability allowing an agent to _request_ info about a content piece in + * Filecoin deals. + */ +export const filecoinInfo = capability({ + can: 'filecoin/info', + /** + * DID of the space the content is stored in. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the piece. + * + * @see https://github.com/filecoin-project/FIPs/pull/758/files + */ + piece: PieceLink, + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 7c44a620c..5d814cda3 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -73,6 +73,7 @@ export const abilitiesAsStrings = [ Storefront.filecoinOffer.can, Storefront.filecoinSubmit.can, Storefront.filecoinAccept.can, + Storefront.filecoinInfo.can, Aggregator.pieceOffer.can, Aggregator.pieceAccept.can, Dealer.aggregateOffer.can, diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index c72d98830..1038fed0b 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -305,6 +305,21 @@ export interface ProofNotFound extends Ucanto.Failure { name: 'ProofNotFound' } +export interface FilecoinInfoSuccess { + piece: PieceLink + deals: FilecoinInfoAcceptedDeal[] +} +export interface FilecoinInfoAcceptedDeal + extends DataAggregationProof, + DealDetails { + aggregate: PieceLink +} + +export type FilecoinInfoFailure = + | ContentNotFound + | InvalidContentPiece + | Ucanto.Failure + // filecoin aggregator export interface PieceOfferSuccess { /** @@ -549,6 +564,9 @@ export type FilecoinSubmit = InferInvokedCapability< export type FilecoinAccept = InferInvokedCapability< typeof StorefrontCaps.filecoinAccept > +export type FilecoinInfo = InferInvokedCapability< + typeof StorefrontCaps.filecoinInfo +> export type PieceOffer = InferInvokedCapability< typeof AggregatorCaps.pieceOffer > @@ -610,6 +628,7 @@ export type AbilitiesArray = [ FilecoinOffer['can'], FilecoinSubmit['can'], FilecoinAccept['can'], + FilecoinInfo['can'], PieceOffer['can'], PieceAccept['can'], AggregateOffer['can'], diff --git a/packages/filecoin-api/src/storefront/api.ts b/packages/filecoin-api/src/storefront/api.ts index 3ffd7da56..11d876b0d 100644 --- a/packages/filecoin-api/src/storefront/api.ts +++ b/packages/filecoin-api/src/storefront/api.ts @@ -10,6 +10,7 @@ import { PieceLink } from '@web3-storage/data-segment' import { AggregatorService, StorefrontService, + DealTrackerService, } from '@web3-storage/filecoin-client/types' import { Store, @@ -64,6 +65,10 @@ export interface ServiceContext { * Stores receipts for tasks. */ receiptStore: ReceiptStore + /** + * Deal tracker connection to find out available deals for an aggregate. + */ + dealTrackerService: ServiceConfig /** * Service options. */ diff --git a/packages/filecoin-api/src/storefront/service.js b/packages/filecoin-api/src/storefront/service.js index 8d756c7ab..5f93a6177 100644 --- a/packages/filecoin-api/src/storefront/service.js +++ b/packages/filecoin-api/src/storefront/service.js @@ -3,9 +3,14 @@ import * as Client from '@ucanto/client' import * as CAR from '@ucanto/transport/car' import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import { DealTracker } from '@web3-storage/filecoin-client' // eslint-disable-next-line no-unused-vars import * as API from '../types.js' -import { QueueOperationFailed, StoreOperationFailed } from '../errors.js' +import { + QueueOperationFailed, + StoreOperationFailed, + RecordNotFoundErrorName, +} from '../errors.js' /** * @param {API.Input} input @@ -226,6 +231,94 @@ async function findDataAggregationProof({ taskStore, receiptStore }, task) { } } +/** + * @param {API.Input} input + * @param {import('./api.js').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const filecoinInfo = async ({ capability }, context) => { + const { piece } = capability.nb + + // Get piece in store + const getPiece = await context.pieceStore.get({ piece }) + if (getPiece.error && getPiece.error.name === RecordNotFoundErrorName) { + return { + error: getPiece.error, + } + } else if (getPiece.error) { + return { error: new StoreOperationFailed(getPiece.error.message) } + } + + // Check if `piece/accept` receipt exists to get to know aggregate where it is included on a deal + const pieceAcceptInvocation = await StorefrontCaps.filecoinAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + content: getPiece.ok.content, + }, + expiration: Infinity, + }) + .delegate() + + const pieceAcceptReceiptGet = await context.receiptStore.get( + pieceAcceptInvocation.link() + ) + if (pieceAcceptReceiptGet.error) { + /** @type {API.UcantoInterface.OkBuilder} */ + const processingResult = Server.ok({ + piece, + deals: [], + }) + return processingResult + } + + const pieceAcceptOut = /** @type {API.FilecoinAcceptSuccess} */ ( + pieceAcceptReceiptGet.ok?.out.ok + ) + + // Query current info of aggregate from deal tracker + const info = await DealTracker.dealInfo( + context.dealTrackerService.invocationConfig, + pieceAcceptOut.aggregate, + { connection: context.dealTrackerService.connection } + ) + + if (info.out.error) { + return { + error: info.out.error, + } + } + const deals = Object.entries(info.out.ok.deals || {}) + if (!deals.length) { + // Should not happen if there is `piece/accept` receipt + return { + error: new Server.Failure( + `no deals were obtained for aggregate ${pieceAcceptOut.aggregate} where piece ${piece} is included` + ), + } + } + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + piece, + deals: deals.map(([dealId, dealDetails]) => ({ + aggregate: pieceAcceptOut.aggregate, + provider: dealDetails.provider, + inclusion: pieceAcceptOut.inclusion, + aux: { + dataType: 0n, + dataSource: { + dealID: BigInt(dealId), + }, + }, + })), + }) + return result +} + export const ProofNotFoundName = /** @type {const} */ ('ProofNotFound') export class ProofNotFound extends Server.Failure { get reason() { @@ -255,6 +348,10 @@ export function createService(context) { capability: StorefrontCaps.filecoinAccept, handler: (input) => filecoinAccept(input, context), }), + info: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinInfo, + handler: (input) => filecoinInfo(input, context), + }), }, } } diff --git a/packages/filecoin-api/test/context/receipts.js b/packages/filecoin-api/test/context/receipts.js index 6f5f859ed..1d3176562 100644 --- a/packages/filecoin-api/test/context/receipts.js +++ b/packages/filecoin-api/test/context/receipts.js @@ -1,4 +1,5 @@ import { Receipt } from '@ucanto/core' +import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' @@ -12,6 +13,7 @@ import * as API from '../../src/types.js' * @param {API.PieceLink} context.aggregate * @param {string} context.group * @param {API.PieceLink} context.piece + * @param {API.CARLink} context.content * @param {import('@ucanto/interface').Block} context.piecesBlock * @param {API.InclusionProof} context.inclusionProof * @param {API.AggregateAcceptSuccess} context.aggregateAcceptStatus @@ -23,10 +25,47 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ aggregate, group, piece, + content, piecesBlock, inclusionProof, aggregateAcceptStatus, }) { + const filecoinOfferInvocation = await StorefrontCaps.filecoinOffer + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() + const filecoinSubmitInvocation = await StorefrontCaps.filecoinSubmit + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() + const filecoinAcceptInvocation = await StorefrontCaps.filecoinAccept + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() const pieceOfferInvocation = await AggregatorCaps.pieceOffer .invoke({ issuer: storefront, @@ -76,6 +115,55 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ expiration: Infinity, }) .delegate() + + // Receipts + const filecoinOfferReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinOfferInvocation.cid, + result: { + ok: /** @type {API.FilecoinOfferSuccess} */ ({ + piece, + }), + }, + fx: { + join: filecoinAcceptInvocation.cid, + fork: [filecoinSubmitInvocation.cid], + }, + }) + + const filecoinSubmitReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinSubmitInvocation.cid, + result: { + ok: /** @type {API.FilecoinSubmitSuccess} */ ({ + piece, + }), + }, + fx: { + join: pieceOfferInvocation.cid, + fork: [], + }, + }) + + const filecoinAcceptReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinAcceptInvocation.cid, + result: { + ok: /** @type {API.FilecoinAcceptSuccess} */ ({ + piece, + aggregate, + inclusion: inclusionProof, + aux: { + ...aggregateAcceptStatus, + }, + }), + }, + fx: { + join: undefined, + fork: [], + }, + }) + const pieceOfferReceipt = await Receipt.issue({ issuer: aggregator, ran: pieceOfferInvocation.cid, @@ -130,12 +218,18 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ return { invocations: { + filecoinOfferInvocation, + filecoinSubmitInvocation, + filecoinAcceptInvocation, pieceOfferInvocation, pieceAcceptInvocation, aggregateOfferInvocation, aggregateAcceptInvocation, }, receipts: { + filecoinOfferReceipt, + filecoinSubmitReceipt, + filecoinAcceptReceipt, pieceOfferReceipt, pieceAcceptReceipt, aggregateOfferReceipt, diff --git a/packages/filecoin-api/test/context/store.js b/packages/filecoin-api/test/context/store.js index 51d19b1fa..02f98e61e 100644 --- a/packages/filecoin-api/test/context/store.js +++ b/packages/filecoin-api/test/context/store.js @@ -1,5 +1,5 @@ import * as API from '../../src/types.js' -import { RecordNotFound, StoreOperationFailed } from '../../src/errors.js' +import { StoreOperationFailed, RecordNotFound } from '../../src/errors.js' /** * @typedef {import('../../src/types.js').StorePutError} StorePutError @@ -47,7 +47,7 @@ export class Store { const t = this.getFn(this.items, item) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { @@ -85,7 +85,7 @@ export class Store { const t = this.queryFn(this.items, search) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { @@ -123,7 +123,7 @@ export class UpdatableStore extends Store { const t = this.updateFn(this.items, key, item) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { diff --git a/packages/filecoin-api/test/events/storefront.js b/packages/filecoin-api/test/events/storefront.js index 9de711d01..ceadee9c8 100644 --- a/packages/filecoin-api/test/events/storefront.js +++ b/packages/filecoin-api/test/events/storefront.js @@ -365,6 +365,7 @@ export const test = { aggregate: aggregate.link, group, piece: piece.link, + content: piece.content, piecesBlock, inclusionProof: { subtree: inclusionProof.ok[0], diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js index 928deac62..fdc9c7c24 100644 --- a/packages/filecoin-api/test/services/storefront.js +++ b/packages/filecoin-api/test/services/storefront.js @@ -1,6 +1,8 @@ import { Filecoin, Aggregator } from '@web3-storage/capabilities' +import * as Server from '@ucanto/server' import { CBOR } from '@ucanto/core' import * as Signer from '@ucanto/principal/ed25519' +import * as DealTrackerCaps from '@web3-storage/capabilities/filecoin/deal-tracker' import pWaitFor from 'p-wait-for' import * as API from '../../src/types.js' @@ -16,6 +18,8 @@ import { createInvocationsAndReceiptsForDealDataProofChain } from '../context/re import { getStoreImplementations } from '../context/store-implementations.js' import { FailingStore } from '../context/store.js' import { FailingQueue } from '../context/queue.js' +import { mockService } from '../context/mocks.js' +import { getConnection } from '../context/service.js' /** * @typedef {import('../../src/storefront/api.js').PieceRecord} PieceRecord @@ -200,7 +204,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, QueueOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, filecoinSubmitQueue: new FailingQueue(), }) @@ -232,7 +236,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, StoreOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, }) @@ -306,7 +310,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, QueueOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceOfferQueue: new FailingQueue(), }) @@ -363,6 +367,7 @@ export const test = { aggregate: aggregate.link, group, piece: piece.link, + content: piece.content, piecesBlock, inclusionProof: { subtree: inclusionProof.ok[0], @@ -447,11 +452,176 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, StoreOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, }) ), + 'filecoin/info gets aggregate where piece was included together with deals and inclusion proof': + wichMockableContext( + async (assert, context) => { + const { agent, aggregator, dealer } = await getServiceContext() + const group = context.id.did() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { aggregate, pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: piece.link.link(), + content: piece.content.link(), + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // Create inclusion proof for test + const inclusionProof = aggregate.resolveProof(piece.link) + if (inclusionProof.error) { + throw new Error('could not compute inclusion proof') + } + + // Create invocations and receipts for chain into DealDataProof + const dealMetadata = { + dataType: 0n, + dataSource: { + dealID: 111n, + }, + } + const { invocations, receipts } = + await createInvocationsAndReceiptsForDealDataProofChain({ + storefront: context.id, + aggregator, + dealer, + aggregate: aggregate.link, + group, + piece: piece.link, + content: piece.content, + piecesBlock, + inclusionProof: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + aggregateAcceptStatus: { + ...dealMetadata, + aggregate: aggregate.link, + }, + }) + + const storedInvocationsAndReceiptsRes = + await storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storedInvocationsAndReceiptsRes.ok) + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link.link(), + }, + }) + + const response = await filecoinInfoInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(piece.link.link())) + assert.equal(response.out.ok.deals.length, 1) + assert.ok(response.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataType), + dealMetadata.dataType + ) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataSource.dealID), + dealMetadata.dataSource.dealID + ) + assert.ok(response.out.ok.deals[0].inclusion.index) + assert.ok(response.out.ok.deals[0].inclusion.subtree) + }, + async (context) => { + /** + * Mock deal tracker to return deals + */ + const dealTrackerSigner = await Signer.generate() + const service = mockService({ + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async () => { + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + deals: { + 111: { + provider: 'f11111', + }, + }, + }) + + return result + }, + }), + }, + }) + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + return { + ...context, + service, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: dealTrackerSigner, + }, + }, + } + } + ), + 'filecoin/info fails if content is not known': async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link, + }, + }) + + const response = await filecoinInfoInv.execute(connection) + assert.ok(response.out.error) + }, } /** @@ -501,12 +671,12 @@ async function getServiceContext() { /** * @param {API.Test} testFn - * @param {(context: StorefrontApi.ServiceContext) => StorefrontApi.ServiceContext} mockContextFunction + * @param {(context: StorefrontApi.ServiceContext) => Promise} mockContextFunction */ function wichMockableContext(testFn, mockContextFunction) { // @ts-ignore - return function (...args) { - const modifiedArgs = [args[0], mockContextFunction(args[1])] + return async function (...args) { + const modifiedArgs = [args[0], await mockContextFunction(args[1])] // @ts-ignore return testFn(...modifiedArgs) } diff --git a/packages/filecoin-api/test/storefront.spec.js b/packages/filecoin-api/test/storefront.spec.js index 8bd534a00..677afe425 100644 --- a/packages/filecoin-api/test/storefront.spec.js +++ b/packages/filecoin-api/test/storefront.spec.js @@ -24,6 +24,7 @@ describe('storefront', () => { define(name, async () => { const storefrontSigner = await Signer.generate() const aggregatorSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() // resources /** @type {Map} */ @@ -34,6 +35,11 @@ describe('storefront', () => { const { storefront: { pieceStore, receiptStore, taskStore }, } = getStoreImplementations() + const service = getMockService() + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection await test( { @@ -54,6 +60,14 @@ describe('storefront', () => { pieceOfferQueue, taskStore, receiptStore, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: storefrontSigner, + with: storefrontSigner.did(), + audience: dealTrackerSigner, + }, + }, queuedMessages, validateAuthorization, } diff --git a/packages/filecoin-client/src/storefront.js b/packages/filecoin-client/src/storefront.js index 917f11b2a..06cd36555 100644 --- a/packages/filecoin-client/src/storefront.js +++ b/packages/filecoin-client/src/storefront.js @@ -147,3 +147,33 @@ export async function filecoinAccept( return await invocation.execute(conn) } + +/** + * The `filecoin/info` task can be executed to request info about a content piece + * in Filecoin. It issues a signed receipt of the execution result. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('./types.js').RequestOptions} [options] + */ +export async function filecoinInfo( + { issuer, with: resource, proofs, audience }, + piece, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = Storefront.filecoinInfo.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STOREFRONT.principal, + with: resource, + nb: { + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/types.ts b/packages/filecoin-client/src/types.ts index 762c522ea..752ae4711 100644 --- a/packages/filecoin-client/src/types.ts +++ b/packages/filecoin-client/src/types.ts @@ -16,6 +16,9 @@ import { FilecoinAccept, FilecoinAcceptSuccess, FilecoinAcceptFailure, + FilecoinInfo, + FilecoinInfoSuccess, + FilecoinInfoFailure, PieceOffer, PieceOfferSuccess, PieceOfferFailure, @@ -75,6 +78,7 @@ export interface StorefrontService { FilecoinAcceptSuccess, FilecoinAcceptFailure > + info: ServiceMethod } } diff --git a/packages/filecoin-client/test/helpers/mocks.js b/packages/filecoin-client/test/helpers/mocks.js index 3b4fb52c4..ebe42207c 100644 --- a/packages/filecoin-client/test/helpers/mocks.js +++ b/packages/filecoin-client/test/helpers/mocks.js @@ -18,6 +18,7 @@ export function mockService(impl) { offer: withCallCount(impl.filecoin?.offer ?? notImplemented), submit: withCallCount(impl.filecoin?.submit ?? notImplemented), accept: withCallCount(impl.filecoin?.accept ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), }, piece: { offer: withCallCount(impl.piece?.offer ?? notImplemented), diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js index acb00d235..557ed0507 100644 --- a/packages/filecoin-client/test/storefront.test.js +++ b/packages/filecoin-client/test/storefront.test.js @@ -9,6 +9,7 @@ import { filecoinOffer, filecoinSubmit, filecoinAccept, + filecoinInfo, } from '../src/storefront.js' import { randomAggregate, randomCargo } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -261,6 +262,55 @@ describe('storefront', () => { // does not include effect fx in receipt assert.ok(!res.fx.join) }) + + it('agent asks info of a filecoin piece', async () => { + const { agent } = await getContext() + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinOfferSuccess} */ + const filecoinOfferResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + info: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinInfo, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), agent.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, StorefrontCaps.filecoinInfo.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + const { piece } = invCap.nb + // piece link + assert.ok(piece.equals(cargo.link.link())) + + return Server.ok({ + piece, + deals: [], + }) + }, + }), + }, + }) + + const res = await filecoinInfo( + { + issuer: agent, + with: agent.did(), + audience: storefrontService, + }, + cargo.link, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.ok(res.out.ok.piece.equals(filecoinOfferResponse.piece)) + assert.deepEqual(res.out.ok.deals, []) + }) }) async function getContext() { diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index acefa960b..313f4584b 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -1,5 +1,7 @@ import * as Signer from '@ucanto/principal/ed25519' import { + getConnection, + getMockService, getStoreImplementations, getQueueImplementations, } from '@web3-storage/filecoin-api/test/context/service' @@ -41,8 +43,15 @@ export const createContext = async ( const usageStorage = new UsageStorage(storeTable) const signer = await Signer.generate() const aggregatorSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() const id = signer.withDID('did:web:test.web3.storage') + const service = getMockService() + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + /** @type {Map} */ const queuedMessages = new Map() const { @@ -86,6 +95,14 @@ export const createContext = async ( receiptStore, taskStore, requirePaymentPlan, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: id, + with: id.did(), + audience: dealTrackerSigner, + }, + }, ...createRevocationChecker({ revocationsStorage }), } diff --git a/packages/w3up-client/README.md b/packages/w3up-client/README.md index d823a3c98..699e60f8f 100644 --- a/packages/w3up-client/README.md +++ b/packages/w3up-client/README.md @@ -406,6 +406,8 @@ sequenceDiagram - [`capability.upload.add`](#capabilityuploadadd) - [`capability.upload.list`](#capabilityuploadlist) - [`capability.upload.remove`](#capabilityuploadremove) + - [`capability.filecoin.offer`](#capabilityfilecoinoffer) + - [`capability.filecoin.info`](#capabilityfilecoininfo) - [Types](#types) - [`Capability`](#capability) - [`CARMetadata`](#carmetadata) @@ -688,6 +690,27 @@ function remove( Remove a upload by root data CID. +### `capability.filecoin.offer` + +```ts +function offer ( + content: CID, + piece: PieceLink, +): Promise +``` + +Offer a Filecoin "piece" to be added to an aggregate that will be offered for Filecoin deal(s). + +### `capability.filecoin.info` + +```ts +function info ( + piece: PieceLink +): Promise +``` + +Get know deals and aggregate info of a Filecoin "piece" previously offered. + ## Types ### `Capability` diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index 616ab88e9..705045e15 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -80,9 +80,10 @@ "@ucanto/interface": "^9.0.0", "@ucanto/principal": "^9.0.0", "@ucanto/transport": "^9.0.0", - "@web3-storage/did-mailto": "workspace:^", "@web3-storage/access": "workspace:^", "@web3-storage/capabilities": "workspace:^", + "@web3-storage/did-mailto": "workspace:^", + "@web3-storage/filecoin-client": "workspace:^", "@web3-storage/upload-client": "workspace:^" }, "devDependencies": { @@ -93,6 +94,7 @@ "@types/mocha": "^10.0.1", "@types/node": "^20.8.4", "@ucanto/server": "^9.0.1", + "@web3-storage/data-segment": "^5.0.0", "@web3-storage/eslint-config-w3up": "workspace:^", "assert": "^2.0.0", "c8": "^7.13.0", diff --git a/packages/w3up-client/src/capability/filecoin.js b/packages/w3up-client/src/capability/filecoin.js new file mode 100644 index 000000000..9a5c8e8d1 --- /dev/null +++ b/packages/w3up-client/src/capability/filecoin.js @@ -0,0 +1,33 @@ +import { Storefront } from '@web3-storage/filecoin-client' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' +import { Base } from '../base.js' + +/** + * Client for interacting with the `filecoin/*` capabilities. + */ +export class FilecoinClient extends Base { + /** + * Offer a Filecoin "piece" to the resource. + * + * @param {import('multiformats').UnknownLink} content + * @param {import('@web3-storage/capabilities/types').PieceLink} piece + */ + async offer(content, piece) { + const conf = await this._invocationConfig([FilecoinCapabilities.offer.can]) + return Storefront.filecoinOffer(conf, content, piece, { + connection: this._serviceConf.filecoin, + }) + } + + /** + * Request info about a content piece in Filecoin deals + * + * @param {import('@web3-storage/capabilities/types').PieceLink} piece + */ + async info(piece) { + const conf = await this._invocationConfig([FilecoinCapabilities.info.can]) + return Storefront.filecoinInfo(conf, piece, { + connection: this._serviceConf.filecoin, + }) + } +} diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index 016719d76..f2662dd59 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -14,6 +14,7 @@ import { StoreClient } from './capability/store.js' import { UploadClient } from './capability/upload.js' import { SpaceClient } from './capability/space.js' import { AccessClient } from './capability/access.js' +import { FilecoinClient } from './capability/filecoin.js' export * as Access from './capability/access.js' export { StoreClient, UploadClient, SpaceClient, AccessClient } @@ -31,6 +32,7 @@ export class Client extends Base { store: new StoreClient(agentData, options), upload: new UploadClient(agentData, options), space: new SpaceClient(agentData, options), + filecoin: new FilecoinClient(agentData, options), } } diff --git a/packages/w3up-client/src/service.js b/packages/w3up-client/src/service.js index 26d00daba..20bb09365 100644 --- a/packages/w3up-client/src/service.js +++ b/packages/w3up-client/src/service.js @@ -26,8 +26,21 @@ export const uploadServiceConnection = connect({ }), }) +export const filecoinServiceURL = new URL('https://up.web3.storage') +export const filecoinServicePrincipal = DID.parse('did:web:web3.storage') + +export const filecoinServiceConnection = connect({ + id: filecoinServicePrincipal, + codec: CAR.outbound, + channel: HTTP.open({ + url: filecoinServiceURL, + method: 'POST', + }), +}) + /** @type {import('./types.js').ServiceConf} */ export const serviceConf = { access: accessServiceConnection, upload: uploadServiceConnection, + filecoin: filecoinServiceConnection, } diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 5201c7109..c2e8bbe28 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -13,6 +13,7 @@ import type { Unit, } from '@ucanto/interface' import { type Client } from './client.js' +import { StorefrontService } from '@web3-storage/filecoin-client/storefront' export * from '@ucanto/interface' export * from '@web3-storage/did-mailto' export type { Agent, CapabilityQuery } from '@web3-storage/access/agent' @@ -28,6 +29,7 @@ export type ProofQuery = Record> export interface ServiceConf { access: ConnectionView upload: ConnectionView + filecoin: ConnectionView } export interface ClientFactoryOptions { diff --git a/packages/w3up-client/test/capability/filecoin.test.js b/packages/w3up-client/test/capability/filecoin.test.js new file mode 100644 index 000000000..53427c4fc --- /dev/null +++ b/packages/w3up-client/test/capability/filecoin.test.js @@ -0,0 +1,162 @@ +import assert from 'assert' +import { + create as createServer, + provide, + provideAdvanced, + ok, +} from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as Signer from '@ucanto/principal/ed25519' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' +import { AgentData } from '@web3-storage/access/agent' + +import { randomAggregate, randomCargo } from '../helpers/random.js' +import { mockService, mockServiceConf } from '../helpers/mocks.js' +import { Client } from '../../src/client.js' +import { validateAuthorization } from '../helpers/utils.js' + +describe('FilecoinClient', () => { + describe('offer', () => { + it('should send an offer', async () => { + const service = mockService({ + filecoin: { + offer: provideAdvanced({ + capability: FilecoinCapabilities.offer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + assert.ok(invCap.nb) + + // Create effect for receipt with self signed queued operation + const submitfx = await FilecoinCapabilities.submit + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + expiration: Infinity, + }) + .delegate() + + const acceptfx = await FilecoinCapabilities.accept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + expiration: Infinity, + }) + .delegate() + + return ok({ + piece: invCap.nb.piece, + }) + .fork(submitfx.link()) + .join(acceptfx.link()) + }, + }), + }, + }) + const server = createServer({ + id: await Signer.generate(), + service, + codec: CAR.inbound, + validateAuthorization, + }) + + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: await mockServiceConf(server), + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + const [cargo] = await randomCargo(1, 100) + const res = await alice.capability.filecoin.offer( + cargo.content, + cargo.link + ) + + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) + assert(res.out.ok) + assert(res.out.ok.piece.equals(cargo.link)) + assert(res.fx.join) + assert(res.fx.fork.length) + }) + }) + describe('info', () => { + it('should get piece info', async () => { + const { pieces, aggregate } = await randomAggregate(10, 100) + const cargo = pieces[0] + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(cargo.link) + if (proof.error) { + throw new Error('could not compute proof') + } + /** @type {import('@web3-storage/capabilities/types').FilecoinInfoSuccess} */ + const filecoinAcceptResponse = { + piece: cargo.link, + deals: [ + { + aggregate: aggregate.link, + provider: 'f1111', + inclusion: { + subtree: proof.ok[0], + index: proof.ok[1], + }, + aux: { + dataType: 0n, + dataSource: { + dealID: 1138n, + }, + }, + }, + ], + } + const service = mockService({ + filecoin: { + info: provide(FilecoinCapabilities.info, ({ invocation }) => { + const invCap = invocation.capabilities[0] + assert.ok(invCap.nb) + + return ok(filecoinAcceptResponse) + }), + }, + }) + const server = createServer({ + id: await Signer.generate(), + service, + codec: CAR.inbound, + validateAuthorization, + }) + + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: await mockServiceConf(server), + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + const res = await alice.capability.filecoin.info(cargo.link) + + assert(service.filecoin.info.called) + assert.equal(service.filecoin.info.callCount, 1) + assert(res.out.ok) + assert(res.out.ok.piece.equals(cargo.link)) + assert.equal(res.out.ok.deals.length, 1) + assert(res.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert(res.out.ok.deals[0].aux.dataSource.dealID) + assert(res.out.ok.deals[0].provider) + assert.deepEqual(res.out.ok.deals[0].inclusion, { + subtree: proof.ok[0], + index: proof.ok[1], + }) + }) + }) +}) diff --git a/packages/w3up-client/test/helpers/car.js b/packages/w3up-client/test/helpers/car.js index 031b85f10..07aa974db 100644 --- a/packages/w3up-client/test/helpers/car.js +++ b/packages/w3up-client/test/helpers/car.js @@ -22,5 +22,5 @@ export async function toCAR(bytes) { const blob = new Blob(chunks) const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - return Object.assign(blob, { cid, roots: [root] }) + return Object.assign(blob, { cid, roots: [root], bytes }) } diff --git a/packages/w3up-client/test/helpers/mocks.js b/packages/w3up-client/test/helpers/mocks.js index 7f98a3544..89fcbed1d 100644 --- a/packages/w3up-client/test/helpers/mocks.js +++ b/packages/w3up-client/test/helpers/mocks.js @@ -14,6 +14,7 @@ const notImplemented = () => { * upload: Partial * space: Partial * ucan: Partial + * filecoin: Partial * }>} impl */ export function mockService(impl) { @@ -42,6 +43,10 @@ export function mockService(impl) { ucan: { revoke: withCallCount(impl.ucan?.revoke ?? notImplemented), }, + filecoin: { + offer: withCallCount(impl.filecoin?.offer ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), + }, } } @@ -72,5 +77,5 @@ export async function mockServiceConf(server) { codec: CAR.outbound, channel: server, }) - return { access: connection, upload: connection } + return { access: connection, upload: connection, filecoin: connection } } diff --git a/packages/w3up-client/test/helpers/random.js b/packages/w3up-client/test/helpers/random.js index b7e8aedb4..b1f512707 100644 --- a/packages/w3up-client/test/helpers/random.js +++ b/packages/w3up-client/test/helpers/random.js @@ -1,3 +1,4 @@ +import { Aggregate, Piece } from '@web3-storage/data-segment' import { toCAR } from './car.js' /** @param {number} size */ @@ -29,3 +30,42 @@ export async function randomCAR(size) { const bytes = await randomBytes(size) return toCAR(bytes) } + +/** + * @param {number} length + * @param {number} size + */ +export async function randomCargo(length, size) { + const cars = await Promise.all( + Array.from({ length }).map(() => randomCAR(size)) + ) + + return cars.map((car) => { + const piece = Piece.fromPayload(car.bytes) + + return { + link: piece.link, + height: piece.height, + root: piece.root, + padding: piece.padding, + content: car.cid, + } + }) +} + +/** + * @param {number} length + * @param {number} size + */ +export async function randomAggregate(length, size) { + const pieces = await randomCargo(length, size) + + const aggregateBuild = Aggregate.build({ + pieces, + }) + + return { + pieces, + aggregate: aggregateBuild, + } +} diff --git a/packages/w3up-client/test/test.js b/packages/w3up-client/test/test.js index 5ffe569ae..7bf1f9105 100644 --- a/packages/w3up-client/test/test.js +++ b/packages/w3up-client/test/test.js @@ -42,6 +42,7 @@ export const setup = async () => { serviceConf: { access: context.connection, upload: context.connection, + filecoin: context.connection, }, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 577ff38e5..f6f92e6dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -552,6 +552,9 @@ importers: '@web3-storage/did-mailto': specifier: workspace:^ version: link:../did-mailto + '@web3-storage/filecoin-client': + specifier: workspace:^ + version: link:../filecoin-client '@web3-storage/upload-client': specifier: workspace:^ version: link:../upload-client @@ -574,6 +577,9 @@ importers: '@ucanto/server': specifier: ^9.0.1 version: 9.0.1 + '@web3-storage/data-segment': + specifier: ^5.0.0 + version: 5.0.0 '@web3-storage/eslint-config-w3up': specifier: workspace:^ version: link:../eslint-config-w3up @@ -3567,6 +3573,14 @@ packages: multiformats: 11.0.2 sync-multihash-sha2: 1.0.0 + /@web3-storage/data-segment@5.0.0: + resolution: {integrity: sha512-5CbElsxec2DsKhEHEh3XRGISAyna+bCjKjjvFrLcYyXLCaiSt/nF3ypcllxwjpE4newMUArymGKGzzZnRWL2kg==} + dependencies: + '@ipld/dag-cbor': 9.0.6 + multiformats: 11.0.2 + sync-multihash-sha2: 1.0.0 + dev: true + /@web3-storage/sigv4@1.0.2: resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==} dependencies: