diff --git a/packages/capabilities/src/filecoin/index.js b/packages/capabilities/src/filecoin/index.js index d7675e0fd..aee774398 100644 --- a/packages/capabilities/src/filecoin/index.js +++ b/packages/capabilities/src/filecoin/index.js @@ -16,4 +16,5 @@ export { filecoinOffer as offer, filecoinSubmit as submit, filecoinAccept as accept, + filecoinInfo as info, } from './storefront.js' diff --git a/packages/capabilities/src/filecoin/storefront.js b/packages/capabilities/src/filecoin/storefront.js index 88b103690..53f0cc4b3 100644 --- a/packages/capabilities/src/filecoin/storefront.js +++ b/packages/capabilities/src/filecoin/storefront.js @@ -106,3 +106,30 @@ export const filecoinAccept = capability({ ) }, }) + +/** + * Capability allowing an agent to _request_ info about a content piece in + * Filecoin deals. + */ +export const filecoinInfo = capability({ + can: 'filecoin/info', + /** + * DID of the space the content is stored in. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the piece. + * + * @see https://github.com/filecoin-project/FIPs/pull/758/files + */ + piece: PieceLink, + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 7c44a620c..5d814cda3 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -73,6 +73,7 @@ export const abilitiesAsStrings = [ Storefront.filecoinOffer.can, Storefront.filecoinSubmit.can, Storefront.filecoinAccept.can, + Storefront.filecoinInfo.can, Aggregator.pieceOffer.can, Aggregator.pieceAccept.can, Dealer.aggregateOffer.can, diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index c72d98830..1038fed0b 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -305,6 +305,21 @@ export interface ProofNotFound extends Ucanto.Failure { name: 'ProofNotFound' } +export interface FilecoinInfoSuccess { + piece: PieceLink + deals: FilecoinInfoAcceptedDeal[] +} +export interface FilecoinInfoAcceptedDeal + extends DataAggregationProof, + DealDetails { + aggregate: PieceLink +} + +export type FilecoinInfoFailure = + | ContentNotFound + | InvalidContentPiece + | Ucanto.Failure + // filecoin aggregator export interface PieceOfferSuccess { /** @@ -549,6 +564,9 @@ export type FilecoinSubmit = InferInvokedCapability< export type FilecoinAccept = InferInvokedCapability< typeof StorefrontCaps.filecoinAccept > +export type FilecoinInfo = InferInvokedCapability< + typeof StorefrontCaps.filecoinInfo +> export type PieceOffer = InferInvokedCapability< typeof AggregatorCaps.pieceOffer > @@ -610,6 +628,7 @@ export type AbilitiesArray = [ FilecoinOffer['can'], FilecoinSubmit['can'], FilecoinAccept['can'], + FilecoinInfo['can'], PieceOffer['can'], PieceAccept['can'], AggregateOffer['can'], diff --git a/packages/filecoin-api/src/storefront/api.ts b/packages/filecoin-api/src/storefront/api.ts index 3ffd7da56..11d876b0d 100644 --- a/packages/filecoin-api/src/storefront/api.ts +++ b/packages/filecoin-api/src/storefront/api.ts @@ -10,6 +10,7 @@ import { PieceLink } from '@web3-storage/data-segment' import { AggregatorService, StorefrontService, + DealTrackerService, } from '@web3-storage/filecoin-client/types' import { Store, @@ -64,6 +65,10 @@ export interface ServiceContext { * Stores receipts for tasks. */ receiptStore: ReceiptStore + /** + * Deal tracker connection to find out available deals for an aggregate. + */ + dealTrackerService: ServiceConfig /** * Service options. */ diff --git a/packages/filecoin-api/src/storefront/service.js b/packages/filecoin-api/src/storefront/service.js index 8d756c7ab..5f93a6177 100644 --- a/packages/filecoin-api/src/storefront/service.js +++ b/packages/filecoin-api/src/storefront/service.js @@ -3,9 +3,14 @@ import * as Client from '@ucanto/client' import * as CAR from '@ucanto/transport/car' import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import { DealTracker } from '@web3-storage/filecoin-client' // eslint-disable-next-line no-unused-vars import * as API from '../types.js' -import { QueueOperationFailed, StoreOperationFailed } from '../errors.js' +import { + QueueOperationFailed, + StoreOperationFailed, + RecordNotFoundErrorName, +} from '../errors.js' /** * @param {API.Input} input @@ -226,6 +231,94 @@ async function findDataAggregationProof({ taskStore, receiptStore }, task) { } } +/** + * @param {API.Input} input + * @param {import('./api.js').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const filecoinInfo = async ({ capability }, context) => { + const { piece } = capability.nb + + // Get piece in store + const getPiece = await context.pieceStore.get({ piece }) + if (getPiece.error && getPiece.error.name === RecordNotFoundErrorName) { + return { + error: getPiece.error, + } + } else if (getPiece.error) { + return { error: new StoreOperationFailed(getPiece.error.message) } + } + + // Check if `piece/accept` receipt exists to get to know aggregate where it is included on a deal + const pieceAcceptInvocation = await StorefrontCaps.filecoinAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + content: getPiece.ok.content, + }, + expiration: Infinity, + }) + .delegate() + + const pieceAcceptReceiptGet = await context.receiptStore.get( + pieceAcceptInvocation.link() + ) + if (pieceAcceptReceiptGet.error) { + /** @type {API.UcantoInterface.OkBuilder} */ + const processingResult = Server.ok({ + piece, + deals: [], + }) + return processingResult + } + + const pieceAcceptOut = /** @type {API.FilecoinAcceptSuccess} */ ( + pieceAcceptReceiptGet.ok?.out.ok + ) + + // Query current info of aggregate from deal tracker + const info = await DealTracker.dealInfo( + context.dealTrackerService.invocationConfig, + pieceAcceptOut.aggregate, + { connection: context.dealTrackerService.connection } + ) + + if (info.out.error) { + return { + error: info.out.error, + } + } + const deals = Object.entries(info.out.ok.deals || {}) + if (!deals.length) { + // Should not happen if there is `piece/accept` receipt + return { + error: new Server.Failure( + `no deals were obtained for aggregate ${pieceAcceptOut.aggregate} where piece ${piece} is included` + ), + } + } + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + piece, + deals: deals.map(([dealId, dealDetails]) => ({ + aggregate: pieceAcceptOut.aggregate, + provider: dealDetails.provider, + inclusion: pieceAcceptOut.inclusion, + aux: { + dataType: 0n, + dataSource: { + dealID: BigInt(dealId), + }, + }, + })), + }) + return result +} + export const ProofNotFoundName = /** @type {const} */ ('ProofNotFound') export class ProofNotFound extends Server.Failure { get reason() { @@ -255,6 +348,10 @@ export function createService(context) { capability: StorefrontCaps.filecoinAccept, handler: (input) => filecoinAccept(input, context), }), + info: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinInfo, + handler: (input) => filecoinInfo(input, context), + }), }, } } diff --git a/packages/filecoin-api/test/context/receipts.js b/packages/filecoin-api/test/context/receipts.js index 6f5f859ed..1d3176562 100644 --- a/packages/filecoin-api/test/context/receipts.js +++ b/packages/filecoin-api/test/context/receipts.js @@ -1,4 +1,5 @@ import { Receipt } from '@ucanto/core' +import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' @@ -12,6 +13,7 @@ import * as API from '../../src/types.js' * @param {API.PieceLink} context.aggregate * @param {string} context.group * @param {API.PieceLink} context.piece + * @param {API.CARLink} context.content * @param {import('@ucanto/interface').Block} context.piecesBlock * @param {API.InclusionProof} context.inclusionProof * @param {API.AggregateAcceptSuccess} context.aggregateAcceptStatus @@ -23,10 +25,47 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ aggregate, group, piece, + content, piecesBlock, inclusionProof, aggregateAcceptStatus, }) { + const filecoinOfferInvocation = await StorefrontCaps.filecoinOffer + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() + const filecoinSubmitInvocation = await StorefrontCaps.filecoinSubmit + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() + const filecoinAcceptInvocation = await StorefrontCaps.filecoinAccept + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() const pieceOfferInvocation = await AggregatorCaps.pieceOffer .invoke({ issuer: storefront, @@ -76,6 +115,55 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ expiration: Infinity, }) .delegate() + + // Receipts + const filecoinOfferReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinOfferInvocation.cid, + result: { + ok: /** @type {API.FilecoinOfferSuccess} */ ({ + piece, + }), + }, + fx: { + join: filecoinAcceptInvocation.cid, + fork: [filecoinSubmitInvocation.cid], + }, + }) + + const filecoinSubmitReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinSubmitInvocation.cid, + result: { + ok: /** @type {API.FilecoinSubmitSuccess} */ ({ + piece, + }), + }, + fx: { + join: pieceOfferInvocation.cid, + fork: [], + }, + }) + + const filecoinAcceptReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinAcceptInvocation.cid, + result: { + ok: /** @type {API.FilecoinAcceptSuccess} */ ({ + piece, + aggregate, + inclusion: inclusionProof, + aux: { + ...aggregateAcceptStatus, + }, + }), + }, + fx: { + join: undefined, + fork: [], + }, + }) + const pieceOfferReceipt = await Receipt.issue({ issuer: aggregator, ran: pieceOfferInvocation.cid, @@ -130,12 +218,18 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ return { invocations: { + filecoinOfferInvocation, + filecoinSubmitInvocation, + filecoinAcceptInvocation, pieceOfferInvocation, pieceAcceptInvocation, aggregateOfferInvocation, aggregateAcceptInvocation, }, receipts: { + filecoinOfferReceipt, + filecoinSubmitReceipt, + filecoinAcceptReceipt, pieceOfferReceipt, pieceAcceptReceipt, aggregateOfferReceipt, diff --git a/packages/filecoin-api/test/context/store.js b/packages/filecoin-api/test/context/store.js index 51d19b1fa..02f98e61e 100644 --- a/packages/filecoin-api/test/context/store.js +++ b/packages/filecoin-api/test/context/store.js @@ -1,5 +1,5 @@ import * as API from '../../src/types.js' -import { RecordNotFound, StoreOperationFailed } from '../../src/errors.js' +import { StoreOperationFailed, RecordNotFound } from '../../src/errors.js' /** * @typedef {import('../../src/types.js').StorePutError} StorePutError @@ -47,7 +47,7 @@ export class Store { const t = this.getFn(this.items, item) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { @@ -85,7 +85,7 @@ export class Store { const t = this.queryFn(this.items, search) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { @@ -123,7 +123,7 @@ export class UpdatableStore extends Store { const t = this.updateFn(this.items, key, item) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { diff --git a/packages/filecoin-api/test/events/storefront.js b/packages/filecoin-api/test/events/storefront.js index 9de711d01..ceadee9c8 100644 --- a/packages/filecoin-api/test/events/storefront.js +++ b/packages/filecoin-api/test/events/storefront.js @@ -365,6 +365,7 @@ export const test = { aggregate: aggregate.link, group, piece: piece.link, + content: piece.content, piecesBlock, inclusionProof: { subtree: inclusionProof.ok[0], diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js index 928deac62..fdc9c7c24 100644 --- a/packages/filecoin-api/test/services/storefront.js +++ b/packages/filecoin-api/test/services/storefront.js @@ -1,6 +1,8 @@ import { Filecoin, Aggregator } from '@web3-storage/capabilities' +import * as Server from '@ucanto/server' import { CBOR } from '@ucanto/core' import * as Signer from '@ucanto/principal/ed25519' +import * as DealTrackerCaps from '@web3-storage/capabilities/filecoin/deal-tracker' import pWaitFor from 'p-wait-for' import * as API from '../../src/types.js' @@ -16,6 +18,8 @@ import { createInvocationsAndReceiptsForDealDataProofChain } from '../context/re import { getStoreImplementations } from '../context/store-implementations.js' import { FailingStore } from '../context/store.js' import { FailingQueue } from '../context/queue.js' +import { mockService } from '../context/mocks.js' +import { getConnection } from '../context/service.js' /** * @typedef {import('../../src/storefront/api.js').PieceRecord} PieceRecord @@ -200,7 +204,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, QueueOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, filecoinSubmitQueue: new FailingQueue(), }) @@ -232,7 +236,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, StoreOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, }) @@ -306,7 +310,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, QueueOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceOfferQueue: new FailingQueue(), }) @@ -363,6 +367,7 @@ export const test = { aggregate: aggregate.link, group, piece: piece.link, + content: piece.content, piecesBlock, inclusionProof: { subtree: inclusionProof.ok[0], @@ -447,11 +452,176 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, StoreOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, }) ), + 'filecoin/info gets aggregate where piece was included together with deals and inclusion proof': + wichMockableContext( + async (assert, context) => { + const { agent, aggregator, dealer } = await getServiceContext() + const group = context.id.did() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { aggregate, pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: piece.link.link(), + content: piece.content.link(), + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // Create inclusion proof for test + const inclusionProof = aggregate.resolveProof(piece.link) + if (inclusionProof.error) { + throw new Error('could not compute inclusion proof') + } + + // Create invocations and receipts for chain into DealDataProof + const dealMetadata = { + dataType: 0n, + dataSource: { + dealID: 111n, + }, + } + const { invocations, receipts } = + await createInvocationsAndReceiptsForDealDataProofChain({ + storefront: context.id, + aggregator, + dealer, + aggregate: aggregate.link, + group, + piece: piece.link, + content: piece.content, + piecesBlock, + inclusionProof: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + aggregateAcceptStatus: { + ...dealMetadata, + aggregate: aggregate.link, + }, + }) + + const storedInvocationsAndReceiptsRes = + await storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storedInvocationsAndReceiptsRes.ok) + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link.link(), + }, + }) + + const response = await filecoinInfoInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(piece.link.link())) + assert.equal(response.out.ok.deals.length, 1) + assert.ok(response.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataType), + dealMetadata.dataType + ) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataSource.dealID), + dealMetadata.dataSource.dealID + ) + assert.ok(response.out.ok.deals[0].inclusion.index) + assert.ok(response.out.ok.deals[0].inclusion.subtree) + }, + async (context) => { + /** + * Mock deal tracker to return deals + */ + const dealTrackerSigner = await Signer.generate() + const service = mockService({ + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async () => { + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + deals: { + 111: { + provider: 'f11111', + }, + }, + }) + + return result + }, + }), + }, + }) + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + return { + ...context, + service, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: dealTrackerSigner, + }, + }, + } + } + ), + 'filecoin/info fails if content is not known': async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link, + }, + }) + + const response = await filecoinInfoInv.execute(connection) + assert.ok(response.out.error) + }, } /** @@ -501,12 +671,12 @@ async function getServiceContext() { /** * @param {API.Test} testFn - * @param {(context: StorefrontApi.ServiceContext) => StorefrontApi.ServiceContext} mockContextFunction + * @param {(context: StorefrontApi.ServiceContext) => Promise} mockContextFunction */ function wichMockableContext(testFn, mockContextFunction) { // @ts-ignore - return function (...args) { - const modifiedArgs = [args[0], mockContextFunction(args[1])] + return async function (...args) { + const modifiedArgs = [args[0], await mockContextFunction(args[1])] // @ts-ignore return testFn(...modifiedArgs) } diff --git a/packages/filecoin-api/test/storefront.spec.js b/packages/filecoin-api/test/storefront.spec.js index 8bd534a00..677afe425 100644 --- a/packages/filecoin-api/test/storefront.spec.js +++ b/packages/filecoin-api/test/storefront.spec.js @@ -24,6 +24,7 @@ describe('storefront', () => { define(name, async () => { const storefrontSigner = await Signer.generate() const aggregatorSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() // resources /** @type {Map} */ @@ -34,6 +35,11 @@ describe('storefront', () => { const { storefront: { pieceStore, receiptStore, taskStore }, } = getStoreImplementations() + const service = getMockService() + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection await test( { @@ -54,6 +60,14 @@ describe('storefront', () => { pieceOfferQueue, taskStore, receiptStore, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: storefrontSigner, + with: storefrontSigner.did(), + audience: dealTrackerSigner, + }, + }, queuedMessages, validateAuthorization, } diff --git a/packages/filecoin-client/src/storefront.js b/packages/filecoin-client/src/storefront.js index 917f11b2a..06cd36555 100644 --- a/packages/filecoin-client/src/storefront.js +++ b/packages/filecoin-client/src/storefront.js @@ -147,3 +147,33 @@ export async function filecoinAccept( return await invocation.execute(conn) } + +/** + * The `filecoin/info` task can be executed to request info about a content piece + * in Filecoin. It issues a signed receipt of the execution result. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('./types.js').RequestOptions} [options] + */ +export async function filecoinInfo( + { issuer, with: resource, proofs, audience }, + piece, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = Storefront.filecoinInfo.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STOREFRONT.principal, + with: resource, + nb: { + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/types.ts b/packages/filecoin-client/src/types.ts index 762c522ea..752ae4711 100644 --- a/packages/filecoin-client/src/types.ts +++ b/packages/filecoin-client/src/types.ts @@ -16,6 +16,9 @@ import { FilecoinAccept, FilecoinAcceptSuccess, FilecoinAcceptFailure, + FilecoinInfo, + FilecoinInfoSuccess, + FilecoinInfoFailure, PieceOffer, PieceOfferSuccess, PieceOfferFailure, @@ -75,6 +78,7 @@ export interface StorefrontService { FilecoinAcceptSuccess, FilecoinAcceptFailure > + info: ServiceMethod } } diff --git a/packages/filecoin-client/test/helpers/mocks.js b/packages/filecoin-client/test/helpers/mocks.js index 3b4fb52c4..ebe42207c 100644 --- a/packages/filecoin-client/test/helpers/mocks.js +++ b/packages/filecoin-client/test/helpers/mocks.js @@ -18,6 +18,7 @@ export function mockService(impl) { offer: withCallCount(impl.filecoin?.offer ?? notImplemented), submit: withCallCount(impl.filecoin?.submit ?? notImplemented), accept: withCallCount(impl.filecoin?.accept ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), }, piece: { offer: withCallCount(impl.piece?.offer ?? notImplemented), diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js index acb00d235..557ed0507 100644 --- a/packages/filecoin-client/test/storefront.test.js +++ b/packages/filecoin-client/test/storefront.test.js @@ -9,6 +9,7 @@ import { filecoinOffer, filecoinSubmit, filecoinAccept, + filecoinInfo, } from '../src/storefront.js' import { randomAggregate, randomCargo } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -261,6 +262,55 @@ describe('storefront', () => { // does not include effect fx in receipt assert.ok(!res.fx.join) }) + + it('agent asks info of a filecoin piece', async () => { + const { agent } = await getContext() + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinOfferSuccess} */ + const filecoinOfferResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + info: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinInfo, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), agent.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, StorefrontCaps.filecoinInfo.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + const { piece } = invCap.nb + // piece link + assert.ok(piece.equals(cargo.link.link())) + + return Server.ok({ + piece, + deals: [], + }) + }, + }), + }, + }) + + const res = await filecoinInfo( + { + issuer: agent, + with: agent.did(), + audience: storefrontService, + }, + cargo.link, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.ok(res.out.ok.piece.equals(filecoinOfferResponse.piece)) + assert.deepEqual(res.out.ok.deals, []) + }) }) async function getContext() { diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index acefa960b..313f4584b 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -1,5 +1,7 @@ import * as Signer from '@ucanto/principal/ed25519' import { + getConnection, + getMockService, getStoreImplementations, getQueueImplementations, } from '@web3-storage/filecoin-api/test/context/service' @@ -41,8 +43,15 @@ export const createContext = async ( const usageStorage = new UsageStorage(storeTable) const signer = await Signer.generate() const aggregatorSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() const id = signer.withDID('did:web:test.web3.storage') + const service = getMockService() + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + /** @type {Map} */ const queuedMessages = new Map() const { @@ -86,6 +95,14 @@ export const createContext = async ( receiptStore, taskStore, requirePaymentPlan, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: id, + with: id.did(), + audience: dealTrackerSigner, + }, + }, ...createRevocationChecker({ revocationsStorage }), } diff --git a/packages/w3up-client/README.md b/packages/w3up-client/README.md index d823a3c98..699e60f8f 100644 --- a/packages/w3up-client/README.md +++ b/packages/w3up-client/README.md @@ -406,6 +406,8 @@ sequenceDiagram - [`capability.upload.add`](#capabilityuploadadd) - [`capability.upload.list`](#capabilityuploadlist) - [`capability.upload.remove`](#capabilityuploadremove) + - [`capability.filecoin.offer`](#capabilityfilecoinoffer) + - [`capability.filecoin.info`](#capabilityfilecoininfo) - [Types](#types) - [`Capability`](#capability) - [`CARMetadata`](#carmetadata) @@ -688,6 +690,27 @@ function remove( Remove a upload by root data CID. +### `capability.filecoin.offer` + +```ts +function offer ( + content: CID, + piece: PieceLink, +): Promise +``` + +Offer a Filecoin "piece" to be added to an aggregate that will be offered for Filecoin deal(s). + +### `capability.filecoin.info` + +```ts +function info ( + piece: PieceLink +): Promise +``` + +Get know deals and aggregate info of a Filecoin "piece" previously offered. + ## Types ### `Capability` diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index 616ab88e9..705045e15 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -80,9 +80,10 @@ "@ucanto/interface": "^9.0.0", "@ucanto/principal": "^9.0.0", "@ucanto/transport": "^9.0.0", - "@web3-storage/did-mailto": "workspace:^", "@web3-storage/access": "workspace:^", "@web3-storage/capabilities": "workspace:^", + "@web3-storage/did-mailto": "workspace:^", + "@web3-storage/filecoin-client": "workspace:^", "@web3-storage/upload-client": "workspace:^" }, "devDependencies": { @@ -93,6 +94,7 @@ "@types/mocha": "^10.0.1", "@types/node": "^20.8.4", "@ucanto/server": "^9.0.1", + "@web3-storage/data-segment": "^5.0.0", "@web3-storage/eslint-config-w3up": "workspace:^", "assert": "^2.0.0", "c8": "^7.13.0", diff --git a/packages/w3up-client/src/capability/filecoin.js b/packages/w3up-client/src/capability/filecoin.js new file mode 100644 index 000000000..9a5c8e8d1 --- /dev/null +++ b/packages/w3up-client/src/capability/filecoin.js @@ -0,0 +1,33 @@ +import { Storefront } from '@web3-storage/filecoin-client' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' +import { Base } from '../base.js' + +/** + * Client for interacting with the `filecoin/*` capabilities. + */ +export class FilecoinClient extends Base { + /** + * Offer a Filecoin "piece" to the resource. + * + * @param {import('multiformats').UnknownLink} content + * @param {import('@web3-storage/capabilities/types').PieceLink} piece + */ + async offer(content, piece) { + const conf = await this._invocationConfig([FilecoinCapabilities.offer.can]) + return Storefront.filecoinOffer(conf, content, piece, { + connection: this._serviceConf.filecoin, + }) + } + + /** + * Request info about a content piece in Filecoin deals + * + * @param {import('@web3-storage/capabilities/types').PieceLink} piece + */ + async info(piece) { + const conf = await this._invocationConfig([FilecoinCapabilities.info.can]) + return Storefront.filecoinInfo(conf, piece, { + connection: this._serviceConf.filecoin, + }) + } +} diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index 016719d76..f2662dd59 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -14,6 +14,7 @@ import { StoreClient } from './capability/store.js' import { UploadClient } from './capability/upload.js' import { SpaceClient } from './capability/space.js' import { AccessClient } from './capability/access.js' +import { FilecoinClient } from './capability/filecoin.js' export * as Access from './capability/access.js' export { StoreClient, UploadClient, SpaceClient, AccessClient } @@ -31,6 +32,7 @@ export class Client extends Base { store: new StoreClient(agentData, options), upload: new UploadClient(agentData, options), space: new SpaceClient(agentData, options), + filecoin: new FilecoinClient(agentData, options), } } diff --git a/packages/w3up-client/src/service.js b/packages/w3up-client/src/service.js index 26d00daba..20bb09365 100644 --- a/packages/w3up-client/src/service.js +++ b/packages/w3up-client/src/service.js @@ -26,8 +26,21 @@ export const uploadServiceConnection = connect({ }), }) +export const filecoinServiceURL = new URL('https://up.web3.storage') +export const filecoinServicePrincipal = DID.parse('did:web:web3.storage') + +export const filecoinServiceConnection = connect({ + id: filecoinServicePrincipal, + codec: CAR.outbound, + channel: HTTP.open({ + url: filecoinServiceURL, + method: 'POST', + }), +}) + /** @type {import('./types.js').ServiceConf} */ export const serviceConf = { access: accessServiceConnection, upload: uploadServiceConnection, + filecoin: filecoinServiceConnection, } diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 5201c7109..c2e8bbe28 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -13,6 +13,7 @@ import type { Unit, } from '@ucanto/interface' import { type Client } from './client.js' +import { StorefrontService } from '@web3-storage/filecoin-client/storefront' export * from '@ucanto/interface' export * from '@web3-storage/did-mailto' export type { Agent, CapabilityQuery } from '@web3-storage/access/agent' @@ -28,6 +29,7 @@ export type ProofQuery = Record> export interface ServiceConf { access: ConnectionView upload: ConnectionView + filecoin: ConnectionView } export interface ClientFactoryOptions { diff --git a/packages/w3up-client/test/capability/filecoin.test.js b/packages/w3up-client/test/capability/filecoin.test.js new file mode 100644 index 000000000..53427c4fc --- /dev/null +++ b/packages/w3up-client/test/capability/filecoin.test.js @@ -0,0 +1,162 @@ +import assert from 'assert' +import { + create as createServer, + provide, + provideAdvanced, + ok, +} from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as Signer from '@ucanto/principal/ed25519' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' +import { AgentData } from '@web3-storage/access/agent' + +import { randomAggregate, randomCargo } from '../helpers/random.js' +import { mockService, mockServiceConf } from '../helpers/mocks.js' +import { Client } from '../../src/client.js' +import { validateAuthorization } from '../helpers/utils.js' + +describe('FilecoinClient', () => { + describe('offer', () => { + it('should send an offer', async () => { + const service = mockService({ + filecoin: { + offer: provideAdvanced({ + capability: FilecoinCapabilities.offer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + assert.ok(invCap.nb) + + // Create effect for receipt with self signed queued operation + const submitfx = await FilecoinCapabilities.submit + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + expiration: Infinity, + }) + .delegate() + + const acceptfx = await FilecoinCapabilities.accept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + expiration: Infinity, + }) + .delegate() + + return ok({ + piece: invCap.nb.piece, + }) + .fork(submitfx.link()) + .join(acceptfx.link()) + }, + }), + }, + }) + const server = createServer({ + id: await Signer.generate(), + service, + codec: CAR.inbound, + validateAuthorization, + }) + + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: await mockServiceConf(server), + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + const [cargo] = await randomCargo(1, 100) + const res = await alice.capability.filecoin.offer( + cargo.content, + cargo.link + ) + + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) + assert(res.out.ok) + assert(res.out.ok.piece.equals(cargo.link)) + assert(res.fx.join) + assert(res.fx.fork.length) + }) + }) + describe('info', () => { + it('should get piece info', async () => { + const { pieces, aggregate } = await randomAggregate(10, 100) + const cargo = pieces[0] + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(cargo.link) + if (proof.error) { + throw new Error('could not compute proof') + } + /** @type {import('@web3-storage/capabilities/types').FilecoinInfoSuccess} */ + const filecoinAcceptResponse = { + piece: cargo.link, + deals: [ + { + aggregate: aggregate.link, + provider: 'f1111', + inclusion: { + subtree: proof.ok[0], + index: proof.ok[1], + }, + aux: { + dataType: 0n, + dataSource: { + dealID: 1138n, + }, + }, + }, + ], + } + const service = mockService({ + filecoin: { + info: provide(FilecoinCapabilities.info, ({ invocation }) => { + const invCap = invocation.capabilities[0] + assert.ok(invCap.nb) + + return ok(filecoinAcceptResponse) + }), + }, + }) + const server = createServer({ + id: await Signer.generate(), + service, + codec: CAR.inbound, + validateAuthorization, + }) + + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: await mockServiceConf(server), + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + const res = await alice.capability.filecoin.info(cargo.link) + + assert(service.filecoin.info.called) + assert.equal(service.filecoin.info.callCount, 1) + assert(res.out.ok) + assert(res.out.ok.piece.equals(cargo.link)) + assert.equal(res.out.ok.deals.length, 1) + assert(res.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert(res.out.ok.deals[0].aux.dataSource.dealID) + assert(res.out.ok.deals[0].provider) + assert.deepEqual(res.out.ok.deals[0].inclusion, { + subtree: proof.ok[0], + index: proof.ok[1], + }) + }) + }) +}) diff --git a/packages/w3up-client/test/helpers/car.js b/packages/w3up-client/test/helpers/car.js index 031b85f10..07aa974db 100644 --- a/packages/w3up-client/test/helpers/car.js +++ b/packages/w3up-client/test/helpers/car.js @@ -22,5 +22,5 @@ export async function toCAR(bytes) { const blob = new Blob(chunks) const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - return Object.assign(blob, { cid, roots: [root] }) + return Object.assign(blob, { cid, roots: [root], bytes }) } diff --git a/packages/w3up-client/test/helpers/mocks.js b/packages/w3up-client/test/helpers/mocks.js index 7f98a3544..89fcbed1d 100644 --- a/packages/w3up-client/test/helpers/mocks.js +++ b/packages/w3up-client/test/helpers/mocks.js @@ -14,6 +14,7 @@ const notImplemented = () => { * upload: Partial * space: Partial * ucan: Partial + * filecoin: Partial * }>} impl */ export function mockService(impl) { @@ -42,6 +43,10 @@ export function mockService(impl) { ucan: { revoke: withCallCount(impl.ucan?.revoke ?? notImplemented), }, + filecoin: { + offer: withCallCount(impl.filecoin?.offer ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), + }, } } @@ -72,5 +77,5 @@ export async function mockServiceConf(server) { codec: CAR.outbound, channel: server, }) - return { access: connection, upload: connection } + return { access: connection, upload: connection, filecoin: connection } } diff --git a/packages/w3up-client/test/helpers/random.js b/packages/w3up-client/test/helpers/random.js index b7e8aedb4..b1f512707 100644 --- a/packages/w3up-client/test/helpers/random.js +++ b/packages/w3up-client/test/helpers/random.js @@ -1,3 +1,4 @@ +import { Aggregate, Piece } from '@web3-storage/data-segment' import { toCAR } from './car.js' /** @param {number} size */ @@ -29,3 +30,42 @@ export async function randomCAR(size) { const bytes = await randomBytes(size) return toCAR(bytes) } + +/** + * @param {number} length + * @param {number} size + */ +export async function randomCargo(length, size) { + const cars = await Promise.all( + Array.from({ length }).map(() => randomCAR(size)) + ) + + return cars.map((car) => { + const piece = Piece.fromPayload(car.bytes) + + return { + link: piece.link, + height: piece.height, + root: piece.root, + padding: piece.padding, + content: car.cid, + } + }) +} + +/** + * @param {number} length + * @param {number} size + */ +export async function randomAggregate(length, size) { + const pieces = await randomCargo(length, size) + + const aggregateBuild = Aggregate.build({ + pieces, + }) + + return { + pieces, + aggregate: aggregateBuild, + } +} diff --git a/packages/w3up-client/test/test.js b/packages/w3up-client/test/test.js index 5ffe569ae..7bf1f9105 100644 --- a/packages/w3up-client/test/test.js +++ b/packages/w3up-client/test/test.js @@ -42,6 +42,7 @@ export const setup = async () => { serviceConf: { access: context.connection, upload: context.connection, + filecoin: context.connection, }, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 577ff38e5..f6f92e6dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -552,6 +552,9 @@ importers: '@web3-storage/did-mailto': specifier: workspace:^ version: link:../did-mailto + '@web3-storage/filecoin-client': + specifier: workspace:^ + version: link:../filecoin-client '@web3-storage/upload-client': specifier: workspace:^ version: link:../upload-client @@ -574,6 +577,9 @@ importers: '@ucanto/server': specifier: ^9.0.1 version: 9.0.1 + '@web3-storage/data-segment': + specifier: ^5.0.0 + version: 5.0.0 '@web3-storage/eslint-config-w3up': specifier: workspace:^ version: link:../eslint-config-w3up @@ -3567,6 +3573,14 @@ packages: multiformats: 11.0.2 sync-multihash-sha2: 1.0.0 + /@web3-storage/data-segment@5.0.0: + resolution: {integrity: sha512-5CbElsxec2DsKhEHEh3XRGISAyna+bCjKjjvFrLcYyXLCaiSt/nF3ypcllxwjpE4newMUArymGKGzzZnRWL2kg==} + dependencies: + '@ipld/dag-cbor': 9.0.6 + multiformats: 11.0.2 + sync-multihash-sha2: 1.0.0 + dev: true + /@web3-storage/sigv4@1.0.2: resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==} dependencies: