From 328039d8a29fec3c1bbab28d1bb9de1643f54f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Sousa=20Andrade?= <659718+joaosa@users.noreply.github.com> Date: Tue, 4 Jun 2024 15:39:20 +0100 Subject: [PATCH] feat: add blob/get (#1484) Adds support for `blob/get` as defined in https://github.com/w3s-project/specs/pull/126 --------- Co-authored-by: Vasco Santos Co-authored-by: Irakli Gozalishvili --- packages/capabilities/src/blob.js | 35 ++++++ packages/capabilities/src/types.ts | 11 ++ packages/upload-api/src/blob.js | 6 + packages/upload-api/src/blob/get.js | 21 ++++ packages/upload-api/src/types.ts | 8 ++ packages/upload-api/src/types/blob.ts | 8 +- .../upload-api/test/handlers/web3.storage.js | 11 +- packages/upload-client/src/blob.js | 49 ++++++++ packages/upload-client/src/types.ts | 11 ++ packages/upload-client/test/blob.test.js | 118 ++++++++++++++++++ packages/upload-client/test/helpers/mocks.js | 6 + packages/w3up-client/src/capability/blob.js | 12 ++ .../w3up-client/test/capability/blob.test.js | 33 +++++ 13 files changed, 314 insertions(+), 15 deletions(-) create mode 100644 packages/upload-api/src/blob/get.js diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index edbc971b7..6af1ae576 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -137,6 +137,41 @@ export const list = capability({ }, }) +/** + * Capability can be used to get the stored Blob from the (memory) + * space identified by `with` field. + */ +export const get = capability({ + can: 'space/blob/get/0/1', + /** + * DID of the (memory) space where Blob is stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A multihash digest of the blob payload bytes, uniquely identifying blob. + */ + digest: Schema.bytes(), + }), + derives: (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } else if ( + delegated.nb.digest && + !equals(delegated.nb.digest, claimed.nb.digest) + ) { + return fail( + `Link ${ + claimed.nb.digest ? `${claimed.nb.digest}` : '' + } violates imposed ${delegated.nb.digest} constraint.` + ) + } + return ok({}) + }, +}) + // ⚠️ We export imports here so they are not omitted in generated typedefs // @see https://github.com/microsoft/TypeScript/issues/51548 export { Schema } diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index c989ab54f..ba28eeb05 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -502,6 +502,7 @@ export type Blob = InferInvokedCapability export type BlobAdd = InferInvokedCapability export type BlobRemove = InferInvokedCapability export type BlobList = InferInvokedCapability +export type BlobGet = InferInvokedCapability export type ServiceBlob = InferInvokedCapability export type BlobAllocate = InferInvokedCapability export type BlobAccept = InferInvokedCapability @@ -550,6 +551,15 @@ export interface BlobListSuccess extends ListResponse {} // TODO: make types more specific export type BlobListFailure = Ucanto.Failure +// Blob get +export interface BlobGetSuccess { + blob: { digest: Uint8Array; size: number } + cause: UnknownLink +} + +// TODO: make types more specific +export type BlobGetFailure = Ucanto.Failure + // Blob allocate export interface BlobAllocateSuccess { size: number @@ -902,6 +912,7 @@ export type ServiceAbilityArray = [ BlobAdd['can'], BlobRemove['can'], BlobList['can'], + BlobGet['can'], ServiceBlob['can'], BlobAllocate['can'], BlobAccept['can'], diff --git a/packages/upload-api/src/blob.js b/packages/upload-api/src/blob.js index 2e5695205..7df55c5da 100644 --- a/packages/upload-api/src/blob.js +++ b/packages/upload-api/src/blob.js @@ -1,6 +1,7 @@ import { blobAddProvider } from './blob/add.js' import { blobListProvider } from './blob/list.js' import { blobRemoveProvider } from './blob/remove.js' +import { blobGetProvider } from './blob/get.js' import * as API from './types.js' export { BlobNotFound } from './blob/lib.js' @@ -13,5 +14,10 @@ export function createService(context) { add: blobAddProvider(context), list: blobListProvider(context), remove: blobRemoveProvider(context), + get: { + 0: { + 1: blobGetProvider(context), + }, + }, } } diff --git a/packages/upload-api/src/blob/get.js b/packages/upload-api/src/blob/get.js new file mode 100644 index 000000000..7d0a2678f --- /dev/null +++ b/packages/upload-api/src/blob/get.js @@ -0,0 +1,21 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' +import { BlobNotFound } from './lib.js' +import { decode } from 'multiformats/hashes/digest' + +/** + * @param {API.BlobServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobGetProvider(context) { + return Server.provide(Blob.get, async ({ capability }) => { + const { digest } = capability.nb + const space = Server.DID.parse(capability.with).did() + const res = await context.allocationsStorage.get(space, digest) + if (res.error && res.error.name === 'RecordNotFound') { + return Server.error(new BlobNotFound(decode(digest))) + } + return res + }) +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 9395ebc7e..55ef6730f 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -75,6 +75,9 @@ import { BlobRemove, BlobRemoveSuccess, BlobRemoveFailure, + BlobGet, + BlobGetSuccess, + BlobGetFailure, BlobAllocate, BlobAllocateSuccess, BlobAllocateFailure, @@ -322,6 +325,11 @@ export interface Service extends StorefrontService, W3sService { add: ServiceMethod remove: ServiceMethod list: ServiceMethod + get: { + 0: { + 1: ServiceMethod + } + } } } plan: { diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index ae3e34d3e..937eb6c66 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -10,6 +10,7 @@ import { Multihash, BlobListItem, BlobRemoveSuccess, + BlobGetSuccess, } from '@web3-storage/capabilities/types' import { RecordKeyConflict, ListResponse } from '../types.js' @@ -21,7 +22,7 @@ export interface AllocationsStorage { get: ( space: DID, blobMultihash: Multihash - ) => Promise> + ) => Promise> exists: ( space: DID, blobMultihash: Multihash @@ -59,11 +60,6 @@ export interface BlobAddInput { export interface BlobAddOutput extends Omit {} -export interface BlobGetOutput { - blob: { digest: Uint8Array; size: number } - cause: UnknownLink -} - export interface BlobsStorage { has: (content: Multihash) => Promise> createUploadUrl: ( diff --git a/packages/upload-api/test/handlers/web3.storage.js b/packages/upload-api/test/handlers/web3.storage.js index 87c702ada..0e2ebfba3 100644 --- a/packages/upload-api/test/handlers/web3.storage.js +++ b/packages/upload-api/test/handlers/web3.storage.js @@ -1,10 +1,7 @@ import * as API from '../../src/types.js' import { equals } from 'uint8arrays' -import { create as createLink } from 'multiformats/link' import { Absentee } from '@ucanto/principal' -import { Digest } from 'multiformats/hashes/digest' import { sha256 } from 'multiformats/hashes/sha2' -import { code as rawCode } from 'multiformats/codecs/raw' import { Assert } from '@web3-storage/content-claims/capability' import * as BlobCapabilities from '@web3-storage/capabilities/blob' import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' @@ -225,7 +222,7 @@ export const test = { } // second blob allocate invocation - const reallocation = await await W3sBlobCapabilities.allocate + const reallocation = await W3sBlobCapabilities.allocate .invoke({ issuer: context.id, audience: context.id, @@ -604,10 +601,6 @@ export const test = { const multihash = await sha256.digest(data) const digest = multihash.bytes const size = data.byteLength - const content = createLink( - rawCode, - new Digest(sha256.code, 32, digest, digest) - ) // create service connection const connection = connect({ @@ -682,7 +675,7 @@ export const test = { equals( // @ts-expect-error nb unknown delegation.capabilities[0].nb.content.digest, - content.multihash.bytes + digest ) ) // @ts-expect-error nb unknown diff --git a/packages/upload-client/src/blob.js b/packages/upload-client/src/blob.js index dd29d7b6c..69f0ac1f7 100644 --- a/packages/upload-client/src/blob.js +++ b/packages/upload-client/src/blob.js @@ -423,3 +423,52 @@ export async function remove( return result.out } + +/** + * Gets a stored Blob file by digest. + * + * @param {import('./types.js').InvocationConfig} conf Configuration + * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. + * + * The `issuer` is the signing authority that is issuing the UCAN + * invocation(s). It is typically the user _agent_. + * + * The `with` is the resource the invocation applies to. It is typically the + * DID of a space. + * + * The `proofs` are a set of capability delegations that prove the issuer + * has the capability to perform the action. + * + * The issuer needs the `blob/get/0/1` delegated capability. + * @param {import('multiformats').MultihashDigest} multihash of the blob + * @param {import('./types.js').RequestOptions} [options] + */ +export async function get( + { issuer, with: resource, proofs, audience }, + multihash, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + const result = await BlobCapabilities.get + .invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? servicePrincipal, + with: SpaceDID.from(resource), + nb: { + digest: multihash.bytes, + }, + proofs, + nonce: options.nonce, + }) + .execute(conn) + + if (!result.out.ok) { + throw new Error(`failed ${BlobCapabilities.get.can} invocation`, { + cause: result.out.error, + }) + } + + return result.out +} diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 23c3b2052..31cfe2a60 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -39,6 +39,9 @@ import { BlobList, BlobListSuccess, BlobListFailure, + BlobGet, + BlobGetSuccess, + BlobGetFailure, IndexAdd, IndexAddSuccess, IndexAddFailure, @@ -106,6 +109,9 @@ export type { BlobList, BlobListSuccess, BlobListFailure, + BlobGet, + BlobGetSuccess, + BlobGetFailure, IndexAdd, IndexAddSuccess, IndexAddFailure, @@ -161,6 +167,11 @@ export interface Service extends StorefrontService { add: ServiceMethod remove: ServiceMethod list: ServiceMethod + get: { + 0: { + 1: ServiceMethod + } + } } index: { add: ServiceMethod diff --git a/packages/upload-client/test/blob.test.js b/packages/upload-client/test/blob.test.js index 06f44a1a8..402f15f7c 100644 --- a/packages/upload-client/test/blob.test.js +++ b/packages/upload-client/test/blob.test.js @@ -864,3 +864,121 @@ describe('Blob.remove', () => { ) }) }) + +describe('Blob.get', () => { + it('get a stored Blob', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() + const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) + + const proofs = [ + await BlobCapabilities.get.delegate({ + issuer: space, + audience: agent, + with: space.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + space: { + blob: { + get: { + 0: { + 1: provide(BlobCapabilities.get, ({ invocation }) => { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, BlobCapabilities.get.can) + assert.equal(invCap.with, space.did()) + assert.equal(String(invCap.nb?.digest), bytesHash.bytes) + return { + ok: { + cause: invocation.link(), + blob: { digest: bytesHash.bytes, size: bytes.length }, + }, + } + }), + }, + }, + }, + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + codec: CAR.inbound, + validateAuthorization, + }) + const connection = Client.connect({ + id: serviceSigner, + codec: CAR.outbound, + channel: server, + }) + + const result = await Blob.get( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, + { connection } + ) + + assert(service.space.blob.get[0][1].called) + assert.equal(service.space.blob.get[0][1].callCount, 1) + + assert(result.ok) + assert.deepEqual(result.ok.blob.digest, bytesHash.bytes) + }) + + it('throws on service error', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() + const bytes = await randomBytes(128) + const bytesHash = await sha256.digest(bytes) + + const proofs = [ + await BlobCapabilities.get.delegate({ + issuer: space, + audience: agent, + with: space.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + space: { + blob: { + get: { + 0: { + 1: provide(BlobCapabilities.get, () => { + throw new Server.Failure('boom') + }), + }, + }, + }, + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + codec: CAR.inbound, + validateAuthorization, + }) + const connection = Client.connect({ + id: serviceSigner, + codec: CAR.outbound, + channel: server, + }) + + await assert.rejects( + Blob.get( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + bytesHash, + { connection } + ), + { message: 'failed space/blob/get/0/1 invocation' } + ) + }) +}) diff --git a/packages/upload-client/test/helpers/mocks.js b/packages/upload-client/test/helpers/mocks.js index 21c79e142..e8530ccdd 100644 --- a/packages/upload-client/test/helpers/mocks.js +++ b/packages/upload-client/test/helpers/mocks.js @@ -18,6 +18,7 @@ const notImplemented = () => { * }>} impl */ export function mockService(impl) { + const get = impl.space?.blob?.get return { ucan: { conclude: withCallCount(impl.ucan?.conclude ?? notImplemented), @@ -27,6 +28,11 @@ export function mockService(impl) { add: withCallCount(impl.space?.blob?.add ?? notImplemented), list: withCallCount(impl.space?.blob?.list ?? notImplemented), remove: withCallCount(impl.space?.blob?.remove ?? notImplemented), + get: { + 0: { + 1: withCallCount(get ? get[0][1] : notImplemented), + }, + }, }, index: { add: withCallCount(impl.space?.index?.add ?? notImplemented), diff --git a/packages/w3up-client/src/capability/blob.js b/packages/w3up-client/src/capability/blob.js index f267d949a..487af1978 100644 --- a/packages/w3up-client/src/capability/blob.js +++ b/packages/w3up-client/src/capability/blob.js @@ -49,4 +49,16 @@ export class BlobClient extends Base { options.connection = this._serviceConf.upload return Blob.remove(conf, digest, options) } + + /** + * Gets a stored blob by multihash digest. + * + * @param {import('multiformats').MultihashDigest} digest - digest of blob to get. + * @param {import('../types.js').RequestOptions} [options] + */ + async get(digest, options = {}) { + const conf = await this._invocationConfig([BlobCapabilities.get.can]) + options.connection = this._serviceConf.upload + return Blob.get(conf, digest, options) + } } diff --git a/packages/w3up-client/test/capability/blob.test.js b/packages/w3up-client/test/capability/blob.test.js index 2ab163812..5f47d7f95 100644 --- a/packages/w3up-client/test/capability/blob.test.js +++ b/packages/w3up-client/test/capability/blob.test.js @@ -119,6 +119,39 @@ export const BlobClient = Test.withContext({ const result = await alice.capability.blob.remove(multihash) assert.ok(result.ok) }, + 'should get a stored blob': async ( + assert, + { connection, provisionsStorage } + ) => { + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: { + access: connection, + upload: connection, + }, + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + await alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + // Then we setup a billing for this account + await provisionsStorage.put({ + // @ts-expect-error + provider: connection.id.did(), + account: alice.agent.did(), + consumer: space.did(), + }) + + const bytes = await randomBytes(128) + const { multihash } = await alice.capability.blob.add(new Blob([bytes]), { + receiptsEndpoint, + }) + + const result = await alice.capability.blob.get(multihash) + assert.ok(result.ok) + }, }) Test.test({ BlobClient })