feat: add store/get and upload/get capabilities (#942)

Capability definitions to allow clients to check for inclusion of cid in
store shards or upload roots. Also allows client to fetch the additional
properties we store. This will, for example, let a client verify all
shards for an upload have been removed before removing an upload.

_I'm aiming to simplify things with this PR so that we return insertedAt
info on the Upload/Store ListItem types, and return the same object
shape for a single object get. If this is pleasing, then I will continue
in this direction and make Remove also return the same shape if it
removes something_

TODO:
- [x] spec update storacha/specs#82
- [x] tests


License: MIT

---------

Signed-off-by: Oli Evans <oli@protocol.ai>
Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>

packages/capabilities/src/index.js

@@ -40,10 +40,12 @@ export const abilitiesAsStrings = [
   Space.info.can,
   Upload.upload.can,
   Upload.add.can,
+  Upload.get.can,
   Upload.remove.can,
   Upload.list.can,
   Store.store.can,
   Store.add.can,
+  Store.get.can,
   Store.remove.can,
   Store.list.can,
   Access.access.can,

packages/capabilities/src/store.js

@@ -77,6 +77,28 @@ export const add = capability({
   },
 })
 
+/**
+ * Capability to get store metadata by shard CID.
+ * Use to check for inclusion, or get shard size and origin
+ *
+ * `nb.link` is optional to allow delegation of `store/get`
+ * capability for any shard CID. If link is specified, then the
+ * capability only allows a get for that specific CID.
+ *
+ * When used as as an invocation, `nb.link` must be specified.
+ */
+export const get = capability({
+  can: 'store/get',
+  with: SpaceDID,
+  nb: Schema.struct({
+    /**
+     * shard CID to fetch info about.
+     */
+    link: Link.optional(),
+  }),
+  derives: equalLink,
+})
+
 /**
  * Capability can be used to remove the stored CAR file from the (memory)
  * space identified by `with` field.

packages/capabilities/src/types.ts

@@ -15,7 +15,7 @@ import type { PieceLink } from '@web3-storage/data-segment'
 import { space, info } from './space.js'
 import * as provider from './provider.js'
 import { top } from './top.js'
-import { add, list, remove, store } from './store.js'
+import * as StoreCaps from './store.js'
 import * as UploadCaps from './upload.js'
 import * as AccessCaps from './access.js'
 import * as CustomerCaps from './customer.js'
@@ -221,14 +221,22 @@ export interface ChainTrackerInfoFailure extends Ucanto.Failure {
 // Upload
 export type Upload = InferInvokedCapability<typeof UploadCaps.upload>
 export type UploadAdd = InferInvokedCapability<typeof UploadCaps.add>
+export type UploadGet = InferInvokedCapability<typeof UploadCaps.get>
 export type UploadRemove = InferInvokedCapability<typeof UploadCaps.remove>
 export type UploadList = InferInvokedCapability<typeof UploadCaps.list>
 
+export interface UploadNotFound extends Ucanto.Failure {
+  name: 'UploadNotFound'
+}
+
+export type UploadGetFailure = UploadNotFound | Ucanto.Failure
+
 // Store
-export type Store = InferInvokedCapability<typeof store>
-export type StoreAdd = InferInvokedCapability<typeof add>
-export type StoreRemove = InferInvokedCapability<typeof remove>
-export type StoreList = InferInvokedCapability<typeof list>
+export type Store = InferInvokedCapability<typeof StoreCaps.store>
+export type StoreAdd = InferInvokedCapability<typeof StoreCaps.add>
+export type StoreGet = InferInvokedCapability<typeof StoreCaps.get>
+export type StoreRemove = InferInvokedCapability<typeof StoreCaps.remove>
+export type StoreList = InferInvokedCapability<typeof StoreCaps.list>
 
 export type StoreAddSuccess = StoreAddSuccessDone | StoreAddSuccessUpload
 export interface StoreAddSuccessDone {
@@ -257,6 +265,10 @@ export interface StoreItemNotFound extends Ucanto.Failure {
 
 export type StoreRemoveFailure = StoreItemNotFound | Ucanto.Failure
 
+export type StoreGetSuccess = StoreListItem
+
+export type StoreGetFailure = StoreItemNotFound | Ucanto.Failure
+
 export interface StoreListSuccess extends ListResponse<StoreListItem> {}
 
 export interface ListResponse<R> {
@@ -271,13 +283,21 @@ export interface StoreListItem {
   link: UnknownLink
   size: number
   origin?: UnknownLink
+  insertedAt: string
 }
 
-export interface UploadAddSuccess {
+export interface UploadListItem {
   root: UnknownLink
   shards?: CARLink[]
+  insertedAt: string
+  updatedAt: string
 }
 
+// TODO: (olizilla) make this an UploadListItem too?
+export type UploadAddSuccess = Omit<UploadListItem, 'insertedAt' | 'updatedAt'>
+
+export type UploadGetSuccess = UploadListItem
+
 export type UploadRemoveSuccess = UploadDidRemove | UploadDidNotRemove
 
 export interface UploadDidRemove extends UploadAddSuccess {}
@@ -289,8 +309,6 @@ export interface UploadDidNotRemove {
 
 export interface UploadListSuccess extends ListResponse<UploadListItem> {}
 
-export interface UploadListItem extends UploadAddSuccess {}
-
 // UCAN core events
 
 export type UCANRevoke = InferInvokedCapability<typeof UCANCaps.revoke>
@@ -392,10 +410,12 @@ export type AbilitiesArray = [
   SpaceInfo['can'],
   Upload['can'],
   UploadAdd['can'],
+  UploadGet['can'],
   UploadRemove['can'],
   UploadList['can'],
   Store['can'],
   StoreAdd['can'],
+  StoreGet['can'],
   StoreRemove['can'],
   StoreList['can'],
   Access['can'],

packages/capabilities/src/upload.js

@@ -77,6 +77,38 @@ export const add = capability({
   },
 })
 
+/**
+ * Capability to get upload metadata by root CID.
+ * Use to check for inclusion, or find the shards for a root.
+ *
+ * `nb.root` is optional to allow delegation of `upload/get`
+ * capability for any root. If root is specified, then the
+ * capability only allows a get for that single cid.
+ *
+ * When used as as an invocation, `nb.root` must be specified.
+ */
+export const get = capability({
+  can: 'upload/get',
+  with: SpaceDID,
+  nb: Schema.struct({
+    /**
+     * Root CID of the DAG to fetch upload info about.
+     */
+    root: Link.optional(),
+  }),
+  derives: (self, from) => {
+    const res = equalWith(self, from)
+    if (res.error) {
+      return res
+    }
+    if (!from.nb.root) {
+      return res
+    }
+    // root must match if specified in the proof
+    return equal(self.nb.root, from.nb.root, 'root')
+  },
+})
+
 /**
  * Capability removes an upload (identified by it's root CID) from the upload
  * list. Please note that removing an upload does not delete corresponding shards

packages/capabilities/src/utils.js

@@ -60,7 +60,7 @@ export function equal(child, parent, constraint) {
 }
 
 /**
- * @template {Types.ParsedCapability<"store/add"|"store/remove", Types.URI<'did:'>, {link?: Types.Link<unknown, number, number, 0|1>}>} T
+ * @template {Types.ParsedCapability<"store/add"|"store/get"|"store/remove", Types.URI<'did:'>, {link?: Types.Link<unknown, number, number, 0|1>}>} T
  * @param {T} claimed
  * @param {T} delegated
  * @returns {Types.Result<{}, Types.Failure>}

packages/upload-api/src/store.js

@@ -1,4 +1,5 @@
 import { storeAddProvider } from './store/add.js'
+import { storeGetProvider } from './store/get.js'
 import { storeListProvider } from './store/list.js'
 import { storeRemoveProvider } from './store/remove.js'
 import * as API from './types.js'
@@ -9,6 +10,7 @@ import * as API from './types.js'
 export function createService(context) {
   return {
     add: storeAddProvider(context),
+    get: storeGetProvider(context),
     list: storeListProvider(context),
     remove: storeRemoveProvider(context),
   }

packages/upload-api/src/store/get.js

@@ -0,0 +1,29 @@
+import * as Server from '@ucanto/server'
+import * as Store from '@web3-storage/capabilities/store'
+import * as API from '../types.js'
+
+/**
+ * @param {API.StoreServiceContext} context
+ * @returns {API.ServiceMethod<API.StoreGet, API.StoreGetSuccess, API.StoreGetFailure>}
+ */
+export function storeGetProvider(context) {
+  return Server.provide(Store.get, async ({ capability }) => {
+    const { link } = capability.nb
+    if (!link) {
+      return Server.fail('nb.link must be set')
+    }
+    const space = Server.DID.parse(capability.with).did()
+    const res = await context.storeTable.get(space, link)
+    if (!res) {
+      return {
+        error: {
+          name: 'StoreItemNotFound',
+          message: 'Store item not found',
+        },
+      }
+    }
+    return {
+      ok: res,
+    }
+  })
+}

packages/upload-api/src/types.ts

@@ -52,6 +52,7 @@ export interface DebugEmail extends Email {
 
 import {
   StoreAdd,
+  StoreGet,
   StoreAddSuccess,
   StoreRemove,
   StoreRemoveSuccess,
@@ -60,6 +61,7 @@ import {
   StoreListSuccess,
   StoreListItem,
   UploadAdd,
+  UploadGet,
   UploadAddSuccess,
   UploadRemove,
   UploadRemoveSuccess,
@@ -109,9 +111,13 @@ import {
   ProviderAddFailure,
   SpaceInfo,
   ProviderDID,
+  StoreGetFailure,
+  UploadGetFailure,
   UCANRevoke,
   ListResponse,
   CARLink,
+  StoreGetSuccess,
+  UploadGetSuccess,
   UCANRevokeSuccess,
   UCANRevokeFailure,
 } from '@web3-storage/capabilities/types'
@@ -137,11 +143,13 @@ export type { RateLimitsStorage, RateLimit } from './types/rate-limits'
 export interface Service {
   store: {
     add: ServiceMethod<StoreAdd, StoreAddSuccess, Failure>
+    get: ServiceMethod<StoreGet, StoreGetSuccess, StoreGetFailure>
     remove: ServiceMethod<StoreRemove, StoreRemoveSuccess, StoreRemoveFailure>
     list: ServiceMethod<StoreList, StoreListSuccess, Failure>
   }
   upload: {
     add: ServiceMethod<UploadAdd, UploadAddSuccess, Failure>
+    get: ServiceMethod<UploadGet, UploadGetSuccess, UploadGetFailure>
     remove: ServiceMethod<UploadRemove, UploadRemoveSuccess, Failure>
     list: ServiceMethod<UploadList, UploadListSuccess, Failure>
   }
@@ -325,9 +333,7 @@ export interface UcantoServerTestContext
   fetch: typeof fetch
 }
 
-export interface StoreTestContext {
-  testStoreTable: TestStoreTable
-}
+export interface StoreTestContext {}
 
 export interface UploadTestContext {}
 
@@ -367,9 +373,9 @@ export interface DudewhereBucket {
 }
 
 export interface StoreTable {
-  inspect: (link: UnknownLink) => Promise<StoreGetOk>
+  inspect: (link: UnknownLink) => Promise<StoreInspectSuccess>
   exists: (space: DID, link: UnknownLink) => Promise<boolean>
-  get: (space: DID, link: UnknownLink) => Promise<StoreAddOutput | undefined>
+  get: (space: DID, link: UnknownLink) => Promise<StoreGetSuccess | undefined>
   insert: (item: StoreAddInput) => Promise<StoreAddOutput>
   remove: (space: DID, link: UnknownLink) => Promise<void>
   list: (
@@ -378,26 +384,16 @@ export interface StoreTable {
   ) => Promise<ListResponse<StoreListItem>>
 }
 
-export interface TestStoreTable {
-  get(
-    space: DID,
-    link: UnknownLink
-  ): Promise<
-    (StoreAddInput & StoreListItem & { insertedAt: string }) | undefined
-  >
-}
-
 export interface UploadTable {
-  inspect: (link: UnknownLink) => Promise<UploadGetOk>
+  inspect: (link: UnknownLink) => Promise<UploadInspectSuccess>
   exists: (space: DID, root: UnknownLink) => Promise<boolean>
+  get: (space: DID, link: UnknownLink) => Promise<UploadGetSuccess | undefined>
   insert: (item: UploadAddInput) => Promise<UploadAddSuccess>
   remove: (space: DID, root: UnknownLink) => Promise<UploadRemoveSuccess | null>
   list: (
     space: DID,
     options?: ListOptions
-  ) => Promise<
-    ListResponse<UploadListItem & { insertedAt: string; updatedAt: string }>
-  >
+  ) => Promise<ListResponse<UploadListItem>>
 }
 
 export type SpaceInfoSuccess = {
@@ -435,7 +431,7 @@ export interface StoreAddInput {
 export interface StoreAddOutput
   extends Omit<StoreAddInput, 'space' | 'issuer' | 'invocation'> {}
 
-export interface StoreGetOk {
+export interface StoreInspectSuccess {
   spaces: Array<{ did: DID; insertedAt: string }>
 }
 
@@ -447,7 +443,7 @@ export interface UploadAddInput {
   invocation: UCANLink
 }
 
-export interface UploadGetOk {
+export interface UploadInspectSuccess {
   spaces: Array<{ did: DID; insertedAt: string }>
 }
 

packages/upload-api/src/upload.js

@@ -1,4 +1,5 @@
 import { uploadAddProvider } from './upload/add.js'
+import { uploadGetProvider } from './upload/get.js'
 import { uploadListProvider } from './upload/list.js'
 import { uploadRemoveProvider } from './upload/remove.js'
 import * as API from './types.js'
@@ -9,6 +10,7 @@ import * as API from './types.js'
 export function createService(context) {
   return {
     add: uploadAddProvider(context),
+    get: uploadGetProvider(context),
     list: uploadListProvider(context),
     remove: uploadRemoveProvider(context),
   }

packages/upload-api/src/upload/get.js

@@ -0,0 +1,29 @@
+import * as Server from '@ucanto/server'
+import * as Upload from '@web3-storage/capabilities/upload'
+import * as API from '../types.js'
+
+/**
+ * @param {API.UploadServiceContext} context
+ * @returns {API.ServiceMethod<API.UploadGet, API.UploadGetSuccess, API.UploadGetFailure>}
+ */
+export function uploadGetProvider(context) {
+  return Server.provide(Upload.get, async ({ capability }) => {
+    const { root } = capability.nb
+    if (!root) {
+      return Server.fail('nb.root must be set')
+    }
+    const space = Server.DID.parse(capability.with).did()
+    const res = await context.uploadTable.get(space, root)
+    if (!res) {
+      return {
+        error: {
+          name: 'UploadNotFound',
+          message: 'Upload not found',
+        },
+      }
+    }
+    return {
+      ok: res,
+    }
+  })
+}