diff --git a/filecoin/functions/handle-filecoin-submit-message.js b/filecoin/functions/handle-filecoin-submit-message.js index 6b744800..61a91206 100644 --- a/filecoin/functions/handle-filecoin-submit-message.js +++ b/filecoin/functions/handle-filecoin-submit-message.js @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/serverless' import * as storefrontEvents from '@web3-storage/filecoin-api/storefront/events' import { createPieceTable } from '../store/piece.js' -import { createDataStore, composeDataStoresWithOrderedStream } from '../store/data.js' +import { useContentStore } from '../store/content.js' import { decodeMessage } from '../queue/filecoin-submit-queue.js' import { mustGetEnv } from './utils.js' @@ -14,7 +14,6 @@ Sentry.AWSLambda.init({ }) const AWS_REGION = process.env.AWS_REGION || 'us-west-2' -const R2_REGION = process.env.R2_REGION || 'auto' /** * Get EventRecord from the SQS Event triggering the handler. @@ -38,24 +37,12 @@ async function handleFilecoinSubmitMessage (sqsEvent) { // create context const { pieceTableName, - s3BucketName, - r2BucketName, - r2BucketEndpoint, - r2BucketAccessKeyId, - r2BucketSecretAccessKey + contentStoreHttpEndpoint + } = getEnv() const context = { pieceStore: createPieceTable(AWS_REGION, pieceTableName), - dataStore: composeDataStoresWithOrderedStream( - createDataStore(AWS_REGION, s3BucketName), - createDataStore(R2_REGION, r2BucketName, { - endpoint: r2BucketEndpoint, - credentials: { - accessKeyId: r2BucketAccessKeyId, - secretAccessKey: r2BucketSecretAccessKey, - }, - }) - ) + contentStore: useContentStore(contentStoreHttpEndpoint) } const { ok, error } = await storefrontEvents.handleFilecoinSubmitMessage(context, record) @@ -78,12 +65,7 @@ async function handleFilecoinSubmitMessage (sqsEvent) { function getEnv () { return { pieceTableName: mustGetEnv('PIECE_TABLE_NAME'), - // carpark buckets - CAR file bytes may be found here with keys like {cid}/{cid}.car - s3BucketName: mustGetEnv('STORE_BUCKET_NAME'), - r2BucketName: mustGetEnv('R2_CARPARK_BUCKET_NAME'), - r2BucketEndpoint: mustGetEnv('R2_ENDPOINT'), - r2BucketAccessKeyId: mustGetEnv('R2_ACCESS_KEY_ID'), - r2BucketSecretAccessKey: mustGetEnv('R2_SECRET_ACCESS_KEY'), + contentStoreHttpEndpoint: new URL(mustGetEnv('CONTENT_STORE_HTTP_ENDPOINT')) } } diff --git a/filecoin/package.json b/filecoin/package.json index 149e5a8f..c7cb3e81 100644 --- a/filecoin/package.json +++ b/filecoin/package.json @@ -17,8 +17,8 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@web3-storage/data-segment": "^5.1.0", - "@web3-storage/filecoin-api": "^5.0.0", - "@web3-storage/filecoin-client": "^3.3.1", + "@web3-storage/filecoin-api": "^6.0.1", + "@web3-storage/filecoin-client": "^3.3.3", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0", "multiformats": "^13.1.0", "p-retry": "^6.2.0", @@ -36,6 +36,7 @@ }, "eslintConfig": { "rules": { + "unicorn/no-useless-undefined": "off", "unicorn/prefer-array-flat": "off", "unicorn/prefer-spread": "off" } diff --git a/filecoin/store/content.js b/filecoin/store/content.js new file mode 100644 index 00000000..b45141c6 --- /dev/null +++ b/filecoin/store/content.js @@ -0,0 +1,60 @@ +import { StoreOperationFailed, RecordNotFound } from '@web3-storage/filecoin-api/errors' +import pRetry from 'p-retry' + +/** + * @typedef {import('multiformats').UnknownLink} UnknownLink + * @typedef {import('@web3-storage/filecoin-api/storefront/api').ContentStore} ContentStore + */ + +/** + * @param {URL} storeHttpEndpoint + * @returns {ContentStore} + */ +export const useContentStore = (storeHttpEndpoint) => { + return { + /** + * Stream Blob bytes for a given CID. + * + * @param {import('@ucanto/interface').UnknownLink} cid + */ + stream: async (cid) => { + // create URL for the link to be fetched + const getUrl = new URL(`/${cid.toString()}`, storeHttpEndpoint) + + // Retry a few times as it looks like R2 sometimes takes a bit to make it available + let res + try { + res = await pRetry((async () => { + const fetchRes = await fetch(getUrl, { + // Follow potential redirects + redirect: 'follow', + }) + if (fetchRes.status === 404) { + throw new RecordNotFound(`blob ${cid.toString()} not found in store`) + } else if (fetchRes.status > 299 || !fetchRes.body) { + throw new StoreOperationFailed(fetchRes.statusText) + } + return fetchRes + }), { retries: 5 }) + } catch (err) { + /** @type {RecordNotFound | StoreOperationFailed} */ + // @ts-ignore + const error = err + return { + error + } + } + + // To satisfy typescript + if (!res.body) { + return { + error: new StoreOperationFailed(res.statusText) + } + } + + return { + ok: res.body + } + } + } +} diff --git a/filecoin/store/data.js b/filecoin/store/data.js deleted file mode 100644 index dc31db32..00000000 --- a/filecoin/store/data.js +++ /dev/null @@ -1,161 +0,0 @@ -import { - S3Client, - GetObjectCommand, - PutObjectCommand -} from '@aws-sdk/client-s3' -import * as CAR from '@ucanto/transport/car' -import { sha256 } from 'multiformats/hashes/sha2' -import { CID } from 'multiformats/cid' -import * as raw from 'multiformats/codecs/raw' -import { CarWriter } from '@ipld/car' -import pRetry from 'p-retry' -import { StoreOperationFailed, RecordNotFound } from '@web3-storage/filecoin-api/errors' - -/** - * Abstraction layer with Factory to perform operations on bucket storing - * data receipts. - * - * @param {string} region - * @param {string} bucketName - * @param {import('@aws-sdk/client-s3').ServiceInputTypes} [options] - */ -export function createDataStore(region, bucketName, options = {}) { - const s3client = new S3Client({ - region, - ...options, - }) - return useDataStore(s3client, bucketName) -} - -/** - * @param {S3Client} s3client - * @param {string} bucketName - * @returns {import('@web3-storage/filecoin-api/storefront/api').DataStore} - */ -export const useDataStore = (s3client, bucketName) => { - return { - // Only used for testing storing a CAR - // until we hook up claims to look for data - put: async (bytes) => { - const hash = await sha256.digest(bytes) - const root = CID.create(1, raw.code, hash) - - const { writer, out } = CarWriter.create(root) - writer.put({ cid: root, bytes }) - writer.close() - - const chunks = [] - for await (const chunk of out) { - chunks.push(chunk) - } - const blob = new Blob(chunks) - const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - - const putCmd = new PutObjectCommand({ - Bucket: bucketName, - Key: `${cid.toString()}/${cid.toString()}.car`, - Body: bytes - }) - await s3client.send(putCmd) - - return { - ok: {} - } - }, - /** - * Stream Blob bytes for a given invocation. - */ - /** - * - * @param {import('@ucanto/interface').UnknownLink} cid - */ - // @ts-expect-error aws Readable stream types are not good - stream: async (cid) => { - // TODO: probably get from Roundabout from R2 when location claims? - const getObjectCmd = new GetObjectCommand({ - Bucket: bucketName, - Key: `${cid.toString()}/${cid.toString()}.car`, - }) - let res - - try { - res = await pRetry(() => s3client.send(getObjectCmd), { retries: 3 }) - } catch (/** @type {any} */ error) { - if (error?.$metadata?.httpStatusCode === 404) { - return { - error: new RecordNotFound(`blob ${cid.toString()} not found in store`) - } - } - return { - error: new StoreOperationFailed(error.message) - } - } - - const stream = res.Body - if (!stream) { - return { - error: new RecordNotFound(`blob ${cid.toString()} not found in store`) - } - } - - return { - ok: stream - } - }, - has: async () => { - return { - error: new StoreOperationFailed('no blob should checked by storefront') - } - } - } -} - -/** - * compose many data stores. - * store#stream will check stores in order until 0-1 `ok` result is found. - * - * @param {import('@web3-storage/filecoin-api/storefront/api').DataStore} dataStore - * @param {Array} moreDataStores - * @returns {import('@web3-storage/filecoin-api/storefront/api').DataStore} - */ -export function composeDataStoresWithOrderedStream(dataStore, ...moreDataStores) { - return { - ...dataStore, - stream: composeSome(dataStore.stream, ...moreDataStores.map(s => s.stream.bind(s))), - } -} - -/** - * @typedef {AsyncIterable} Rec - * @typedef {import('@web3-storage/filecoin-api/types').StoreGetError} StoreGetError - * @typedef {import('@ucanto/interface').Result} Result - */ - -/** - * compose async functions that return Promise>. - * The returned function will have the same signature, - * but will try the composed functions in order until one (or none) returns 'ok'. - * - * @template T - * @param {Array<(e: T) => Promise>} streamFunctions - * - */ -function composeSome(...streamFunctions) { - /** - * @param {T} e - */ - return async function (e) { - /** @type {Result | undefined} */ - let result - for (const stream of streamFunctions) { - result = await stream(e) - if (result.ok) { - return result - } - } - if (result === undefined) { - throw new Error('no result received') - } - return result - } -} \ No newline at end of file diff --git a/filecoin/test/helpers/content-store.js b/filecoin/test/helpers/content-store.js new file mode 100644 index 00000000..63780198 --- /dev/null +++ b/filecoin/test/helpers/content-store.js @@ -0,0 +1,113 @@ +import { useContentStore } from '../../store/content.js' +import http from 'http' +import { CID } from 'multiformats' +import { sha256 } from 'multiformats/hashes/sha2' +import * as raw from 'multiformats/codecs/raw' + +/** + * @typedef {import('multiformats').UnknownLink} UnknownLink + * @typedef {import('@web3-storage/filecoin-api/test/types').TestContentStore} TestContentStoreInterface + * @typedef {import('@web3-storage/filecoin-api/storefront/api').ContentStore} ContentStore + */ + +/** + * Test content store implementation. + * Enables tests imported from `filecoin-api/storefront` to Put content into the store before adding + * messages to submit queue to have Pieces derived from content bytes so that PieceCID provided is checked. + * A HTTP Server is created to better emulate the production setup using roundabout under the hood. + * + * @implements {TestContentStoreInterface} + */ +export class TestContentStore { + static async activate() { + const contentStorage = new Map() + // Create Content Store HTTP Server to simulate Roundabout + const server = http.createServer(async (request, response) => { + if (request.method === 'GET') { + const { pathname } = new URL(request.url || '/', url) + const blobBytes = contentStorage.get(pathname.replaceAll('/', '')) + if (!blobBytes) { + response.writeHead(404) + } else { + response.writeHead(200) + response.write(blobBytes) + } + } else { + response.writeHead(405) + } + + response.end() + // otherwise it keep connection lingering + response.destroy() + }) + await new Promise((resolve) => server.listen(resolve)) + + // @ts-ignore - this is actually what it returns on http + const port = server.address().port + const url = new URL(`http://localhost:${port}`) + + return new TestContentStore( + useContentStore(url), + server, + contentStorage + ) + } + + /** + * @returns {Promise} + */ + async deactivate() { + const { server } = this + if (server) { + await new Promise((resolve, reject) => { + // does not exist in node 16 + if (typeof server.closeAllConnections === 'function') { + server.closeAllConnections() + } + + server.close((error) => { + if (error) { + reject(error) + } else { + resolve(undefined) + } + }) + }) + } + } + + /** + * @param {ContentStore} contentStore + * @param {import('http').Server} server + * @param {Map} contentStorage + */ + constructor(contentStore, server, contentStorage) { + this.server = server + this.contentStore = contentStore + this.contentStorage = contentStorage + } + + /** + * + * @param {import('@ucanto/interface').UnknownLink} cid + */ + async stream (cid) { + return await this.contentStore.stream(cid) + } + + /** + * @param {Uint8Array} bytes + */ + async put (bytes) { + // Put raw CID to content Storage, as this is going to be requested in tests this way + // Content Storage in production may not how to differentiate between Blob, CAR, etc. + const hash = await sha256.digest(bytes) + const cid = CID.create(1, raw.code, hash) + this.contentStorage.set(cid.toString(), bytes) + + return { + ok: {} + } + } +} + diff --git a/filecoin/test/helpers/service-context.js b/filecoin/test/helpers/service-context.js index 3700383c..f9fde5b3 100644 --- a/filecoin/test/helpers/service-context.js +++ b/filecoin/test/helpers/service-context.js @@ -1,8 +1,6 @@ import * as CAR from '@ucanto/transport/car' import pRetry from 'p-retry' -import { - PutObjectCommand, -} from '@aws-sdk/client-s3' +import { PutObjectCommand } from '@aws-sdk/client-s3' import { createBucket } from './resources.js' import { createDynamoTable } from './tables.js' @@ -12,10 +10,10 @@ import { encodeAgentMessage } from './ucan.js' import { pieceTableProps } from '../../store/index.js' // store clients -import { useDataStore as createDataStoreClient } from '../../store/data.js' import { usePieceTable as createPieceStoreClient } from '../../store/piece.js' import { useTaskStore as createTaskStoreClient } from '../../store/task.js' import { useReceiptStore as createReceiptStoreClient } from '../../store/receipt.js' +import { TestContentStore } from './content-store.js' // queue clients import { createClient as createPieceOfferQueueClient } from '../../queue/piece-offer-queue.js' @@ -27,17 +25,18 @@ import { createClient as createFilecoinSubmitQueueClient } from '../../queue/fil export async function getStores (ctx) { const { dynamoClient, s3Client } = ctx const pieceStore = await createDynamoTable(dynamoClient, pieceTableProps) - const [ invocationBucketName, workflowBucketName, dataStoreBucketName ] = await Promise.all([ - createBucket(s3Client), + const [ invocationBucketName, workflowBucketName ] = await Promise.all([ createBucket(s3Client), createBucket(s3Client), ]) + const testContentStore = await TestContentStore.activate() return { pieceStore: createPieceStoreClient(dynamoClient, pieceStore), taskStore: getTaskStoreClient(s3Client, invocationBucketName, workflowBucketName), receiptStore: getReceiptStoreClient(s3Client, invocationBucketName, workflowBucketName), - dataStore: createDataStoreClient(s3Client, dataStoreBucketName) + contentStore: testContentStore.contentStore, + testContentStore } } diff --git a/package-lock.json b/package-lock.json index b27e57b9..3996d00e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,9 +37,9 @@ "@ucanto/validator": "^9.0.2", "@web-std/blob": "^3.0.4", "@web-std/fetch": "^4.1.0", - "@web3-storage/data-segment": "5.0.0", - "@web3-storage/filecoin-client": "3.0.1", - "@web3-storage/w3up-client": "^12.4.1", + "@web3-storage/data-segment": "^5.1.0", + "@web3-storage/filecoin-client": "^3.3.3", + "@web3-storage/w3up-client": "^12.5.3", "ava": "^4.3.3", "chalk": "4.1.2", "constructs": "10.3.0", @@ -169,8 +169,8 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@web3-storage/data-segment": "^5.1.0", - "@web3-storage/filecoin-api": "^5.0.0", - "@web3-storage/filecoin-client": "^3.3.1", + "@web3-storage/filecoin-api": "^6.0.1", + "@web3-storage/filecoin-client": "^3.3.3", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0", "multiformats": "^13.1.0", "p-retry": "^6.2.0", @@ -187,38 +187,6 @@ "testcontainers": "^10.7.1" } }, - "filecoin/node_modules/@web3-storage/data-segment": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.1.0.tgz", - "integrity": "sha512-FYdmtKvNiVz+maZ++k4PdD43rfJW5DeagLpstq2y84CyOKNRBWbHLCZ/Ec5zT9iGI+0WgsCGbpC/WlG0jlrnhA==", - "dependencies": { - "@ipld/dag-cbor": "^9.0.5", - "multiformats": "^11.0.2", - "sync-multihash-sha2": "^1.0.0" - } - }, - "filecoin/node_modules/@web3-storage/data-segment/node_modules/multiformats": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", - "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } - }, - "filecoin/node_modules/@web3-storage/filecoin-client": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-client/-/filecoin-client-3.3.1.tgz", - "integrity": "sha512-GLJW32stM0zhW6Tv9RX2QhXNaJpa3kBsNYyDeYn/VWqTHEhenruSdtseWBnyCi1Y3NGQ2xL/JojOCD94EXA4Yw==", - "dependencies": { - "@ipld/dag-ucan": "^3.4.0", - "@ucanto/client": "^9.0.1", - "@ucanto/core": "^10.0.1", - "@ucanto/interface": "^10.0.1", - "@ucanto/transport": "^9.1.1", - "@web3-storage/capabilities": "^13.2.1" - } - }, "filecoin/node_modules/uint8arrays": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-4.0.6.tgz", @@ -5635,9 +5603,9 @@ } }, "node_modules/@web3-storage/access": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/@web3-storage/access/-/access-18.3.1.tgz", - "integrity": "sha512-NMNBqCcw+mfabrpRqoDWKmWfkioTCTClvSkb0vCpTb0yA/yntkJYG70QIaGdtoHV2Kz1gk4Hj1x9th5nHIKSpA==", + "version": "18.3.2", + "resolved": "https://registry.npmjs.org/@web3-storage/access/-/access-18.3.2.tgz", + "integrity": "sha512-Ry+lMzWkXnr7UXSWWwSBTlbTDfLAfBiwZvocLvrTe1BdBPMI/eQfbI+SNuvMdar0cvJLaLspjkK/IVADsdlBVg==", "dependencies": { "@ipld/car": "^5.1.1", "@ipld/dag-ucan": "^3.4.0", @@ -5648,7 +5616,7 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@ucanto/validator": "^9.0.2", - "@web3-storage/capabilities": "^13.3.1", + "@web3-storage/capabilities": "^14.0.2", "@web3-storage/did-mailto": "^2.1.0", "bigint-mod-arith": "^3.1.2", "conf": "11.0.2", @@ -5659,6 +5627,52 @@ "uint8arrays": "^4.0.6" } }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/capabilities": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-14.0.2.tgz", + "integrity": "sha512-0BTzzn60S7eC2xwZjla3v2SNiyxSuVYD2bAokHuO4ztfi0O7L76R2pVDpOI67ZnIL+Cl3FX022NKt+qLxFIoSg==", + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" + } + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/capabilities/node_modules/multiformats": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/capabilities/node_modules/uint8arrays": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", + "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/data-segment": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", + "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.5", + "multiformats": "^11.0.2", + "sync-multihash-sha2": "^1.0.0" + } + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/data-segment/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@web3-storage/access/node_modules/multiformats": { "version": "12.1.3", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", @@ -5669,9 +5683,9 @@ } }, "node_modules/@web3-storage/capabilities": { - "version": "13.3.1", - "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-13.3.1.tgz", - "integrity": "sha512-oiQVsSuT4sxOHrYZz05LTCSV3xfT0ae5sixhvEZZaQAwzpv4xPBI0e5ro1SVNV/8kpCzRT2bq3VY4QlN1EfK/Q==", + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-15.0.0.tgz", + "integrity": "sha512-q371HIdR4MoSAjv7WnHNzjr6PRSrg3DC4/uZNtcf6NA/g2scTCvyEu+cWYPL8KPWFAl7zUdSo/ZkES9BObf8+Q==", "dependencies": { "@ucanto/core": "^10.0.1", "@ucanto/interface": "^10.0.1", @@ -5692,7 +5706,7 @@ "sync-multihash-sha2": "^1.0.0" } }, - "node_modules/@web3-storage/capabilities/node_modules/multiformats": { + "node_modules/@web3-storage/capabilities/node_modules/@web3-storage/data-segment/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", @@ -5709,11 +5723,6 @@ "multiformats": "^13.0.0" } }, - "node_modules/@web3-storage/capabilities/node_modules/uint8arrays/node_modules/multiformats": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", - "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" - }, "node_modules/@web3-storage/content-claims": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-3.2.1.tgz", @@ -5736,10 +5745,9 @@ } }, "node_modules/@web3-storage/data-segment": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.0.0.tgz", - "integrity": "sha512-5CbElsxec2DsKhEHEh3XRGISAyna+bCjKjjvFrLcYyXLCaiSt/nF3ypcllxwjpE4newMUArymGKGzzZnRWL2kg==", - "dev": true, + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.1.0.tgz", + "integrity": "sha512-FYdmtKvNiVz+maZ++k4PdD43rfJW5DeagLpstq2y84CyOKNRBWbHLCZ/Ec5zT9iGI+0WgsCGbpC/WlG0jlrnhA==", "dependencies": { "@ipld/dag-cbor": "^9.0.5", "multiformats": "^11.0.2", @@ -5750,7 +5758,6 @@ "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", - "dev": true, "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -5765,9 +5772,9 @@ } }, "node_modules/@web3-storage/filecoin-api": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-api/-/filecoin-api-5.0.0.tgz", - "integrity": "sha512-zKpZ9bv3EvjxH46QPVViLLu+VMxTxH64MtEgG0CZrTOlcwCXIvIgn6LsG58DdXihTZZggTL9PdpxQqPxJIEC5A==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-api/-/filecoin-api-6.0.1.tgz", + "integrity": "sha512-KfMochbXmHCO/uEjLSx7bemVXO4cx42RMFszHKD8wWtoKQcp1hdjw16dN1Ch+Rxc2gw5WL7qqQSFRNQ+HUC8Lg==", "dependencies": { "@ipld/dag-ucan": "^3.4.0", "@ucanto/client": "^9.0.1", @@ -5775,7 +5782,7 @@ "@ucanto/interface": "^10.0.1", "@ucanto/server": "^10.0.0", "@ucanto/transport": "^9.1.1", - "@web3-storage/capabilities": "^13.3.1", + "@web3-storage/capabilities": "^16.0.0", "@web3-storage/content-claims": "^4.0.4", "@web3-storage/data-segment": "^4.0.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0", @@ -5796,10 +5803,43 @@ "@ucanto/validator": "^9.0.1" } }, + "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/capabilities": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-16.0.0.tgz", + "integrity": "sha512-wCjLpYc6t8tFRZrF2k2vBteJDWzHkmQjoJG0Yy/fjA04IjNN48iVZaCMQIANHXZxDGlYRGxhwzDwl4dovAdSTQ==", + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" + } + }, + "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/capabilities/node_modules/@web3-storage/data-segment": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", + "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.5", + "multiformats": "^11.0.2", + "sync-multihash-sha2": "^1.0.0" + } + }, + "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/capabilities/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/content-claims": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-4.0.4.tgz", - "integrity": "sha512-zt5psR3SkLbPPHzGzhFXYSJEssDl/ELYbNhEez+tNZLZiagv3Vl0RSt+x3CFFgR5ovO6Zn+pLJJcMjpMiHw0Yw==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-4.0.5.tgz", + "integrity": "sha512-+WpCkTN8aRfUCrCm0kOMZad+FRnFymVDFvS6/+PJMPGP17cci1/c5lqYdrjFV+5MkhL+BkUJVtRTx02G31FHmQ==", "dependencies": { "@ucanto/client": "^9.0.1", "@ucanto/interface": "^10.0.0", @@ -5809,15 +5849,6 @@ "multiformats": "^12.0.1" } }, - "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/content-claims/node_modules/multiformats": { - "version": "12.1.3", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", - "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } - }, "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/data-segment": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-4.0.0.tgz", @@ -5828,7 +5859,7 @@ "sync-multihash-sha2": "^1.0.0" } }, - "node_modules/@web3-storage/filecoin-api/node_modules/multiformats": { + "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/data-segment/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", @@ -5837,78 +5868,82 @@ "npm": ">=7.0.0" } }, - "node_modules/@web3-storage/filecoin-client": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-client/-/filecoin-client-3.0.1.tgz", - "integrity": "sha512-8CCdZ1DDjufrxjzG5MhcBHuunuEWB53cpSszajWLEiKmRAhD+Lyaeucb6I4R4JJpjr6YYQg/heG+TDPEowVmSg==", - "dev": true, - "dependencies": { - "@ipld/dag-ucan": "^3.4.0", - "@ucanto/client": "^9.0.0", - "@ucanto/core": "^9.0.0", - "@ucanto/interface": "^9.0.0", - "@ucanto/transport": "^9.0.0", - "@web3-storage/capabilities": "^11.1.0" + "node_modules/@web3-storage/filecoin-api/node_modules/multiformats": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", + "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" } }, - "node_modules/@web3-storage/filecoin-client/node_modules/@ucanto/core": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-9.0.1.tgz", - "integrity": "sha512-SsYvKCO3FD27roTVcg8ASxnixjn+j96sPlijpVq1uBUxq7SmuNxNPYFZqpxXKj2R4gty/Oc8XTse12ebB9Kofg==", - "dev": true, + "node_modules/@web3-storage/filecoin-api/node_modules/uint8arrays": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", + "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", "dependencies": { - "@ipld/car": "^5.1.0", - "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-ucan": "^3.4.0", - "@ucanto/interface": "^9.0.0", - "multiformats": "^11.0.2" + "multiformats": "^13.0.0" } }, - "node_modules/@web3-storage/filecoin-client/node_modules/@ucanto/interface": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-9.0.0.tgz", - "integrity": "sha512-Y9185yj+CRNpT43EAHTe9MpskCgU9DyWvmYyLMMmF40w+ujp6EYy5JVI/gVjJAsh+2Y9ruvWHOF0M+21TnLQyg==", - "dev": true, + "node_modules/@web3-storage/filecoin-api/node_modules/uint8arrays/node_modules/multiformats": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" + }, + "node_modules/@web3-storage/filecoin-client": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-client/-/filecoin-client-3.3.3.tgz", + "integrity": "sha512-xFL8odr5PpTjQvpfw/4jphcm7ZvcBRMSKHn3ReEaVcFjxQL45Rojjleuq/QEdMwrNfsLCqqAxC54jk55o5/ERQ==", "dependencies": { "@ipld/dag-ucan": "^3.4.0", - "multiformats": "^11.0.2" + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/transport": "^9.1.1", + "@web3-storage/capabilities": "^16.0.0" } }, "node_modules/@web3-storage/filecoin-client/node_modules/@web3-storage/capabilities": { - "version": "11.4.1", - "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-11.4.1.tgz", - "integrity": "sha512-PjIewEg/T3wfNavxzsZZ5MpH2WBldNz94qOQOKg5iH/4UrS8SPWWGsJx/Tu760O+PFhpTFwvi5cHCtkb08OdAA==", - "dev": true, + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-16.0.0.tgz", + "integrity": "sha512-wCjLpYc6t8tFRZrF2k2vBteJDWzHkmQjoJG0Yy/fjA04IjNN48iVZaCMQIANHXZxDGlYRGxhwzDwl4dovAdSTQ==", "dependencies": { - "@ucanto/core": "^9.0.1", - "@ucanto/interface": "^9.0.0", - "@ucanto/principal": "^9.0.0", - "@ucanto/transport": "^9.0.0", - "@ucanto/validator": "^9.0.0", - "@web3-storage/data-segment": "^3.2.0" + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" } }, "node_modules/@web3-storage/filecoin-client/node_modules/@web3-storage/data-segment": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", - "dev": true, "dependencies": { "@ipld/dag-cbor": "^9.0.5", "multiformats": "^11.0.2", "sync-multihash-sha2": "^1.0.0" } }, - "node_modules/@web3-storage/filecoin-client/node_modules/multiformats": { + "node_modules/@web3-storage/filecoin-client/node_modules/@web3-storage/data-segment/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", - "dev": true, "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" } }, + "node_modules/@web3-storage/filecoin-client/node_modules/uint8arrays": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", + "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", + "dependencies": { + "multiformats": "^13.0.0" + } + }, "node_modules/@web3-storage/multipart-parser": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@web3-storage/multipart-parser/-/multipart-parser-1.0.0.tgz", @@ -5924,9 +5959,9 @@ } }, "node_modules/@web3-storage/upload-api": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-10.0.0.tgz", - "integrity": "sha512-aYC7iFv+WJt8rEDjkptZNvL03oo3g9xatm3PqcZPb9kOIIsoKrUjx1DUP3IsOZv3jTjeuhgGY+hzFWsg8ZII4Q==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-11.0.0.tgz", + "integrity": "sha512-qrVfTHi/PjdE6iuH9sxuk9tzPi3z0pGy8WLOvYYg5sburWUz3Ea621MMiMuOFkz2rdO2rokxz9OuzscP/nOXMA==", "dependencies": { "@ucanto/client": "^9.0.1", "@ucanto/interface": "^10.0.1", @@ -5934,11 +5969,11 @@ "@ucanto/server": "^10.0.0", "@ucanto/transport": "^9.1.1", "@ucanto/validator": "^9.0.2", - "@web3-storage/access": "^18.3.1", - "@web3-storage/capabilities": "^14.0.0", + "@web3-storage/access": "^18.3.2", + "@web3-storage/capabilities": "^15.0.0", "@web3-storage/content-claims": "^4.0.4", "@web3-storage/did-mailto": "^2.1.0", - "@web3-storage/filecoin-api": "^5.0.0", + "@web3-storage/filecoin-api": "^6.0.0", "multiformats": "^12.1.2", "p-retry": "^5.1.2", "uint8arrays": "^5.0.3" @@ -5963,20 +5998,6 @@ "@ucanto/validator": "^9.0.1" } }, - "node_modules/@web3-storage/upload-api/node_modules/@web3-storage/capabilities": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-14.0.0.tgz", - "integrity": "sha512-i57wEzIjBsz5iCdBJJZCCAN/j0Vknns2NDFEpV732Vo/VxW2PbYqVb0eqEKegDwxdAmkZwyTT6iZQfsebgK7hw==", - "dependencies": { - "@ucanto/core": "^10.0.1", - "@ucanto/interface": "^10.0.1", - "@ucanto/principal": "^9.0.1", - "@ucanto/transport": "^9.1.1", - "@ucanto/validator": "^9.0.2", - "@web3-storage/data-segment": "^3.2.0", - "uint8arrays": "^5.0.3" - } - }, "node_modules/@web3-storage/upload-api/node_modules/@web3-storage/content-claims": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-4.0.4.tgz", @@ -5990,25 +6011,6 @@ "multiformats": "^12.0.1" } }, - "node_modules/@web3-storage/upload-api/node_modules/@web3-storage/data-segment": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", - "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", - "dependencies": { - "@ipld/dag-cbor": "^9.0.5", - "multiformats": "^11.0.2", - "sync-multihash-sha2": "^1.0.0" - } - }, - "node_modules/@web3-storage/upload-api/node_modules/@web3-storage/data-segment/node_modules/multiformats": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", - "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } - }, "node_modules/@web3-storage/upload-api/node_modules/multiformats": { "version": "12.1.3", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", @@ -6047,24 +6049,24 @@ "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" }, "node_modules/@web3-storage/upload-client": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-13.0.1.tgz", - "integrity": "sha512-vfAtp1qIIgcoNQ21X/+jrHZjQU28af516YWWkYWbGV68dBzBd9c7r+8KjQwRDIV8kSl0Pmho6Dy1OWlhpZfE6g==", + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-13.2.2.tgz", + "integrity": "sha512-dtedxw9azYUG5LTbZ37CnjggCRIuSsi0nD8phpWjWN20v3pqlrMCCh4cwl0z1bDk/GnYmOxDBViE19paEaYPBw==", "dev": true, "dependencies": { "@ipld/car": "^5.2.2", "@ipld/dag-cbor": "^9.0.6", "@ipld/dag-ucan": "^3.4.0", "@ipld/unixfs": "^2.1.1", - "@ucanto/client": "^9.0.0", - "@ucanto/interface": "^9.0.0", - "@ucanto/transport": "^9.0.0", - "@web3-storage/capabilities": "^13.1.1", - "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0", + "@ucanto/client": "^9.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/transport": "^9.1.1", + "@web3-storage/capabilities": "^14.0.2", + "@web3-storage/data-segment": "^5.1.0", + "@web3-storage/filecoin-client": "^3.3.2", "ipfs-utils": "^9.0.14", "multiformats": "^12.1.2", "p-retry": "^5.1.2", - "parallel-transform-web": "^1.0.1", "varint": "^6.0.0" } }, @@ -6074,17 +6076,33 @@ "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==", "dev": true }, - "node_modules/@web3-storage/upload-client/node_modules/@ucanto/interface": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-9.0.0.tgz", - "integrity": "sha512-Y9185yj+CRNpT43EAHTe9MpskCgU9DyWvmYyLMMmF40w+ujp6EYy5JVI/gVjJAsh+2Y9ruvWHOF0M+21TnLQyg==", + "node_modules/@web3-storage/upload-client/node_modules/@web3-storage/capabilities": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-14.0.2.tgz", + "integrity": "sha512-0BTzzn60S7eC2xwZjla3v2SNiyxSuVYD2bAokHuO4ztfi0O7L76R2pVDpOI67ZnIL+Cl3FX022NKt+qLxFIoSg==", "dev": true, "dependencies": { - "@ipld/dag-ucan": "^3.4.0", - "multiformats": "^11.0.2" + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" } }, - "node_modules/@web3-storage/upload-client/node_modules/@ucanto/interface/node_modules/multiformats": { + "node_modules/@web3-storage/upload-client/node_modules/@web3-storage/capabilities/node_modules/@web3-storage/data-segment": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", + "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", + "dev": true, + "dependencies": { + "@ipld/dag-cbor": "^9.0.5", + "multiformats": "^11.0.2", + "sync-multihash-sha2": "^1.0.0" + } + }, + "node_modules/@web3-storage/upload-client/node_modules/@web3-storage/capabilities/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", @@ -6120,6 +6138,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@web3-storage/upload-client/node_modules/uint8arrays": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", + "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", + "dev": true, + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/@web3-storage/upload-client/node_modules/uint8arrays/node_modules/multiformats": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==", + "dev": true + }, "node_modules/@web3-storage/w3infra-billing": { "resolved": "billing", "link": true @@ -6149,48 +6182,40 @@ "link": true }, "node_modules/@web3-storage/w3up-client": { - "version": "12.4.1", - "resolved": "https://registry.npmjs.org/@web3-storage/w3up-client/-/w3up-client-12.4.1.tgz", - "integrity": "sha512-Ti4k5UScn5kdF0A5rz0WkDefD0LofUCUJNDGRYQFudw17FmaZHfQGSQvZ0CozYrCJg8K7JPTbU4/m3joWu0ufQ==", + "version": "12.5.3", + "resolved": "https://registry.npmjs.org/@web3-storage/w3up-client/-/w3up-client-12.5.3.tgz", + "integrity": "sha512-tpkjSU6amVDojiBRK6l43bIPQYcwx63T3uO3jNMMyOo9khzru+wBV2P0mTsMd1VkaBMdxo8ylzLdDPp8m2X5rQ==", "dev": true, "dependencies": { "@ipld/dag-ucan": "^3.4.0", - "@ucanto/client": "^9.0.0", - "@ucanto/core": "^9.0.1", - "@ucanto/interface": "^9.0.0", - "@ucanto/principal": "^9.0.0", - "@ucanto/transport": "^9.0.0", - "@web3-storage/access": "^18.2.0", - "@web3-storage/capabilities": "^13.1.1", + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@web3-storage/access": "^18.3.2", + "@web3-storage/capabilities": "^14.0.2", "@web3-storage/did-mailto": "^2.1.0", - "@web3-storage/filecoin-client": "^3.2.0", - "@web3-storage/upload-client": "^13.0.1" + "@web3-storage/filecoin-client": "^3.3.2", + "@web3-storage/upload-client": "^13.2.2" }, "engines": { "node": ">=18" } }, - "node_modules/@web3-storage/w3up-client/node_modules/@ucanto/core": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-9.0.1.tgz", - "integrity": "sha512-SsYvKCO3FD27roTVcg8ASxnixjn+j96sPlijpVq1uBUxq7SmuNxNPYFZqpxXKj2R4gty/Oc8XTse12ebB9Kofg==", - "dev": true, - "dependencies": { - "@ipld/car": "^5.1.0", - "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-ucan": "^3.4.0", - "@ucanto/interface": "^9.0.0", - "multiformats": "^11.0.2" - } - }, - "node_modules/@web3-storage/w3up-client/node_modules/@ucanto/interface": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-9.0.0.tgz", - "integrity": "sha512-Y9185yj+CRNpT43EAHTe9MpskCgU9DyWvmYyLMMmF40w+ujp6EYy5JVI/gVjJAsh+2Y9ruvWHOF0M+21TnLQyg==", + "node_modules/@web3-storage/w3up-client/node_modules/@web3-storage/capabilities": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-14.0.2.tgz", + "integrity": "sha512-0BTzzn60S7eC2xwZjla3v2SNiyxSuVYD2bAokHuO4ztfi0O7L76R2pVDpOI67ZnIL+Cl3FX022NKt+qLxFIoSg==", "dev": true, "dependencies": { - "@ipld/dag-ucan": "^3.4.0", - "multiformats": "^11.0.2" + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" } }, "node_modules/@web3-storage/w3up-client/node_modules/@web3-storage/data-segment": { @@ -6204,35 +6229,7 @@ "sync-multihash-sha2": "^1.0.0" } }, - "node_modules/@web3-storage/w3up-client/node_modules/@web3-storage/filecoin-client": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-client/-/filecoin-client-3.2.0.tgz", - "integrity": "sha512-4kSyXcN7jPAnpO2U8afheYBRJ4E/8aRJvCvPgHF+HZEtEaLHYuuQzU72Aro94qV0bm5ZRxXPNh6wRSlz/XZLlg==", - "dev": true, - "dependencies": { - "@ipld/dag-ucan": "^3.4.0", - "@ucanto/client": "^9.0.0", - "@ucanto/core": "^9.0.1", - "@ucanto/interface": "^9.0.0", - "@ucanto/transport": "^9.0.0", - "@web3-storage/capabilities": "^12.1.0" - } - }, - "node_modules/@web3-storage/w3up-client/node_modules/@web3-storage/filecoin-client/node_modules/@web3-storage/capabilities": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-12.1.0.tgz", - "integrity": "sha512-SlYdPqCokDHb55zlZOvh+n8uEMOrEU413Z1MzQ8HvULpbzfcEtGyOiDgrAhdNEZtPnWHqaUEtU7o829Yw2Ra5w==", - "dev": true, - "dependencies": { - "@ucanto/core": "^9.0.1", - "@ucanto/interface": "^9.0.0", - "@ucanto/principal": "^9.0.0", - "@ucanto/transport": "^9.0.0", - "@ucanto/validator": "^9.0.1", - "@web3-storage/data-segment": "^3.2.0" - } - }, - "node_modules/@web3-storage/w3up-client/node_modules/multiformats": { + "node_modules/@web3-storage/w3up-client/node_modules/@web3-storage/data-segment/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", @@ -6242,6 +6239,15 @@ "npm": ">=7.0.0" } }, + "node_modules/@web3-storage/w3up-client/node_modules/uint8arrays": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", + "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", + "dev": true, + "dependencies": { + "multiformats": "^13.0.0" + } + }, "node_modules/@whatwg-node/events": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/@whatwg-node/events/-/events-0.0.2.tgz", @@ -14572,12 +14578,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/parallel-transform-web": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parallel-transform-web/-/parallel-transform-web-1.0.1.tgz", - "integrity": "sha512-RtPU/7IuwPZ4ePcqoPxNCpjtaXYOkCVtnhh5tW3O78wy9jqVoV2hQHms17kUeu8DTYoOP+mykFLg2agwVKlwBw==", - "dev": true - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -18285,17 +18285,6 @@ "testcontainers": "^10.7.1" } }, - "roundabout/node_modules/@web3-storage/data-segment": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.1.0.tgz", - "integrity": "sha512-FYdmtKvNiVz+maZ++k4PdD43rfJW5DeagLpstq2y84CyOKNRBWbHLCZ/Ec5zT9iGI+0WgsCGbpC/WlG0jlrnhA==", - "dev": true, - "dependencies": { - "@ipld/dag-cbor": "^9.0.5", - "multiformats": "^11.0.2", - "sync-multihash-sha2": "^1.0.0" - } - }, "roundabout/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", @@ -18365,6 +18354,17 @@ "sync-multihash-sha2": "^1.0.0" } }, + "tools/node_modules/@web3-storage/data-segment": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.0.0.tgz", + "integrity": "sha512-5CbElsxec2DsKhEHEh3XRGISAyna+bCjKjjvFrLcYyXLCaiSt/nF3ypcllxwjpE4newMUArymGKGzzZnRWL2kg==", + "dev": true, + "dependencies": { + "@ipld/dag-cbor": "^9.0.5", + "multiformats": "^11.0.2", + "sync-multihash-sha2": "^1.0.0" + } + }, "tools/node_modules/multiformats": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", @@ -18409,10 +18409,10 @@ "@ucanto/transport": "^9.1.1", "@ucanto/validator": "^9.0.2", "@web-std/fetch": "^4.1.0", - "@web3-storage/access": "^18.3.1", - "@web3-storage/capabilities": "^14.0.0", + "@web3-storage/access": "^18.3.2", + "@web3-storage/capabilities": "^15.0.0", "@web3-storage/did-mailto": "^2.1.0", - "@web3-storage/upload-api": "^10.0.0", + "@web3-storage/upload-api": "^11.0.0", "multiformats": "^13.1.0", "nanoid": "^5.0.2", "preact": "^10.14.1", @@ -18470,47 +18470,6 @@ "web-streams-polyfill": "^3.1.1" } }, - "upload-api/node_modules/@web3-storage/capabilities": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-14.0.0.tgz", - "integrity": "sha512-i57wEzIjBsz5iCdBJJZCCAN/j0Vknns2NDFEpV732Vo/VxW2PbYqVb0eqEKegDwxdAmkZwyTT6iZQfsebgK7hw==", - "dependencies": { - "@ucanto/core": "^10.0.1", - "@ucanto/interface": "^10.0.1", - "@ucanto/principal": "^9.0.1", - "@ucanto/transport": "^9.1.1", - "@ucanto/validator": "^9.0.2", - "@web3-storage/data-segment": "^3.2.0", - "uint8arrays": "^5.0.3" - } - }, - "upload-api/node_modules/@web3-storage/capabilities/node_modules/uint8arrays": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.0.3.tgz", - "integrity": "sha512-6LBuKji28kHjgPJMkQ6GDaBb1lRwIhyOYq6pDGwYMoDPfImE9SkuYENVmR0yu9yGgs2clHUSY9fKDukR+AXfqQ==", - "dependencies": { - "multiformats": "^13.0.0" - } - }, - "upload-api/node_modules/@web3-storage/data-segment": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.2.0.tgz", - "integrity": "sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==", - "dependencies": { - "@ipld/dag-cbor": "^9.0.5", - "multiformats": "^11.0.2", - "sync-multihash-sha2": "^1.0.0" - } - }, - "upload-api/node_modules/@web3-storage/data-segment/node_modules/multiformats": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", - "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } - }, "upload-api/node_modules/nanoid": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.6.tgz", diff --git a/package.json b/package.json index 6226e9b7..61689840 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,9 @@ "@ucanto/validator": "^9.0.2", "@web-std/blob": "^3.0.4", "@web-std/fetch": "^4.1.0", - "@web3-storage/data-segment": "5.0.0", - "@web3-storage/filecoin-client": "3.0.1", - "@web3-storage/w3up-client": "^12.4.1", + "@web3-storage/data-segment": "^5.1.0", + "@web3-storage/filecoin-client": "^3.3.3", + "@web3-storage/w3up-client": "^12.5.3", "ava": "^4.3.3", "chalk": "4.1.2", "constructs": "10.3.0", diff --git a/sst.config.ts b/sst.config.ts index c95239b9..ee52b140 100644 --- a/sst.config.ts +++ b/sst.config.ts @@ -41,6 +41,7 @@ export default { app.stack(BusStack) app.stack(UploadDbStack) + app.stack(RoundaboutStack) app.stack(BillingDbStack) app.stack(CarparkStack) app.stack(UcanInvocationStack) @@ -50,7 +51,6 @@ export default { app.stack(UploadApiStack) app.stack(ReplicatorStack) app.stack(UcanFirehoseStack) - app.stack(RoundaboutStack) // tags let us discover all the aws resource costs incurred by this app // see: https://docs.sst.dev/advanced/tagging-resources diff --git a/stacks/filecoin-stack.js b/stacks/filecoin-stack.js index 00b0437a..24aa3f08 100644 --- a/stacks/filecoin-stack.js +++ b/stacks/filecoin-stack.js @@ -11,6 +11,7 @@ import { BusStack } from './bus-stack.js' import { CarparkStack } from './carpark-stack.js' import { UploadDbStack } from './upload-db-stack.js' import { UcanInvocationStack } from './ucan-invocation-stack.js' +import { RoundaboutStack } from './roundabout-stack.js' import { setupSentry, getEnv, getCdkNames, getCustomDomain, getEventSourceConfig } from './config.js' import { CARPARK_EVENT_BRIDGE_SOURCE_EVENT } from '../carpark/event-bus/source.js' import { Status } from '../filecoin/store/piece.js' @@ -43,6 +44,7 @@ export function FilecoinStack({ stack, app }) { const { pieceTable, privateKey, contentClaimsPrivateKey, adminMetricsTable } = use(UploadDbStack) // Get UCAN store references const { workflowBucket, invocationBucket, ucanStream } = use(UcanInvocationStack) + const { roundaboutApiUrl } = use(RoundaboutStack) /** * 1st processor queue - filecoin submit @@ -64,13 +66,7 @@ export function FilecoinStack({ stack, app }) { handler: 'filecoin/functions/handle-filecoin-submit-message.main', environment : { PIECE_TABLE_NAME: pieceTable.tableName, - // Setup both buckets - STORE_BUCKET_NAME: carparkBucket.bucketName, - R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID ?? '', - R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY ?? '', - R2_REGION: process.env.R2_REGION ?? '', - R2_ENDPOINT: process.env.R2_ENDPOINT ?? '', - R2_CARPARK_BUCKET_NAME: process.env.R2_CARPARK_BUCKET_NAME ?? '', + CONTENT_STORE_HTTP_ENDPOINT: roundaboutApiUrl }, permissions: [pieceTable], // piece is computed in this lambda diff --git a/stacks/roundabout-stack.js b/stacks/roundabout-stack.js index fe6eb877..74e4629d 100644 --- a/stacks/roundabout-stack.js +++ b/stacks/roundabout-stack.js @@ -42,4 +42,8 @@ export function RoundaboutStack({ stack, app }) { ApiEndpoint: api.url, CustomDomain: customDomain ? `https://${customDomain.domainName}` : 'Set HOSTED_ZONE in env to deploy to a custom domain' }) + + return { + roundaboutApiUrl: api.url + } } diff --git a/test/filecoin.test.js b/test/filecoin.test.js index b3ee194f..d04ee208 100644 --- a/test/filecoin.test.js +++ b/test/filecoin.test.js @@ -51,7 +51,7 @@ test('w3filecoin integration flow', async t => { } console.log('uploading 4 files') - const uploads = await Promise.all(Array.from({ length: 4 }).map(async () => { + const uploads = await Promise.all(Array.from({ length: 6 }).map(async () => { const file = await randomFile(1024) /** @type {{ content: CARLink, piece: PieceLink}[]} */ diff --git a/upload-api/package.json b/upload-api/package.json index 436c1e1f..86d2277c 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -22,10 +22,10 @@ "@ucanto/transport": "^9.1.1", "@ucanto/validator": "^9.0.2", "@web-std/fetch": "^4.1.0", - "@web3-storage/access": "^18.3.1", - "@web3-storage/capabilities": "^14.0.0", + "@web3-storage/access": "^18.3.2", + "@web3-storage/capabilities": "^15.0.0", "@web3-storage/did-mailto": "^2.1.0", - "@web3-storage/upload-api": "^10.0.0", + "@web3-storage/upload-api": "^11.0.0", "multiformats": "^13.1.0", "nanoid": "^5.0.2", "preact": "^10.14.1",