From 100aa85fa84197d1e7c911df16507cccb415d171 Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Tue, 29 Oct 2024 18:23:36 +0100 Subject: [PATCH] Add publish rsr to smart contract (#383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add `rsrContract` * add `storacha` * persist round `cids` and `details` * add publish rsr (untested) * add git commit * update env var name * refactor `createStorachaClient()` * update schema * hide `GIT_COMMIT` in scope * `gitCommit` -> `sparkEvaluateVersion` * `postEvaluate` -> `prepareAcceptedRetrievalTaskMeasurementsCommitment` * `round.cids` -> `round.measurementCommitments` * fix `.total` * `publish_rsr_rounds` -> `unpublished_rsr_rounds` * update contract * update contract * fix rsr calculation * share logic for building retrieval stats * rename method to match contract * Update lib/publish-rsr.js Co-authored-by: Miroslav Bajtoš * get date string from db * fix deletion logic * format * always publish oldest publishable date * update column to match smart contract * add contract address * consistent naming * measurement_commitments -> _batches * update schema (wip) * upload dag-json * unify terminology * upload round details to storacha * minerId -> providerId * update schema * refactor * consistent naming * consistent naming * doc * add passing tests * add passing test * improve error message * add passing test * move stuff around * fix lint * add passing test * add passing test * fix * add passing test * consistent naming * add passing test * improve car/cid tests * fix query with test * docs: fix CID * add test and fixes * add passing test * fix lint * add passing test * fix test name * refactor * add passing test * add test and fix * clean up * add passing tests * add passing test * add passing test * consistent naming * refactor * fix lint --------- Co-authored-by: Miroslav Bajtoš --- .github/workflows/ci.yml | 2 +- Dockerfile | 3 + README.md | 2 +- bin/cancel-pending-tx.js | 4 +- bin/dry-run.js | 9 +- bin/fetch-recent-miner-measurements.js | 7 +- bin/spark-evaluate.js | 50 +- index.js | 6 +- lib/car.js | 37 + lib/{ie-contract.js => contracts.js} | 21 +- lib/evaluate.js | 13 +- lib/preprocess.js | 1 + lib/provider-retrieval-result-stats.js | 134 +++ lib/public-stats.js | 20 +- lib/round.js | 2 + lib/rsrContract.json | 1 + lib/storacha.js | 21 + ...ublish-provider-retrieval-result-stats.sql | 10 + package-lock.json | 1030 ++++++++++++++++- package.json | 5 + test/car.test.js | 34 + test/evaluate.js | 58 +- test/helpers/test-data.js | 2 + test/preprocess.js | 1 + test/provider-retrieval-result-stats.test.js | 489 ++++++++ 25 files changed, 1857 insertions(+), 105 deletions(-) create mode 100644 lib/car.js rename lib/{ie-contract.js => contracts.js} (58%) create mode 100644 lib/provider-retrieval-result-stats.js create mode 100644 lib/rsrContract.json create mode 100644 lib/storacha.js create mode 100644 migrations/020.do.publish-provider-retrieval-result-stats.sql create mode 100644 test/car.test.js create mode 100644 test/provider-retrieval-result-stats.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb00051..ba61e9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy --remote-only + - run: flyctl deploy --remote-only —-build-arg GIT_COMMIT=$(git rev-parse HEAD) env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - if: failure() diff --git a/Dockerfile b/Dockerfile index d13f94c..1af407f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,9 @@ ENV NODE_ENV=production # 4096MB available memory - 200MB for anything else ENV NODE_OPTIONS="--max-old-space-size=3896" +ARG GIT_COMMIT +ENV GIT_COMMIT=$GIT_COMMIT + # Throw-away build stage to reduce size of final image FROM base AS build diff --git a/README.md b/README.md index 6b2abbe..566a8c0 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ You can perform a dry-run evaluation of a given Meridan round using the script ` You can optionally specify the smart contract address, round index and list of CIDs of measurements to load. For example, run the following command to evaluate round `273` of the Meridian version `0x3113b83ccec38a18df936f31297de490485d7b2e` with measurements from CID -`bafybeie5rekb2jox77ow64wjjd2bjdsp6d3yeivhzzd234hnbpscfjarv4z`: +`bafybeie5rekb2jox77ow64wjjd2bjdsp6d3yeivhzzd234hnbpscfjarv4`: ```shell node bin/dry-run.js \ diff --git a/bin/cancel-pending-tx.js b/bin/cancel-pending-tx.js index b5928d3..b3d7e33 100644 --- a/bin/cancel-pending-tx.js +++ b/bin/cancel-pending-tx.js @@ -6,7 +6,7 @@ import { ethers } from 'ethers' import { CoinType, newDelegatedEthAddress } from '@glif/filecoin-address' import pRetry from 'p-retry' -import { createMeridianContract } from '../lib/ie-contract.js' +import { createContracts } from '../lib/contracts.js' const { WALLET_SEED @@ -17,7 +17,7 @@ const [, , tx] = process.argv assert(WALLET_SEED, 'WALLET_SEED required') assert(tx, 'Transaction hash must be provided as the first argument') -const { provider } = await createMeridianContract() +const { provider } = createContracts() const signer = ethers.Wallet.fromPhrase(WALLET_SEED, provider) const walletDelegatedAddress = newDelegatedEthAddress(/** @type {any} */(signer.address), CoinType.MAIN).toString() diff --git a/bin/dry-run.js b/bin/dry-run.js index 62f677c..b36eb47 100644 --- a/bin/dry-run.js +++ b/bin/dry-run.js @@ -12,7 +12,7 @@ import path from 'node:path' import { fileURLToPath } from 'node:url' import pg from 'pg' import { RoundData } from '../lib/round.js' -import { createMeridianContract } from '../lib/ie-contract.js' +import { createContracts } from '../lib/contracts.js' import * as SparkImpactEvaluator from '@filecoin-station/spark-impact-evaluator' /** @typedef {import('../lib/preprocess.js').Measurement} Measurement */ @@ -142,7 +142,8 @@ const { ignoredErrors } = await evaluate({ setScores, logger: console, recordTelemetry, - createPgClient + createPgClient, + prepareProviderRetrievalResultStats: async () => {} }) console.log('Duration: %sms', Date.now() - started) @@ -213,7 +214,7 @@ async function fetchMeasurementsAddedEvents (contractAddress, roundIndex) { * @param {bigint} roundIndex */ async function fetchMeasurementsAddedFromChain (contractAddress, roundIndex) { - const { ieContract, provider } = await createMeridianContract(contractAddress) + const { ieContract, provider } = createContracts(contractAddress) console.log('Fetching MeasurementsAdded events from the ledger') @@ -268,6 +269,6 @@ function isEventLog (logOrEventLog) { * @param {string} contractAddress */ async function fetchLastRoundIndex (contractAddress) { - const { ieContract } = await createMeridianContract(contractAddress) + const { ieContract } = await createContracts(contractAddress) return await ieContract.currentRoundIndex() } diff --git a/bin/fetch-recent-miner-measurements.js b/bin/fetch-recent-miner-measurements.js index f47ad96..46d35ce 100644 --- a/bin/fetch-recent-miner-measurements.js +++ b/bin/fetch-recent-miner-measurements.js @@ -9,7 +9,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import pMap from 'p-map' -import { createMeridianContract } from '../lib/ie-contract.js' +import { createContracts } from '../lib/contracts.js' import { fetchMeasurements, preprocess } from '../lib/preprocess.js' import { RoundData } from '../lib/round.js' import * as SparkImpactEvaluator from '@filecoin-station/spark-impact-evaluator' @@ -140,7 +140,7 @@ console.error('Wrote human-readable summary for %s to %s', minerId, MINER_SUMMAR * @returns */ async function getRecentMeasurementsAddedEvents (contractAddress, blocksToQuery = Number.POSITIVE_INFINITY) { - const { ieContract } = await createMeridianContract(contractAddress) + const { ieContract } = createContracts(contractAddress) // max look-back period allowed by Glif.io is 2000 blocks (approx 16h40m) // in practice, requests for the last 2000 blocks are usually rejected, @@ -214,7 +214,8 @@ async function processRound (roundIndex, measurementCids, resultCounts) { recordTelemetry, logger: { log: debug, error: debug }, ieContract, - setScores: async () => {} + setScores: async () => {}, + prepareProviderRetrievalResultStats: async () => {} }) for (const m of round.measurements) { diff --git a/bin/spark-evaluate.js b/bin/spark-evaluate.js index 9e15238..729c98e 100644 --- a/bin/spark-evaluate.js +++ b/bin/spark-evaluate.js @@ -9,12 +9,17 @@ import { recordTelemetry } from '../lib/telemetry.js' import { fetchMeasurements } from '../lib/preprocess.js' import { migrateWithPgConfig } from '../lib/migrate.js' import pg from 'pg' -import { createMeridianContract } from '../lib/ie-contract.js' +import { createContracts } from '../lib/contracts.js' import { setScores } from '../lib/submit-scores.js' +import * as providerRetrievalResultStats from '../lib/provider-retrieval-result-stats.js' +import { createStorachaClient } from '../lib/storacha.js' const { SENTRY_ENVIRONMENT = 'development', - WALLET_SEED + WALLET_SEED, + STORACHA_SECRET_KEY, + STORACHA_PROOF, + GIT_COMMIT } = process.env Sentry.init({ @@ -25,10 +30,16 @@ Sentry.init({ }) assert(WALLET_SEED, 'WALLET_SEED required') +assert(STORACHA_SECRET_KEY, 'STORACHA_SECRET_KEY required') +assert(STORACHA_PROOF, 'STORACHA_PROOF required') await migrateWithPgConfig({ connectionString: DATABASE_URL }) -const { ieContract, provider } = await createMeridianContract() +const storachaClient = await createStorachaClient({ + secretKey: STORACHA_SECRET_KEY, + proof: STORACHA_PROOF +}) +const { ieContract, ieContractAddress, rsrContract, provider } = createContracts() const signer = ethers.Wallet.fromPhrase(WALLET_SEED, provider) const walletDelegatedAddress = newDelegatedEthAddress(/** @type {any} */(signer.address), CoinType.MAIN).toString() @@ -41,12 +52,27 @@ const createPgClient = async () => { return pgClient } -await startEvaluate({ - ieContract, - fetchMeasurements, - fetchRoundDetails, - recordTelemetry, - createPgClient, - logger: console, - setScores: (participants, values) => setScores(signer, participants, values) -}) +await Promise.all([ + startEvaluate({ + ieContract, + fetchMeasurements, + fetchRoundDetails, + recordTelemetry, + createPgClient, + logger: console, + setScores: (participants, values) => setScores(signer, participants, values), + prepareProviderRetrievalResultStats: (round, committees) => providerRetrievalResultStats.prepare({ + storachaClient, + createPgClient, + round, + committees, + sparkEvaluateVersion: GIT_COMMIT, + ieContractAddress + }) + }), + providerRetrievalResultStats.runPublishLoop({ + createPgClient, + storachaClient, + rsrContract: rsrContract.connect(signer) + }) +]) diff --git a/index.js b/index.js index f9a1582..dc6bc23 100644 --- a/index.js +++ b/index.js @@ -19,7 +19,8 @@ export const startEvaluate = async ({ recordTelemetry, createPgClient, logger, - setScores + setScores, + prepareProviderRetrievalResultStats }) => { assert(typeof createPgClient === 'function', 'createPgClient must be a function') @@ -116,7 +117,8 @@ export const startEvaluate = async ({ recordTelemetry, createPgClient, logger, - setScores + setScores, + prepareProviderRetrievalResultStats }).catch(err => { console.error('CANNOT EVALUATE ROUND %s:', evaluatedRoundIndex, err) Sentry.captureException(err, { diff --git a/lib/car.js b/lib/car.js new file mode 100644 index 0000000..fb0736b --- /dev/null +++ b/lib/car.js @@ -0,0 +1,37 @@ +import { CarWriter } from '@ipld/car' +import * as dagJSON from '@ipld/dag-json' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats' + +/** + * @param {*} json + * @returns {Promise<{ cid: CID, car: Blob }>} + */ +export async function createDagJsonCar (json) { + const bytes = dagJSON.encode(json) + const hash = await sha256.digest(bytes) + const cid = CID.create(1, dagJSON.code, hash) + const car = await createCar({ cid, bytes }, cid) + return { cid, car } +} + +/** + * @param {{ cid: CID, bytes: dagJSON.ByteView }} block + * @param {CID} root + * @returns {Promise} + */ +export async function createCar (block, root) { + const { writer, out } = CarWriter.create(root) + const [chunks] = await Promise.all([ + (async () => { + const chunks = [] + for await (const chunk of out) chunks.push(chunk) + return chunks + })(), + (async () => { + await writer.put(block) + await writer.close() + })() + ]) + return Object.assign(new Blob(chunks), { version: 1, roots: [root] }) +} diff --git a/lib/ie-contract.js b/lib/contracts.js similarity index 58% rename from lib/ie-contract.js rename to lib/contracts.js index 94578e3..ec707f7 100644 --- a/lib/ie-contract.js +++ b/lib/contracts.js @@ -1,8 +1,17 @@ import { ethers } from 'ethers' import { rpcUrls, GLIF_TOKEN } from './config.js' import * as SparkImpactEvaluator from '@filecoin-station/spark-impact-evaluator' +import fs from 'node:fs/promises' +import { fileURLToPath } from 'node:url' -export const createMeridianContract = async (contractAddress = SparkImpactEvaluator.ADDRESS) => { +const rsrContractAbi = JSON.parse( + await fs.readFile( + fileURLToPath(new URL('./rsrContract.json', import.meta.url)), + 'utf8' + ) +).abi + +export const createContracts = (ieContractAddress = SparkImpactEvaluator.ADDRESS) => { const provider = new ethers.FallbackProvider(rpcUrls.map(url => { const fetchRequest = new ethers.FetchRequest(url) fetchRequest.setHeader('Authorization', `Bearer ${GLIF_TOKEN}`) @@ -16,10 +25,16 @@ export const createMeridianContract = async (contractAddress = SparkImpactEvalua // provider.on('debug', d => console.log('[ethers:debug %s] %s %o', new Date().toISOString().split('T')[1], d.action, d.payload ?? d.result)) const ieContract = new ethers.Contract( - contractAddress, + ieContractAddress, SparkImpactEvaluator.ABI, provider ) - return { ieContract, provider } + const rsrContract = new ethers.Contract( + '0x620bfc5AdE7eeEE90034B05DC9Bb5b540336ff90', + rsrContractAbi, + provider + ) + + return { ieContract, ieContractAddress, rsrContract, provider } } diff --git a/lib/evaluate.js b/lib/evaluate.js index f6953bb..0461b9a 100644 --- a/lib/evaluate.js +++ b/lib/evaluate.js @@ -26,6 +26,7 @@ export const REQUIRED_COMMITTEE_SIZE = 30 * @param {import('./typings.js').RecordTelemetryFn} args.recordTelemetry * @param {import('./typings.js').CreatePgClient} [args.createPgClient] * @param {Pick} args.logger + * @param {(round: import('./round.js').RoundData, committees: Iterable) => Promise} args.prepareProviderRetrievalResultStats */ export const evaluate = async ({ round, @@ -36,7 +37,8 @@ export const evaluate = async ({ fetchRoundDetails, recordTelemetry, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats }) => { requiredCommitteeSize ??= REQUIRED_COMMITTEE_SIZE @@ -47,6 +49,7 @@ export const evaluate = async ({ // Detect fraud const sparkRoundDetails = await fetchRoundDetails(await ieContract.getAddress(), roundIndex, recordTelemetry) + round.details = sparkRoundDetails // Omit the roundDetails object from the format string to get nicer formatting debug('ROUND DETAILS for round=%s', roundIndex, sparkRoundDetails) @@ -202,6 +205,14 @@ export const evaluate = async ({ } } + try { + await prepareProviderRetrievalResultStats(round, committees) + } catch (err) { + console.error('Cannot prepare provider retrieval result stats.', err) + ignoredErrors.push(err) + Sentry.captureException(err) + } + return { ignoredErrors } } diff --git a/lib/preprocess.js b/lib/preprocess.js index c1f8cac..9a166ed 100644 --- a/lib/preprocess.js +++ b/lib/preprocess.js @@ -116,6 +116,7 @@ export const preprocess = async ({ logger.log('Retrieval Success Rate: %s%s (%s of %s)', Math.round(100 * okCount / total), '%', okCount, total) round.measurements.push(...validMeasurements) + round.measurementBatches.push(cid) recordTelemetry('preprocess', point => { point.intField('round_index', roundIndex) diff --git a/lib/provider-retrieval-result-stats.js b/lib/provider-retrieval-result-stats.js new file mode 100644 index 0000000..02e0010 --- /dev/null +++ b/lib/provider-retrieval-result-stats.js @@ -0,0 +1,134 @@ +import * as Sentry from '@sentry/node' +import timers from 'node:timers/promises' +import pRetry from 'p-retry' +import { createDagJsonCar } from './car.js' +import { CID } from 'multiformats' + +const ONE_HOUR = 60 * 60 * 1000 + +const withPgClient = fn => async ({ createPgClient, ...args }) => { + const pgClient = await createPgClient() + try { + return await fn({ pgClient, ...args }) + } finally { + await pgClient.end() + } +} + +export const build = committees => { + /** @type {Map} */ + const providerRetrievalResultStats = new Map() + for (const c of committees) { + // IMPORTANT: include minority results in the calculation + for (const m of c.measurements) { + const minerId = m.minerId + const retrievalStats = providerRetrievalResultStats.get(minerId) ?? { total: 0, successful: 0 } + retrievalStats.total++ + if (m.retrievalResult === 'OK') retrievalStats.successful++ + providerRetrievalResultStats.set(minerId, retrievalStats) + } + } + return providerRetrievalResultStats +} + +export const publishRoundDetails = async ({ storachaClient, round }) => { + const { car, cid } = await createDagJsonCar(round.details) + await pRetry(() => storachaClient.uploadCAR(car)) + return cid +} + +export const prepare = withPgClient(async ({ storachaClient, pgClient, round, committees, sparkEvaluateVersion, ieContractAddress }) => { + const roundDetailsCid = await publishRoundDetails({ storachaClient, round }) + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6) + `, [ + round.index, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + roundDetailsCid.toString(), + Object.fromEntries(build(committees).entries()) + ]) +}) + +export const publish = withPgClient(async ({ pgClient, storachaClient, rsrContract }) => { + const { rows } = await pgClient.query(` + WITH oldest_publishable_round AS ( + SELECT * + FROM unpublished_provider_retrieval_result_stats_rounds + WHERE evaluated_at < now()::date + ORDER BY evaluated_at + LIMIT 1 + ) + SELECT *, evaluated_at::DATE::TEXT as date + FROM unpublished_provider_retrieval_result_stats_rounds + WHERE + (SELECT evaluated_at::date FROM oldest_publishable_round) <= evaluated_at + AND evaluated_at < (SELECT evaluated_at::date FROM oldest_publishable_round) + interval '1 day' + AND contract_address = (SELECT contract_address FROM oldest_publishable_round) + ORDER BY round_index + `) + if (rows.length === 0) { + return + } + + const providerRetrievalResultStats = new Map() + for (const row of rows) { + for (const [providerId, { successful, total }] of Object.entries(row.provider_retrieval_result_stats)) { + if (!providerRetrievalResultStats.has(providerId)) { + providerRetrievalResultStats.set(providerId, { successful: 0, total: 0 }) + } + providerRetrievalResultStats.get(providerId).successful += successful + providerRetrievalResultStats.get(providerId).total += total + } + } + + const { cid, car } = await createDagJsonCar({ + date: rows[0].date, + meta: { + rounds: rows.map(row => ({ + index: row.round_index, + contractAddress: row.contract_address, + sparkEvaluateVersion: { + gitCommit: row.spark_evaluate_version + }, + measurementBatches: row.measurement_batches.map(cid => CID.parse(cid)), + details: CID.parse(row.round_details) + })) + }, + providerRetrievalResultStats: [...providerRetrievalResultStats.entries()] + .map(([providerId, { successful, total }]) => ({ providerId, successful, total })) + }) + await pRetry(() => storachaClient.uploadCAR(car)) + console.log(`https://${cid}.ipfs.w3s.link`) + + const tx = await pRetry(() => rsrContract.addProviderRetrievalResultStats(cid.toString())) + console.log(tx.hash) + await tx.wait() + + await pgClient.query(` + DELETE FROM unpublished_provider_retrieval_result_stats_rounds + WHERE + round_index <= $1 + AND contract_address = $2 + `, [ + rows.at(-1).round_index, + rows[0].contract_address + ]) +}) + +export const runPublishLoop = async ({ createPgClient, storachaClient, rsrContract }) => { + while (true) { + try { + await publish({ createPgClient, storachaClient, rsrContract }) + } catch (err) { + console.error(err) + Sentry.captureException(err) + } finally { + await timers.setTimeout(ONE_HOUR) + } + } +} diff --git a/lib/public-stats.js b/lib/public-stats.js index 4e6a476..40ffdf2 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -1,6 +1,6 @@ import * as Sentry from '@sentry/node' import createDebug from 'debug' - +import * as providerRetrievalResultStats from './provider-retrieval-result-stats.js' import { updatePlatformStats } from './platform-stats.js' import { getTaskId } from './retrieval-stats.js' @@ -18,23 +18,11 @@ const debug = createDebug('spark:public-stats') * @param {(minerId: string, cid: string) => (string[] | undefined)} args.findDealClients */ export const updatePublicStats = async ({ createPgClient, committees, honestMeasurements, allMeasurements, findDealClients }) => { - /** @type {Map} */ - const minerRetrievalStats = new Map() - for (const c of committees) { - // IMPORTANT: include minority results in the calculation - for (const m of c.measurements) { - const minerId = m.minerId - const retrievalStats = minerRetrievalStats.get(minerId) ?? { total: 0, successful: 0 } - retrievalStats.total++ - if (m.retrievalResult === 'OK') retrievalStats.successful++ - minerRetrievalStats.set(minerId, retrievalStats) - } - } - + const stats = providerRetrievalResultStats.build(committees) const pgClient = await createPgClient() try { - for (const [minerId, retrievalStats] of minerRetrievalStats.entries()) { - await updateRetrievalStats(pgClient, minerId, retrievalStats) + for (const [minerId, retrievalResultStats] of stats.entries()) { + await updateRetrievalStats(pgClient, minerId, retrievalResultStats) } await updateIndexerQueryStats(pgClient, committees) await updateDailyDealsStats(pgClient, committees, findDealClients) diff --git a/lib/round.js b/lib/round.js index 6791415..da24f07 100644 --- a/lib/round.js +++ b/lib/round.js @@ -8,7 +8,9 @@ export class RoundData { constructor (index) { this.index = index /** @type {import('./preprocess.js').Measurement[]} */ + this.measurementBatches = [] this.measurements = [] + this.details = null this.#knownStrings = new Map() // defined as a rocket function that can be detached diff --git a/lib/rsrContract.json b/lib/rsrContract.json new file mode 100644 index 0000000..7f6e9af --- /dev/null +++ b/lib/rsrContract.json @@ -0,0 +1 @@ +{"abi":[{"type":"constructor","inputs":[{"name":"_writer","type":"address","internalType":"address"}],"stateMutability":"nonpayable"},{"type":"function","name":"addProviderRetrievalResultStats","inputs":[{"name":"cid","type":"string","internalType":"string"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"providerRetrievalResultStats","inputs":[{"name":"","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"string","internalType":"string"}],"stateMutability":"view"}],"bytecode":{"object":"0x608060405234801561001057600080fd5b506040516105ef3803806105ef83398101604081905261002f91610054565b600180546001600160a01b0319166001600160a01b0392909216919091179055610084565b60006020828403121561006657600080fd5b81516001600160a01b038116811461007d57600080fd5b9392505050565b61055c806100936000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063bec0bf8e1461003b578063c295be7114610050575b600080fd5b61004e610049366004610216565b610079565b005b61006361005e3660046102e5565b61013b565b60405161007091906102fe565b60405180910390f35b60015473ffffffffffffffffffffffffffffffffffffffff1633146100fe576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601d60248201527f6f6e6c7920746865207772697465722063616e20616464207374617473000000604482015260640160405180910390fd5b600080546001810182559080527f290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e56301610137828261040c565b5050565b6000818154811061014b57600080fd5b9060005260206000200160009150905080546101669061036a565b80601f01602080910402602001604051908101604052809291908181526020018280546101929061036a565b80156101df5780601f106101b4576101008083540402835291602001916101df565b820191906000526020600020905b8154815290600101906020018083116101c257829003601f168201915b505050505081565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b60006020828403121561022857600080fd5b813567ffffffffffffffff8082111561024057600080fd5b818401915084601f83011261025457600080fd5b813581811115610266576102666101e7565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f011681019083821181831017156102ac576102ac6101e7565b816040528281528760208487010111156102c557600080fd5b826020860160208301376000928101602001929092525095945050505050565b6000602082840312156102f757600080fd5b5035919050565b600060208083528351808285015260005b8181101561032b5785810183015185820160400152820161030f565b5060006040828601015260407fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f8301168501019250505092915050565b600181811c9082168061037e57607f821691505b6020821081036103b7577f4e487b7100000000000000000000000000000000000000000000000000000000600052602260045260246000fd5b50919050565b601f82111561040757600081815260208120601f850160051c810160208610156103e45750805b601f850160051c820191505b81811015610403578281556001016103f0565b5050505b505050565b815167ffffffffffffffff811115610426576104266101e7565b61043a81610434845461036a565b846103bd565b602080601f83116001811461048d57600084156104575750858301515b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff600386901b1c1916600185901b178555610403565b6000858152602081207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08616915b828110156104da578886015182559484019460019091019084016104bb565b508582101561051657878501517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff600388901b60f8161c191681555b5050505050600190811b0190555056fea26469706673582212201771f7d797ea5e4af4515439bf454771e225dffbf33b92c2f72a35fcef48a95564736f6c63430008150033","sourceMap":"75:359:23:-:0;;;170:62;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;209:6;:16;;-1:-1:-1;;;;;;209:16:23;-1:-1:-1;;;;;209:16:23;;;;;;;;;;75:359;;14:290:25;84:6;137:2;125:9;116:7;112:23;108:32;105:52;;;153:1;150;143:12;105:52;179:16;;-1:-1:-1;;;;;224:31:25;;214:42;;204:70;;270:1;267;260:12;204:70;293:5;14:290;-1:-1:-1;;;14:290:25:o;:::-;75:359:23;;;;;;","linkReferences":{}},"deployedBytecode":{"object":"0x608060405234801561001057600080fd5b50600436106100365760003560e01c8063bec0bf8e1461003b578063c295be7114610050575b600080fd5b61004e610049366004610216565b610079565b005b61006361005e3660046102e5565b61013b565b60405161007091906102fe565b60405180910390f35b60015473ffffffffffffffffffffffffffffffffffffffff1633146100fe576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601d60248201527f6f6e6c7920746865207772697465722063616e20616464207374617473000000604482015260640160405180910390fd5b600080546001810182559080527f290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e56301610137828261040c565b5050565b6000818154811061014b57600080fd5b9060005260206000200160009150905080546101669061036a565b80601f01602080910402602001604051908101604052809291908181526020018280546101929061036a565b80156101df5780601f106101b4576101008083540402835291602001916101df565b820191906000526020600020905b8154815290600101906020018083116101c257829003601f168201915b505050505081565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b60006020828403121561022857600080fd5b813567ffffffffffffffff8082111561024057600080fd5b818401915084601f83011261025457600080fd5b813581811115610266576102666101e7565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f011681019083821181831017156102ac576102ac6101e7565b816040528281528760208487010111156102c557600080fd5b826020860160208301376000928101602001929092525095945050505050565b6000602082840312156102f757600080fd5b5035919050565b600060208083528351808285015260005b8181101561032b5785810183015185820160400152820161030f565b5060006040828601015260407fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f8301168501019250505092915050565b600181811c9082168061037e57607f821691505b6020821081036103b7577f4e487b7100000000000000000000000000000000000000000000000000000000600052602260045260246000fd5b50919050565b601f82111561040757600081815260208120601f850160051c810160208610156103e45750805b601f850160051c820191505b81811015610403578281556001016103f0565b5050505b505050565b815167ffffffffffffffff811115610426576104266101e7565b61043a81610434845461036a565b846103bd565b602080601f83116001811461048d57600084156104575750858301515b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff600386901b1c1916600185901b178555610403565b6000858152602081207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08616915b828110156104da578886015182559484019460019091019084016104bb565b508582101561051657878501517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff600388901b60f8161c191681555b5050505050600190811b0190555056fea26469706673582212201771f7d797ea5e4af4515439bf454771e225dffbf33b92c2f72a35fcef48a95564736f6c63430008150033","sourceMap":"75:359:23:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;238:194;;;;;;:::i;:::-;;:::i;:::-;;99:44;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;238:194;337:6;;;;323:10;:20;315:62;;;;;;;2188:2:25;315:62:23;;;2170:21:25;2227:2;2207:18;;;2200:30;2266:31;2246:18;;;2239:59;2315:18;;315:62:23;;;;;;;;387:28;:38;;;;;;;;;;;;;421:3;387:38;;:::i;:::-;;238:194;:::o;99:44::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;14:184:25:-;66:77;63:1;56:88;163:4;160:1;153:15;187:4;184:1;177:15;203:981;272:6;325:2;313:9;304:7;300:23;296:32;293:52;;;341:1;338;331:12;293:52;381:9;368:23;410:18;451:2;443:6;440:14;437:34;;;467:1;464;457:12;437:34;505:6;494:9;490:22;480:32;;550:7;543:4;539:2;535:13;531:27;521:55;;572:1;569;562:12;521:55;608:2;595:16;630:2;626;623:10;620:36;;;636:18;;:::i;:::-;770:2;764:9;832:4;824:13;;675:66;820:22;;;844:2;816:31;812:40;800:53;;;868:18;;;888:22;;;865:46;862:72;;;914:18;;:::i;:::-;954:10;950:2;943:22;989:2;981:6;974:18;1029:7;1024:2;1019;1015;1011:11;1007:20;1004:33;1001:53;;;1050:1;1047;1040:12;1001:53;1106:2;1101;1097;1093:11;1088:2;1080:6;1076:15;1063:46;1151:1;1129:15;;;1146:2;1125:24;1118:35;;;;-1:-1:-1;1133:6:25;203:981;-1:-1:-1;;;;;203:981:25:o;1189:180::-;1248:6;1301:2;1289:9;1280:7;1276:23;1272:32;1269:52;;;1317:1;1314;1307:12;1269:52;-1:-1:-1;1340:23:25;;1189:180;-1:-1:-1;1189:180:25:o;1374:607::-;1486:4;1515:2;1544;1533:9;1526:21;1576:6;1570:13;1619:6;1614:2;1603:9;1599:18;1592:34;1644:1;1654:140;1668:6;1665:1;1662:13;1654:140;;;1763:14;;;1759:23;;1753:30;1729:17;;;1748:2;1725:26;1718:66;1683:10;;1654:140;;;1658:3;1843:1;1838:2;1829:6;1818:9;1814:22;1810:31;1803:42;1972:2;1902:66;1897:2;1889:6;1885:15;1881:88;1870:9;1866:104;1862:113;1854:121;;;;1374:607;;;;:::o;2344:437::-;2423:1;2419:12;;;;2466;;;2487:61;;2541:4;2533:6;2529:17;2519:27;;2487:61;2594:2;2586:6;2583:14;2563:18;2560:38;2557:218;;2631:77;2628:1;2621:88;2732:4;2729:1;2722:15;2760:4;2757:1;2750:15;2557:218;;2344:437;;;:::o;2912:545::-;3014:2;3009:3;3006:11;3003:448;;;3050:1;3075:5;3071:2;3064:17;3120:4;3116:2;3106:19;3190:2;3178:10;3174:19;3171:1;3167:27;3161:4;3157:38;3226:4;3214:10;3211:20;3208:47;;;-1:-1:-1;3249:4:25;3208:47;3304:2;3299:3;3295:12;3292:1;3288:20;3282:4;3278:31;3268:41;;3359:82;3377:2;3370:5;3367:13;3359:82;;;3422:17;;;3403:1;3392:13;3359:82;;;3363:3;;;3003:448;2912:545;;;:::o;3693:1471::-;3819:3;3813:10;3846:18;3838:6;3835:30;3832:56;;;3868:18;;:::i;:::-;3897:97;3987:6;3947:38;3979:4;3973:11;3947:38;:::i;:::-;3941:4;3897:97;:::i;:::-;4049:4;;4113:2;4102:14;;4130:1;4125:782;;;;4951:1;4968:6;4965:89;;;-1:-1:-1;5020:19:25;;;5014:26;4965:89;3599:66;3590:1;3586:11;;;3582:84;3578:89;3568:100;3674:1;3670:11;;;3565:117;5067:81;;4095:1063;;4125:782;2859:1;2852:14;;;2896:4;2883:18;;4173:66;4161:79;;;4338:236;4352:7;4349:1;4346:14;4338:236;;;4441:19;;;4435:26;4420:42;;4533:27;;;;4501:1;4489:14;;;;4368:19;;4338:236;;;4342:3;4602:6;4593:7;4590:19;4587:261;;;4663:19;;;4657:26;4764:66;4746:1;4742:14;;;4758:3;4738:24;4734:97;4730:102;4715:118;4700:134;;4587:261;-1:-1:-1;;;;;4894:1:25;4878:14;;;4874:22;4861:36;;-1:-1:-1;3693:1471:25:o","linkReferences":{}},"methodIdentifiers":{"addProviderRetrievalResultStats(string)":"bec0bf8e","providerRetrievalResultStats(uint256)":"c295be71"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.21+commit.d9974bed\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[{\"internalType\":\"address\",\"name\":\"_writer\",\"type\":\"address\"}],\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"cid\",\"type\":\"string\"}],\"name\":\"addProviderRetrievalResultStats\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"name\":\"providerRetrievalResultStats\",\"outputs\":[{\"internalType\":\"string\",\"name\":\"\",\"type\":\"string\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/SparkRsr.sol\":\"SparkRsr\"},\"evmVersion\":\"paris\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":10000000},\"remappings\":[\":forge-std/=lib/forge-std/src/\"]},\"sources\":{\"src/SparkRsr.sol\":{\"keccak256\":\"0xe362980f20b5c44c07077ff0a81055d8407c6a836ff285cfa63099fbf505523e\",\"license\":\"(MIT or Apache-2.0)\",\"urls\":[\"bzz-raw://4e8d19b944e506186fbe817328810730c7c379e623f2a7057597951bb8da8173\",\"dweb:/ipfs/QmdnLukqycacLGCnUdbUSPkAyGrPujcP8MFD1pM5mt1iSt\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.21+commit.d9974bed"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"address","name":"_writer","type":"address"}],"stateMutability":"nonpayable","type":"constructor"},{"inputs":[{"internalType":"string","name":"cid","type":"string"}],"stateMutability":"nonpayable","type":"function","name":"addProviderRetrievalResultStats"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function","name":"providerRetrievalResultStats","outputs":[{"internalType":"string","name":"","type":"string"}]}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["forge-std/=lib/forge-std/src/"],"optimizer":{"enabled":true,"runs":10000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/SparkRsr.sol":"SparkRsr"},"evmVersion":"paris","libraries":{}},"sources":{"src/SparkRsr.sol":{"keccak256":"0xe362980f20b5c44c07077ff0a81055d8407c6a836ff285cfa63099fbf505523e","urls":["bzz-raw://4e8d19b944e506186fbe817328810730c7c379e623f2a7057597951bb8da8173","dweb:/ipfs/QmdnLukqycacLGCnUdbUSPkAyGrPujcP8MFD1pM5mt1iSt"],"license":"(MIT or Apache-2.0)"}},"version":1},"id":23} \ No newline at end of file diff --git a/lib/storacha.js b/lib/storacha.js new file mode 100644 index 0000000..e127893 --- /dev/null +++ b/lib/storacha.js @@ -0,0 +1,21 @@ +import * as Client from '@web3-storage/w3up-client' +import { ed25519 } from '@ucanto/principal' +import { CarReader } from '@ipld/car' +import { importDAG } from '@ucanto/core/delegation' + +async function parseProof (data) { + const blocks = [] + const reader = await CarReader.fromBytes(Buffer.from(data, 'base64')) + for await (const block of reader.blocks()) { + blocks.push(block) + } + return importDAG(blocks) +} + +export async function createStorachaClient ({ secretKey, proof }) { + const principal = ed25519.Signer.parse(secretKey) + const client = await Client.create({ principal }) + const space = await client.addSpace(await parseProof(proof)) + await client.setCurrentSpace(space.did()) + return client +} diff --git a/migrations/020.do.publish-provider-retrieval-result-stats.sql b/migrations/020.do.publish-provider-retrieval-result-stats.sql new file mode 100644 index 0000000..215b4ef --- /dev/null +++ b/migrations/020.do.publish-provider-retrieval-result-stats.sql @@ -0,0 +1,10 @@ +CREATE TABLE unpublished_provider_retrieval_result_stats_rounds ( + round_index NUMERIC NOT NULL, + contract_address TEXT NOT NULL, + evaluated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + spark_evaluate_version TEXT NOT NULL, + measurement_batches TEXT[] NOT NULL, + round_details TEXT NOT NULL, + provider_retrieval_result_stats JSONB NOT NULL, + PRIMARY KEY (round_index, contract_address) +); diff --git a/package-lock.json b/package-lock.json index baaa4f3..798904f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,12 @@ "@glif/filecoin-address": "^3.0.12", "@influxdata/influxdb-client": "^1.35.0", "@ipld/car": "^5.3.2", + "@ipld/dag-json": "^10.2.2", "@sentry/node": "^8.35.0", + "@ucanto/core": "^10.0.1", + "@ucanto/principal": "^9.0.1", "@web3-storage/car-block-validator": "^1.2.0", + "@web3-storage/w3up-client": "^16.2.0", "debug": "^4.3.7", "drand-client": "^1.2.6", "ethers": "^6.13.4", @@ -20,6 +24,7 @@ "just-percentile": "^4.2.0", "k-closest": "^1.3.0", "ms": "^2.1.3", + "multiformats": "^13.3.0", "p-map": "^7.0.2", "p-retry": "^6.2.0", "pg": "^8.13.1", @@ -335,6 +340,12 @@ "node": ">=14.0.0" } }, + "node_modules/@glif/filecoin-address/node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, "node_modules/@glif/filecoin-address/node_modules/uint8arrays": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-3.1.0.tgz", @@ -385,6 +396,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/@ipld/car/-/car-5.3.2.tgz", "integrity": "sha512-Bb4XrCFlnsCb9tTzZ1I8zo9O61D9qm7HfvuYrQ9gzdE8YhjyVIjrjmHmnoSWV/uCmyc2/bcqiDPIg+9WljXNzg==", + "license": "Apache-2.0 OR MIT", "dependencies": { "@ipld/dag-cbor": "^9.0.7", "cborg": "^4.0.5", @@ -396,11 +408,6 @@ "npm": ">=7.0.0" } }, - "node_modules/@ipld/car/node_modules/multiformats": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.0.0.tgz", - "integrity": "sha512-xiIB0p7EKmETm3wyKedOg/xuyQ18PoWwXCzzgpZAiDxL9ktl3XTh8AqoDT5kAqRg+DU48XAGPsUJL2Rn6Bx3Lw==" - }, "node_modules/@ipld/dag-cbor": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.2.1.tgz", @@ -414,15 +421,11 @@ "npm": ">=7.0.0" } }, - "node_modules/@ipld/dag-cbor/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" - }, "node_modules/@ipld/dag-json": { "version": "10.2.2", "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-10.2.2.tgz", "integrity": "sha512-NnU8HdHKwAoGyrW3S09NMa8aZw0tImLRyR64hoafpLpDpAbA9g1+fb24JsdlugbL4sXUQVwDVA+qK4Ud8V83lA==", + "license": "Apache-2.0 OR MIT", "dependencies": { "cborg": "^4.0.0", "multiformats": "^13.1.0" @@ -432,11 +435,6 @@ "npm": ">=7.0.0" } }, - "node_modules/@ipld/dag-json/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" - }, "node_modules/@ipld/dag-pb": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-4.1.2.tgz", @@ -449,10 +447,26 @@ "npm": ">=7.0.0" } }, - "node_modules/@ipld/dag-pb/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" + "node_modules/@ipld/dag-ucan": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@ipld/dag-ucan/-/dag-ucan-3.4.0.tgz", + "integrity": "sha512-sW4R43w3DbEdoGWWJZCwsblwXa600HCanG9p2w1MJPVBNTNjhvqc3XI0uEqKhT2oqKWrND7uInVtcPmZme7hhA==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/dag-cbor": "^9.0.0", + "@ipld/dag-json": "^10.0.0", + "multiformats": "^11.0.0" + } + }, + "node_modules/@ipld/dag-ucan/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } }, "node_modules/@ipld/unixfs": { "version": "3.0.0", @@ -468,11 +482,6 @@ "rabin-rs": "^2.1.0" } }, - "node_modules/@ipld/unixfs/node_modules/multiformats": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.0.1.tgz", - "integrity": "sha512-bt3R5iXe2O8xpp3wkmQhC73b/lC4S2ihU8Dndwcsysqbydqb8N+bpP116qMcClZ17g58iSIwtXUTcg2zT4sniA==" - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -525,6 +534,12 @@ "multiformats": "^9.5.4" } }, + "node_modules/@multiformats/blake2/node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, "node_modules/@multiformats/murmur3": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-2.1.8.tgz", @@ -538,11 +553,6 @@ "npm": ">=7.0.0" } }, - "node_modules/@multiformats/murmur3/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" - }, "node_modules/@multiformats/sha3": { "version": "2.0.17", "resolved": "https://registry.npmjs.org/@multiformats/sha3/-/sha3-2.0.17.tgz", @@ -552,6 +562,12 @@ "multiformats": "^9.5.4" } }, + "node_modules/@multiformats/sha3/node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, "node_modules/@noble/curves": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.2.0.tgz", @@ -563,6 +579,18 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@noble/ed25519": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/@noble/ed25519/-/ed25519-1.7.3.tgz", + "integrity": "sha512-iR8GBkDt0Q3GyaVcIu7mSsVIqnFbkbRzGLWlvhwunacoLwt4J3swfKhfaM6rN6WY+TBGoYT1GtT1mIh2/jGbRQ==", + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "license": "MIT" + }, "node_modules/@noble/hashes": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.3.2.tgz", @@ -1238,6 +1266,40 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@scure/base": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.9.tgz", + "integrity": "sha512-8YKhl8GHiNI/pU2VMaofa2Tor7PJRAjwQLBBuilkJ9L5+13yVbC7JO/wS7piioAvPSwR3JKM1IJ/u4xQzbcXKg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.4.0.tgz", + "integrity": "sha512-BEEm6p8IueV/ZTfQLp/0vhw4NPnT9oWf5+28nvmeUICjP99f4vr2d+qc7AVGDDtwRep6ifR43Yed9ERVmiITzw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.5.0", + "@scure/base": "~1.1.8" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39/node_modules/@noble/hashes": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.5.0.tgz", + "integrity": "sha512-1j6kQFb7QRru7eKN3ZDvRcP13rugwdxZqCjbiAVZfIJwgj2A65UmT4TgARXGlXgnRkORLTDTrO19ZErt7+QXgA==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@sentry/core": { "version": "8.35.0", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.35.0.tgz", @@ -1342,6 +1404,12 @@ "node": ">= 8" } }, + "node_modules/@storacha/one-webcrypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@storacha/one-webcrypto/-/one-webcrypto-1.0.1.tgz", + "integrity": "sha512-bD+vWmcgsEBqU0Dz04BR43SA03bBoLTAY29vaKasY9Oe8cb6XIP0/vkm0OS2UwKC13c8uRgFW4rjJUgDCNLejQ==", + "license": "MIT" + }, "node_modules/@types/connect": { "version": "3.4.36", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", @@ -1356,6 +1424,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/minimatch": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", + "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", + "license": "MIT" + }, "node_modules/@types/mocha": { "version": "10.0.9", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.9.tgz", @@ -1458,6 +1532,226 @@ "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.2.0.tgz", "integrity": "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg==" }, + "node_modules/@ucanto/client": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/client/-/client-9.0.1.tgz", + "integrity": "sha512-cV8w3AnaZaYCdUmyFFICj8YhFckDoy2DvWgAzGDMkPz0WbUW4lw9Tjm4hEE8x5kiP47wYej/pHKWCcoELiU0qw==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ucanto/core": "^10.0.0", + "@ucanto/interface": "^10.0.0" + } + }, + "node_modules/@ucanto/core": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-10.0.1.tgz", + "integrity": "sha512-1BfUaJu0/c9Rl/WdZSDbScJJLsPsPe1g4ynl5kubUj3xDD/lyp/Q12PQVQ2X7hDiWwkpwmxCkRMkOxwc70iNKQ==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/car": "^5.1.0", + "@ipld/dag-cbor": "^9.0.0", + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/interface": "^10.0.1", + "multiformats": "^11.0.2" + } + }, + "node_modules/@ucanto/core/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ucanto/interface": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-10.0.1.tgz", + "integrity": "sha512-+Vr/N4mLsdynV9/bqtdFiq7WsUf3265/Qx2aHJmPtXo9/QvWKthJtpe0g8U4NWkWpVfqIFvyAO2db6D9zWQfQw==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "multiformats": "^11.0.2" + } + }, + "node_modules/@ucanto/interface/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ucanto/principal": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/principal/-/principal-9.0.1.tgz", + "integrity": "sha512-8eAvaZHW1vyET4X90rkJv6pmW1IOdEYlZYwO3wDgTkC5m9VytBEywCvpzP57cavdYIbbPse5QS9nMEGvk87zhw==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "@noble/curves": "^1.2.0", + "@noble/ed25519": "^1.7.3", + "@noble/hashes": "^1.3.2", + "@ucanto/interface": "^10.0.0", + "multiformats": "^11.0.2", + "one-webcrypto": "^1.0.3" + } + }, + "node_modules/@ucanto/principal/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ucanto/transport": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/@ucanto/transport/-/transport-9.1.1.tgz", + "integrity": "sha512-3CR17nEemOVaTuMZa6waWgVL4sLxSPcxYvpaNeJ6NZo1rfsqdyRXOtbVV/RcI2BtUL0Cao6JM6P9+gdghfc5ng==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ucanto/core": "^10.0.0", + "@ucanto/interface": "^10.0.0" + } + }, + "node_modules/@ucanto/validator": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/@ucanto/validator/-/validator-9.0.2.tgz", + "integrity": "sha512-LxhRbDMIoLt9LYHq/Rz1WCEH8AtmdsBTS/it28Ij/A3W0zyoSwUpAUxBtXaKRh/gpbxdWmjxX+nVfFJYL//b4g==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/car": "^5.1.0", + "@ipld/dag-cbor": "^9.0.0", + "@ucanto/core": "^10.0.0", + "@ucanto/interface": "^10.0.0", + "multiformats": "^11.0.2" + } + }, + "node_modules/@ucanto/validator/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/access": { + "version": "20.0.1", + "resolved": "https://registry.npmjs.org/@web3-storage/access/-/access-20.0.1.tgz", + "integrity": "sha512-JlCTp1BlFmxrxpkkLo73tytHIv7J+l8++dP5ghYk5oEo2Om3mUq+T3KkkkjZ8d3omoWam6tGiTQXbc/awHJjDw==", + "license": "(Apache-2.0 OR MIT)", + "dependencies": { + "@ipld/car": "^5.1.1", + "@ipld/dag-ucan": "^3.4.0", + "@scure/bip39": "^1.2.1", + "@storacha/one-webcrypto": "^1.0.1", + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/capabilities": "^17.2.0", + "@web3-storage/did-mailto": "^2.1.0", + "bigint-mod-arith": "^3.1.2", + "conf": "11.0.2", + "multiformats": "^12.1.2", + "p-defer": "^4.0.0", + "type-fest": "^4.9.0", + "uint8arrays": "^4.0.6" + } + }, + "node_modules/@web3-storage/access/node_modules/multiformats": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", + "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/access/node_modules/type-fest": { + "version": "4.26.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz", + "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/uint8arrays": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-4.0.10.tgz", + "integrity": "sha512-AnJNUGGDJAgFw/eWu/Xb9zrVKEGlwJJCaeInlf3BkecE/zcTobk5YXYIPNQJO1q5Hh1QZrQQHf0JvcHqz2hqoA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "multiformats": "^12.0.1" + } + }, + "node_modules/@web3-storage/blob-index": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@web3-storage/blob-index/-/blob-index-1.0.4.tgz", + "integrity": "sha512-04+PrmVHFT+xzRhyIPdcvGc8Y2NDffUe8R1gJOyErVzEVz5N1I9Q/BrlFHYt/A4HrjM5JBsxqSrZgTIkjfPmLA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/dag-cbor": "^9.0.6", + "@storacha/one-webcrypto": "^1.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@web3-storage/capabilities": "^17.2.0", + "carstream": "^2.1.0", + "multiformats": "^13.0.1", + "uint8arrays": "^5.0.3" + }, + "engines": { + "node": ">=16.15" + } + }, + "node_modules/@web3-storage/blob-index/node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/@web3-storage/capabilities": { + "version": "17.3.0", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-17.3.0.tgz", + "integrity": "sha512-9415OPNVYO5gXDVf1vzZywkjndKTVA9IPnU04lQXxUaYfYZ5S5kzV2PI1SvySMOsCNE7u7uSCTiclblx5gPYAg==", + "license": "(Apache-2.0 OR MIT)", + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^5.2.0", + "uint8arrays": "^5.0.3" + } + }, + "node_modules/@web3-storage/capabilities/node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "multiformats": "^13.0.0" + } + }, "node_modules/@web3-storage/car-block-validator": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@web3-storage/car-block-validator/-/car-block-validator-1.2.0.tgz", @@ -1479,6 +1773,150 @@ "murmurhash3js-revisited": "^3.0.0" } }, + "node_modules/@web3-storage/car-block-validator/node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, + "node_modules/@web3-storage/data-segment": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.3.0.tgz", + "integrity": "sha512-zFJ4m+pEKqtKatJNsFrk/2lHeFSbkXZ6KKXjBe7/2ayA9wAar7T/unewnOcZrrZTnCWmaxKsXWqdMFy9bXK9dw==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/dag-cbor": "^9.2.1", + "multiformats": "^13.3.0", + "sync-multihash-sha2": "^1.0.0" + } + }, + "node_modules/@web3-storage/did-mailto": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/did-mailto/-/did-mailto-2.1.0.tgz", + "integrity": "sha512-TRmfSXj1IhtX3ESurSNOylZSBKi0z/VJNoMLpof+AVRdovgZjjocpiePQTs2pfHKqHTHfJXc9AboWyK4IKTWMw==", + "license": "(Apache-2.0 OR MIT)", + "engines": { + "node": ">=16.15" + } + }, + "node_modules/@web3-storage/filecoin-client": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-client/-/filecoin-client-3.3.4.tgz", + "integrity": "sha512-T2xur1NPvuH09yajyjCWEl7MBH712nqHERj51w4nDp6f8libMCKY6lca0frCrm4OC5s8oy0ZtoRFhsRYxgTzSg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/transport": "^9.1.1", + "@web3-storage/capabilities": "^17.3.0" + } + }, + "node_modules/@web3-storage/upload-client": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-17.0.1.tgz", + "integrity": "sha512-+ELz3y32YmiMvuPD/fZgCEqn/KvvoUKcZ2ao+9wosfs6GJ8/j6lhxkEkwnICiwU2tLEB+FUsOuCDFquOSW1rKg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/car": "^5.2.2", + "@ipld/dag-cbor": "^9.0.6", + "@ipld/dag-ucan": "^3.4.0", + "@ipld/unixfs": "^2.1.1", + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/transport": "^9.1.1", + "@web3-storage/blob-index": "^1.0.3", + "@web3-storage/capabilities": "^17.2.0", + "@web3-storage/data-segment": "^5.1.0", + "@web3-storage/filecoin-client": "^3.3.3", + "ipfs-utils": "^9.0.14", + "multiformats": "^12.1.2", + "p-retry": "^5.1.2", + "varint": "^6.0.0" + } + }, + "node_modules/@web3-storage/upload-client/node_modules/@ipld/unixfs": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ipld/unixfs/-/unixfs-2.2.0.tgz", + "integrity": "sha512-lDQ2eRhJlbFaBoO3bhOmDVCLmpOnhwtwbilqUgAAhbhoPSmLrnv7gsBuToZjXOdPaEGSL7apkmm6nFrcU6zh4Q==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/dag-pb": "^4.0.0", + "@multiformats/murmur3": "^2.1.3", + "@perma/map": "^1.0.2", + "actor": "^2.3.1", + "multiformats": "^11.0.1", + "protobufjs": "^7.1.2", + "rabin-rs": "^2.1.0" + } + }, + "node_modules/@web3-storage/upload-client/node_modules/@ipld/unixfs/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/upload-client/node_modules/@types/retry": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.1.tgz", + "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==", + "license": "MIT" + }, + "node_modules/@web3-storage/upload-client/node_modules/multiformats": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", + "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/upload-client/node_modules/p-retry": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-5.1.2.tgz", + "integrity": "sha512-couX95waDu98NfNZV+i/iLt+fdVxmI7CbrrdC2uDWfPdUAApyxT4wmDlyOtR5KtTDmkDO0zDScDjDou9YHhd9g==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.1", + "retry": "^0.13.1" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/w3up-client": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/@web3-storage/w3up-client/-/w3up-client-16.2.0.tgz", + "integrity": "sha512-gqzq03gcu14UrNw5Nwi7j6bqA3HLjnPDITo/qJBzaeRRfoBgnB6nrakIkR/SVhKqZ43VRy0MpxUhUmTLxW3kFQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@web3-storage/access": "^20.0.1", + "@web3-storage/blob-index": "^1.0.4", + "@web3-storage/capabilities": "^17.2.0", + "@web3-storage/did-mailto": "^2.1.0", + "@web3-storage/filecoin-client": "^3.3.3", + "@web3-storage/upload-client": "^17.0.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/acorn": { "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", @@ -1533,6 +1971,45 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/ansi-colors": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", @@ -1566,6 +2043,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/any-signal": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/any-signal/-/any-signal-3.0.1.tgz", + "integrity": "sha512-xgZgJtKEa9YmDqXodIgl7Fl1C8yNXr8w6gXjqK3LW4GcEiYT+6AQfJSE/8SPsEpLLmcvbv8YU+qet94UewHxqg==", + "license": "MIT" + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -1734,6 +2217,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/atomically": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-2.0.3.tgz", + "integrity": "sha512-kU6FmrwZ3Lx7/7y3hPS5QnbJfaohcIul5fGqf7ok+4KklIEk9tJ0C2IQPdacSbVUWv6zVHXEBWoWd6NrVMT7Cw==", + "dependencies": { + "stubborn-fs": "^1.2.5", + "when-exit": "^2.1.1" + } + }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -1773,6 +2265,15 @@ } ] }, + "node_modules/bigint-mod-arith": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/bigint-mod-arith/-/bigint-mod-arith-3.3.1.tgz", + "integrity": "sha512-pX/cYW3dCa87Jrzv6DAr8ivbbJRzEX5yGhdt8IutnX/PCIXfpx+mabWNK/M8qqh+zQ0J3thftUBHW0ByuUlG0w==", + "license": "MIT", + "engines": { + "node": ">=10.4.0" + } + }, "node_modules/bignumber.js": { "version": "9.1.2", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", @@ -1844,6 +2345,12 @@ "node": ">=8" } }, + "node_modules/browser-readablestream-to-it": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/browser-readablestream-to-it/-/browser-readablestream-to-it-1.0.3.tgz", + "integrity": "sha512-+12sHB+Br8HIh6VAMVEG5r3UXCyESIgDW7kzk3BjIXa43DVqVwL7GC5TW3jeh+72dtcH99pPVpw0X8i0jt+/kw==", + "license": "ISC" + }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", @@ -1933,6 +2440,17 @@ "node": ">=6" } }, + "node_modules/carstream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-2.2.0.tgz", + "integrity": "sha512-/gHkK0lQjmGM45fhdx8JD+x7a1XS1qUk3T9xWWSt3oZiWPLq4u/lnDstp+N55K7hqTKKlb0CCr43EHTrlbmJSQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^13.0.1", + "uint8arraylist": "^2.4.3" + } + }, "node_modules/cborg": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.0.5.tgz", @@ -2040,6 +2558,62 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, + "node_modules/conf": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/conf/-/conf-11.0.2.tgz", + "integrity": "sha512-jjyhlQ0ew/iwmtwsS2RaB6s8DBifcE2GYBEaw2SJDUY/slJJbNfY4GlDVzOs/ff8cM/Wua5CikqXgbFl5eu85A==", + "license": "MIT", + "dependencies": { + "ajv": "^8.12.0", + "ajv-formats": "^2.1.1", + "atomically": "^2.0.0", + "debounce-fn": "^5.1.2", + "dot-prop": "^7.2.0", + "env-paths": "^3.0.0", + "json-schema-typed": "^8.0.1", + "semver": "^7.3.8" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/conf/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/conf/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/conf/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2105,6 +2679,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/debounce-fn": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-5.1.2.tgz", + "integrity": "sha512-Sr4SdOZ4vw6eQDvPYNxHogvrxmCIld/VenC5JbNrFwMiwd7lY/Z18ZFfo+EWNG4DD9nFlAujWAo/wGuOPHmy5A==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -2182,6 +2771,33 @@ "node": ">=6.0.0" } }, + "node_modules/dot-prop": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz", + "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==", + "license": "MIT", + "dependencies": { + "type-fest": "^2.11.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dot-prop/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -2229,12 +2845,45 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/electron-fetch": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/electron-fetch/-/electron-fetch-1.9.1.tgz", + "integrity": "sha512-M9qw6oUILGVrcENMSRRefE1MbHPIz0h79EKIeJWK9v563aT9Qkh8aEHPO1H5vi970wPirNY+jO9OpFoLiMsMGA==", + "license": "MIT", + "dependencies": { + "encoding": "^0.1.13" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/err-code": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", @@ -2943,8 +3592,13 @@ "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", @@ -2958,6 +3612,12 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-uri": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.3.tgz", + "integrity": "sha512-aLrHthzCjH5He4Z2H9YZ+v6Ujb9ocRuW6ZzkJQOrTxleEijANq4v1TsaPaVG1PZcuurEzrLcWRyYBYXD5cEiaw==", + "license": "BSD-3-Clause" + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -3127,6 +3787,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-iterator": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/get-iterator/-/get-iterator-1.0.2.tgz", + "integrity": "sha512-v+dm9bNVfOYsY1OrhaCrmyOcYoSeVvbt+hHZ0Au+T+p1y+0Uyj9aMaGIeUTT6xdpRbWzDeYKvfOslPhggQMcsg==", + "license": "MIT" + }, "node_modules/get-stdin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", @@ -3249,11 +3915,6 @@ "uint8arrays": "^5.0.1" } }, - "node_modules/hamt-sharding/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" - }, "node_modules/hamt-sharding/node_modules/uint8arrays": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", @@ -3363,6 +4024,18 @@ "he": "bin/he" } }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -3450,11 +4123,6 @@ "multiformats": "^13.2.3" } }, - "node_modules/interface-blockstore/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" - }, "node_modules/interface-store": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-6.0.2.tgz", @@ -3510,11 +4178,6 @@ "node": ">=18" } }, - "node_modules/ipfs-car/node_modules/multiformats": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.0.1.tgz", - "integrity": "sha512-bt3R5iXe2O8xpp3wkmQhC73b/lC4S2ihU8Dndwcsysqbydqb8N+bpP116qMcClZ17g58iSIwtXUTcg2zT4sniA==" - }, "node_modules/ipfs-unixfs-exporter": { "version": "13.6.1", "resolved": "https://registry.npmjs.org/ipfs-unixfs-exporter/-/ipfs-unixfs-exporter-13.6.1.tgz", @@ -3552,10 +4215,33 @@ "npm": ">=7.0.0" } }, - "node_modules/ipfs-unixfs-exporter/node_modules/multiformats": { - "version": "13.3.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==" + "node_modules/ipfs-utils": { + "version": "9.0.14", + "resolved": "https://registry.npmjs.org/ipfs-utils/-/ipfs-utils-9.0.14.tgz", + "integrity": "sha512-zIaiEGX18QATxgaS0/EOQNoo33W0islREABAcxXE8n7y2MGAlB+hdsxXn4J0hGZge8IqVQhW8sWIb+oJz2yEvg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "any-signal": "^3.0.0", + "browser-readablestream-to-it": "^1.0.0", + "buffer": "^6.0.1", + "electron-fetch": "^1.7.2", + "err-code": "^3.0.1", + "is-electron": "^2.2.0", + "iso-url": "^1.1.5", + "it-all": "^1.0.4", + "it-glob": "^1.0.1", + "it-to-stream": "^1.0.0", + "merge-options": "^3.0.4", + "nanoid": "^3.1.20", + "native-fetch": "^3.0.0", + "node-fetch": "^2.6.8", + "react-native-fetch-api": "^3.0.0", + "stream-to-it": "^0.2.2" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } }, "node_modules/is-array-buffer": { "version": "3.0.4", @@ -3687,6 +4373,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-electron": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", + "integrity": "sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==", + "license": "MIT" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -3816,7 +4508,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", - "dev": true, "engines": { "node": ">=8" } @@ -3981,6 +4672,12 @@ "node": ">=12" } }, + "node_modules/it-all": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/it-all/-/it-all-1.0.6.tgz", + "integrity": "sha512-3cmCc6Heqe3uWi3CVM/k51fa/XbMFpQVzFoDsV0IZNHSQDyAXl3c4MjHkFX5kF3922OGj7Myv1nSEUgRtcuM1A==", + "license": "ISC" + }, "node_modules/it-filter": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/it-filter/-/it-filter-3.1.1.tgz", @@ -3989,6 +4686,16 @@ "it-peekable": "^3.0.0" } }, + "node_modules/it-glob": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/it-glob/-/it-glob-1.0.2.tgz", + "integrity": "sha512-Ch2Dzhw4URfB9L/0ZHyY+uqOnKvBNeS/SMcRiPmJfpHiM0TsUZn+GkpcZxAoF3dJVdPm/PuIk3A4wlV7SUo23Q==", + "license": "ISC", + "dependencies": { + "@types/minimatch": "^3.0.4", + "minimatch": "^3.0.4" + } + }, "node_modules/it-last": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/it-last/-/it-last-3.0.6.tgz", @@ -4054,6 +4761,29 @@ "npm": ">=7.0.0" } }, + "node_modules/it-to-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/it-to-stream/-/it-to-stream-1.0.0.tgz", + "integrity": "sha512-pLULMZMAB/+vbdvbZtebC0nWBTbG581lk6w8P7DfIIIKUfa8FbY7Oi0FxZcFPbxvISs7A9E+cMpLDBc1XhpAOA==", + "license": "MIT", + "dependencies": { + "buffer": "^6.0.3", + "fast-fifo": "^1.0.0", + "get-iterator": "^1.0.2", + "p-defer": "^3.0.0", + "p-fifo": "^1.0.0", + "readable-stream": "^3.6.0" + } + }, + "node_modules/it-to-stream/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/iterator.prototype": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz", @@ -4118,6 +4848,12 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "node_modules/json-schema-typed": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.1.tgz", + "integrity": "sha512-XQmWYj2Sm4kn4WeTYvmpKEbyPsL7nBsb647c7pMe6l02/yx2+Jfc4dT6UZkEXnIUb5LhD55r2HPsJ1milQ4rDg==", + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -4294,6 +5030,30 @@ "node": ">=10" } }, + "node_modules/merge-options": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/merge-options/-/merge-options-3.0.4.tgz", + "integrity": "sha512-2Sug1+knBjkaMsMgf1ctR1Ujx+Ayku4EdJN4Z+C2+JzoeF7A3OZ9KM2GY0CpQS51NR61LTurMJrRKPhSs3ZRTQ==", + "license": "MIT", + "dependencies": { + "is-plain-obj": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -4423,9 +5183,10 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/multiformats": { - "version": "9.9.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", - "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==" + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.0.tgz", + "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==", + "license": "Apache-2.0 OR MIT" }, "node_modules/murmurhash3js-revisited": { "version": "3.0.0", @@ -4435,12 +5196,59 @@ "node": ">=8.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/native-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/native-fetch/-/native-fetch-3.0.0.tgz", + "integrity": "sha512-G3Z7vx0IFb/FQ4JxvtqGABsOTIqRWvgQz6e+erkB+JJD6LrszQtMozEHI4EkmgZQvnGHrpLVzUWk7t4sJCIkVw==", + "license": "MIT", + "peerDependencies": { + "node-fetch": "*" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -4572,6 +5380,12 @@ "wrappy": "1" } }, + "node_modules/one-webcrypto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/one-webcrypto/-/one-webcrypto-1.0.3.tgz", + "integrity": "sha512-fu9ywBVBPx0gS9K0etIROTiCkvI5S1TDjFsYFb3rC1ewFxeOqsbzq7aIMBHsYfrTHBcGXJaONXXjTl8B01cW1Q==", + "license": "MIT" + }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -4600,6 +5414,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-fifo": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-fifo/-/p-fifo-1.0.0.tgz", + "integrity": "sha512-IjoCxXW48tqdtDFz6fqo5q1UfFVjjVZe8TC1QRflvNUJtNfCUhxOUw6MOVZhDPjqhSzc26xKdugsO17gmzd5+A==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.0.0", + "p-defer": "^3.0.0" + } + }, + "node_modules/p-fifo/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -5120,6 +5953,24 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true }, + "node_modules/react-native-fetch-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-native-fetch-api/-/react-native-fetch-api-3.0.0.tgz", + "integrity": "sha512-g2rtqPjdroaboDKTsJCTlcmtw54E25OjyaunUP0anOZn4Fuo2IKs8BVfe02zVggA/UysbmfSnRJIqtNkAgggNA==", + "license": "MIT", + "dependencies": { + "p-defer": "^3.0.0" + } + }, + "node_modules/react-native-fetch-api/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -5205,6 +6056,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-in-the-middle": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.3.0.tgz", @@ -5364,6 +6224,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -5537,6 +6403,15 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/stream-to-it": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/stream-to-it/-/stream-to-it-0.2.4.tgz", + "integrity": "sha512-4vEbkSs83OahpmBybNJXlJd7d6/RxzkkSdT3I0mnGt79Xd2Kk+e1JqbvAvsQfCeKj3aKb0QIWkyK3/n0j506vQ==", + "license": "MIT", + "dependencies": { + "get-iterator": "^1.0.2" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -5677,6 +6552,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stubborn-fs": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -5700,6 +6580,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sync-multihash-sha2": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/sync-multihash-sha2/-/sync-multihash-sha2-1.0.0.tgz", + "integrity": "sha512-A5gVpmtKF0ov+/XID0M0QRJqF2QxAsj3x/LlDC8yivzgoYCoWkV+XaZPfVu7Vj1T/hYzYS1tfjwboSbXjqocug==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@noble/hashes": "^1.3.1" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -5726,6 +6615,12 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -5886,6 +6781,12 @@ "multiformats": "^9.4.2" } }, + "node_modules/uint8arrays/node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -5934,6 +6835,27 @@ "node": ">=0.10.48" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/when-exit": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.3.tgz", + "integrity": "sha512-uVieSTccFIr/SFQdFWN/fFaQYmV37OKtuaGphMAzi4DmmUlrvRBJW5WSLkHyjNQY/ePJMz3LoiX9R3yy1Su6Hw==" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index c335818..5031e2f 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,12 @@ "@glif/filecoin-address": "^3.0.12", "@influxdata/influxdb-client": "^1.35.0", "@ipld/car": "^5.3.2", + "@ipld/dag-json": "^10.2.2", "@sentry/node": "^8.35.0", + "@ucanto/core": "^10.0.1", + "@ucanto/principal": "^9.0.1", "@web3-storage/car-block-validator": "^1.2.0", + "@web3-storage/w3up-client": "^16.2.0", "debug": "^4.3.7", "drand-client": "^1.2.6", "ethers": "^6.13.4", @@ -32,6 +36,7 @@ "just-percentile": "^4.2.0", "k-closest": "^1.3.0", "ms": "^2.1.3", + "multiformats": "^13.3.0", "p-map": "^7.0.2", "p-retry": "^6.2.0", "pg": "^8.13.1", diff --git a/test/car.test.js b/test/car.test.js new file mode 100644 index 0000000..dcf9589 --- /dev/null +++ b/test/car.test.js @@ -0,0 +1,34 @@ +import { createDagJsonCar, createCar } from '../lib/car.js' +import assert from 'node:assert' +import * as dagJSON from '@ipld/dag-json' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats' +import { CarReader } from '@ipld/car' + +describe('CAR', () => { + describe('createDagJsonCar', () => { + it('Creates a CAR file (with CID) from a JSON object', async () => { + const object = { beep: 'boop' } + const { cid, car } = await createDagJsonCar(object) + assert.strictEqual(cid.toString(), 'baguqeerawg5jfpiy2g5xp5d422uwa3mpyzkmiguoeecesds7q65mn2hdoa4q') + const reader = await CarReader.fromBytes(await car.bytes()) + const block = await reader.get(cid) + assert.deepStrictEqual(block.cid, cid) + assert.deepStrictEqual(dagJSON.decode(block.bytes), object) + }) + }) + describe('createCar', () => { + it('Creates a CAR blob from a block', async () => { + const payload = 'hi' + const bytes = dagJSON.encode(payload) + const hash = await sha256.digest(bytes) + const cid = CID.create(1, dagJSON.code, hash) + const car = await createCar({ cid, bytes }, cid) + assert.strictEqual(cid.toString(), 'baguqeerawsixpycync327ducuzcmdtra4uq26rsjplpk77ugduuu3g2lw5pa') + const reader = await CarReader.fromBytes(await car.bytes()) + const block = await reader.get(cid) + assert.deepStrictEqual(block.cid, cid) + assert.strictEqual(dagJSON.decode(block.bytes), payload) + }) + }) +}) diff --git a/test/evaluate.js b/test/evaluate.js index 0d289f4..52825a7 100644 --- a/test/evaluate.js +++ b/test/evaluate.js @@ -42,6 +42,7 @@ describe('evaluate', async function () { beforeEach(async () => { await pgClient.query('DELETE FROM retrieval_stats') + await pgClient.query('DELETE FROM unpublished_provider_retrieval_result_stats_rounds') }) after(async () => { @@ -73,7 +74,8 @@ describe('evaluate', async function () { fetchRoundDetails, recordTelemetry, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats: async () => {} }) assert.strictEqual(setScoresCalls.length, 1) assert.deepStrictEqual(setScoresCalls[0].participantAddresses, [VALID_MEASUREMENT.participantAddress]) @@ -120,7 +122,8 @@ describe('evaluate', async function () { fetchRoundDetails, recordTelemetry, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats: async () => {} }) assert.strictEqual(setScoresCalls.length, 1) assert.deepStrictEqual(setScoresCalls[0].participantAddresses, [ @@ -170,7 +173,8 @@ describe('evaluate', async function () { fetchRoundDetails, recordTelemetry, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats: async () => {} }) assert.strictEqual(setScoresCalls.length, 1) assert.deepStrictEqual(setScoresCalls[0].participantAddresses, [ @@ -214,7 +218,8 @@ describe('evaluate', async function () { recordTelemetry, fetchRoundDetails, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats: async () => {} }) assert.strictEqual(setScoresCalls.length, 1) assert.deepStrictEqual(setScoresCalls[0].participantAddresses.sort(), ['0x123', '0x234']) @@ -262,7 +267,8 @@ describe('evaluate', async function () { recordTelemetry, fetchRoundDetails, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats: async () => {} }) assert.strictEqual(setScoresCalls.length, 1) const { scores, participantAddresses } = setScoresCalls[0] @@ -306,7 +312,8 @@ describe('evaluate', async function () { recordTelemetry, fetchRoundDetails, createPgClient, - logger + logger, + prepareProviderRetrievalResultStats: async () => {} }) let point = telemetry.find(p => p.name === 'retrieval_stats_honest') @@ -323,6 +330,45 @@ describe('evaluate', async function () { assertPointFieldValue(point, 'unique_tasks', '2i') assertPointFieldValue(point, 'success_rate', '0.5') }) + + it('prepares provider retrieval result stats', async () => { + const round = new RoundData(0n) + for (let i = 0; i < 5; i++) { + round.measurements.push({ ...VALID_MEASUREMENT }) + } + const setScores = async () => {} + const prepareProviderRetrievalResultStatsCalls = [] + const prepareProviderRetrievalResultStats = async (round, committees) => { + prepareProviderRetrievalResultStatsCalls.push({ round, committees }) + } + const ieContract = { + async getAddress () { + return '0x811765AccE724cD5582984cb35f5dE02d587CA12' + } + } + /** @returns {Promise} */ + const roundDetails = { ...SPARK_ROUND_DETAILS, retrievalTasks: [VALID_TASK] } + const fetchRoundDetails = async () => roundDetails + await evaluate({ + round, + roundIndex: 0n, + requiredCommitteeSize: 1, + ieContract, + setScores, + recordTelemetry, + fetchRoundDetails, + createPgClient, + logger, + prepareProviderRetrievalResultStats + }) + + assert.strictEqual(prepareProviderRetrievalResultStatsCalls.length, 1) + const call = prepareProviderRetrievalResultStatsCalls[0] + assert(call.round instanceof RoundData) + assert.strictEqual(call.round.details, roundDetails) + assert(Array.isArray(call.committees)) + assert.strictEqual(call.committees.length, 1) + }) }) describe('fraud detection', function () { diff --git a/test/helpers/test-data.js b/test/helpers/test-data.js index fe7b076..2be7ff5 100644 --- a/test/helpers/test-data.js +++ b/test/helpers/test-data.js @@ -3,6 +3,8 @@ import { groupMeasurementsToCommittees } from '../../lib/committee.js' export const VALID_PARTICIPANT_ADDRESS = '0x000000000000000000000000000000000000dEaD' export const VALID_STATION_ID = '8800000000000000000000000000000000000000000000000000000000000000000000000000000000000000' export const VALID_INET_GROUP = 'some-group-id' +export const MEASUREMENT_BATCH = 'bafybeie5rekb2jox77ow64wjjd2bjdsp6d3yeivhzzd234hnbpscfjarv4' +export const ROUND_DETAILS = 'bafybeie5rekb2jox77ow64wjjd2bjdsp6d3yeivhzzd234hnbpscfjarv4' /** @import { Measurement} from '../../lib/preprocess.js' */ diff --git a/test/preprocess.js b/test/preprocess.js index e30a5f8..cbf8462 100644 --- a/test/preprocess.js +++ b/test/preprocess.js @@ -53,6 +53,7 @@ describe('preprocess', () => { }) ]) assert.deepStrictEqual(getCalls, [cid]) + assert.deepStrictEqual(round.measurementBatches, [cid]) const point = assertRecordedTelemetryPoint(telemetry, 'spark_versions') assertPointFieldValue(point, 'round_index', '0i') diff --git a/test/provider-retrieval-result-stats.test.js b/test/provider-retrieval-result-stats.test.js new file mode 100644 index 0000000..697f39f --- /dev/null +++ b/test/provider-retrieval-result-stats.test.js @@ -0,0 +1,489 @@ +import assert from 'node:assert/strict' +import * as providerRetrievalResultStats from '../lib/provider-retrieval-result-stats.js' +import { CID } from 'multiformats' +import pg from 'pg' +import { DATABASE_URL } from '../lib/config.js' +import { migrateWithPgClient } from '../lib/migrate.js' +import { CarReader } from '@ipld/car' +import * as dagJSON from '@ipld/dag-json' +import { MEASUREMENT_BATCH, ROUND_DETAILS } from './helpers/test-data.js' + +const createPgClient = async () => { + const pgClient = new pg.Client({ connectionString: DATABASE_URL }) + await pgClient.connect() + return pgClient +} + +describe('Provider Retrieval Result Stats', () => { + let pgClient + before(async () => { + pgClient = await createPgClient() + await migrateWithPgClient(pgClient) + }) + + beforeEach(async () => { + await pgClient.query('TRUNCATE unpublished_provider_retrieval_result_stats_rounds') + }) + + after(async () => { + await pgClient.end() + }) + + describe('build()', () => { + it('should build provider retrieval result stats', () => { + // TODO: Add more committee edge cases + const stats = providerRetrievalResultStats.build([ + { + measurements: [ + { + minerId: '0', + retrievalResult: 'OK' + }, + { + minerId: '1', + retrievalResult: 'TIMEOUT' + } + ] + }, { + measurements: [ + { + minerId: '0', + retrievalResult: 'OK' + }, + { + minerId: '1', + retrievalResult: 'TIMEOUT' + } + ] + } + ]) + assert.deepStrictEqual(stats, new Map([ + ['0', { total: 2, successful: 2 }], + ['1', { total: 2, successful: 0 }] + ])) + }) + }) + describe('publishRoundDetails()', () => { + it('should publish round details', async () => { + const roundDetails = { + beep: 'boop' + } + const uploadCARCalls = [] + const storachaClient = { + uploadCAR: async car => { + uploadCARCalls.push(car) + } + } + const cid = await providerRetrievalResultStats.publishRoundDetails({ + storachaClient, + round: { + details: roundDetails + } + }) + assert(cid instanceof CID) + assert.strictEqual(uploadCARCalls.length, 1) + const reader = await CarReader.fromBytes(await uploadCARCalls[0].bytes()) + const block = await reader.get(cid) + assert.deepStrictEqual(dagJSON.decode(block.bytes), roundDetails) + }) + }) + describe('prepare()', () => { + it('should publish round details', async () => { + const uploadCARCalls = [] + const storachaClient = { + uploadCAR: async car => { + uploadCARCalls.push(car) + } + } + const createPgClient = async () => { + return { + query: async () => {}, + end: async () => {} + } + } + await providerRetrievalResultStats.prepare({ + storachaClient, + round: { + details: { + beep: 'boop' + } + }, + createPgClient, + committees: [], + sparkEvaluateVersion: 'v0', + ieContractAddress: '0x0' + }) + assert.strictEqual(uploadCARCalls.length, 1) + }) + it('should insert into the database', async () => { + const storachaClient = { + uploadCAR: async () => {} + } + const round = { + index: 0, + measurementBatches: ['0x0'], + details: { + beep: 'boop' + } + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + await providerRetrievalResultStats.prepare({ + storachaClient, + round, + createPgClient, + committees: [ + { + measurements: [ + { + minerId: '0', + retrievalResult: 'OK' + }, + { + minerId: '1', + retrievalResult: 'TIMEOUT' + } + ] + }, { + measurements: [ + { + minerId: '0', + retrievalResult: 'OK' + }, + { + minerId: '1', + retrievalResult: 'TIMEOUT' + } + ] + } + ], + sparkEvaluateVersion: 'v0', + ieContractAddress + }) + const { rows } = await pgClient.query('SELECT * FROM unpublished_provider_retrieval_result_stats_rounds') + assert.strictEqual(rows.length, 1) + assert(rows[0].evaluated_at instanceof Date) + delete rows[0].evaluated_at + assert.deepStrictEqual(rows, [{ + contract_address: ieContractAddress, + measurement_batches: round.measurementBatches, + provider_retrieval_result_stats: { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + }, + round_details: 'baguqeerawg5jfpiy2g5xp5d422uwa3mpyzkmiguoeecesds7q65mn2hdoa4q', + round_index: String(round.index), + spark_evaluate_version: sparkEvaluateVersion + }]) + }) + }) + describe('publish()', () => { + it('should upload stats to Storacha', async () => { + const round = { + index: 0, + measurementBatches: [MEASUREMENT_BATCH] + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + const yesterday = new Date() + yesterday.setDate(yesterday.getDate() - 1) + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (evaluated_at, round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + `, [ + yesterday, + round.index, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + ROUND_DETAILS, + { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + } + ]) + const uploadCARCalls = [] + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async car => { + uploadCARCalls.push(car) + } + }, + rsrContract: { + addProviderRetrievalResultStats: async () => { + return { + wait: async () => {} + } + } + } + }) + assert.strictEqual(uploadCARCalls.length, 1) + const reader = await CarReader.fromBytes(await uploadCARCalls[0].bytes()) + const block = await reader.get(uploadCARCalls[0].roots[0]) + assert.deepStrictEqual(dagJSON.decode(block.bytes), { + date: yesterday.toISOString().split('T')[0], + meta: { + rounds: [{ + contractAddress: ieContractAddress, + details: CID.parse(ROUND_DETAILS), + index: String(round.index), + measurementBatches: round.measurementBatches.map(c => CID.parse(c)), + sparkEvaluateVersion: { + gitCommit: sparkEvaluateVersion + } + }] + }, + providerRetrievalResultStats: [ + { + providerId: '0', + successful: 2, + total: 2 + }, { + providerId: '1', + successful: 0, + total: 2 + } + ] + }) + }) + it('should add stats to the RSR contract', async () => { + const round = { + index: 0, + measurementBatches: [MEASUREMENT_BATCH] + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + const roundDetailsCid = ROUND_DETAILS + const yesterday = new Date() + yesterday.setDate(yesterday.getDate() - 1) + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (evaluated_at, round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + `, [ + yesterday, + round.index, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + roundDetailsCid, + { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + } + ]) + const addProviderRetrievalResultStatsCalls = [] + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async () => {} + }, + rsrContract: { + addProviderRetrievalResultStats: async cid => { + addProviderRetrievalResultStatsCalls.push(cid) + return { + wait: async () => {} + } + } + } + }) + assert.deepStrictEqual( + addProviderRetrievalResultStatsCalls, + ['baguqeeramqwzxhhqzofl5e56ugixuthw3dyxbvzg2e4efh4guyi55pyvy5sa'] + ) + }) + it('should delete published stats from the database', async () => { + const round = { + index: 0, + measurementBatches: [MEASUREMENT_BATCH] + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + const roundDetailsCid = ROUND_DETAILS + for (let i = 0; i < 2; i++) { + const date = new Date() + date.setDate(date.getDate() - 1 + i) + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (evaluated_at, round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + `, [ + date, + round.index + i, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + roundDetailsCid, + { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + } + ]) + } + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async () => {} + }, + rsrContract: { + addProviderRetrievalResultStats: async () => { + return { + wait: async () => {} + } + } + } + }) + const { rows } = await pgClient.query('SELECT * FROM unpublished_provider_retrieval_result_stats_rounds') + assert.strictEqual(rows.length, 1) + }) + it('should noop when there is nothing in the database', async () => { + const uploadCARCalls = [] + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async car => { + uploadCARCalls.push(car) + } + }, + rsrContract: {} + }) + assert.strictEqual(uploadCARCalls.length, 0) + }) + it('should choose the rounds with the oldest evaluated_at date', async () => { + const round = { + index: 0, + measurementBatches: [MEASUREMENT_BATCH] + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + const roundDetailsCid = ROUND_DETAILS + for (let i = 0; i < 3; i++) { + const date = new Date() + date.setDate(date.getDate() - 2 + i) + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (evaluated_at, round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + `, [ + date, + round.index + i, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + roundDetailsCid, + { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + } + ]) + } + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async () => {} + }, + rsrContract: { + addProviderRetrievalResultStats: async () => { + return { + wait: async () => {} + } + } + } + }) + const { rows } = await pgClient.query('SELECT * FROM unpublished_provider_retrieval_result_stats_rounds') + assert.strictEqual(rows.length, 2) + }) + it('should publish multiple rounds for a date', async () => { + const round = { + index: 0, + measurementBatches: [MEASUREMENT_BATCH] + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + const roundDetailsCid = ROUND_DETAILS + const yesterday = new Date() + yesterday.setDate(yesterday.getDate() - 1) + for (let i = 0; i < 2; i++) { + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (evaluated_at, round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + `, [ + yesterday, + round.index + i, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + roundDetailsCid, + { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + } + ]) + } + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async () => {} + }, + rsrContract: { + addProviderRetrievalResultStats: async () => { + return { + wait: async () => {} + } + } + } + }) + const { rows } = await pgClient.query('SELECT * FROM unpublished_provider_retrieval_result_stats_rounds') + assert.strictEqual(rows.length, 0) + }) + it('should ignore data from today', async () => { + const round = { + index: 0, + measurementBatches: [MEASUREMENT_BATCH] + } + const ieContractAddress = '0x' + const sparkEvaluateVersion = 'v0' + const roundDetailsCid = ROUND_DETAILS + await pgClient.query(` + INSERT INTO unpublished_provider_retrieval_result_stats_rounds + (evaluated_at, round_index, contract_address, spark_evaluate_version, measurement_batches, round_details, provider_retrieval_result_stats) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + `, [ + new Date(), + round.index, + ieContractAddress, + sparkEvaluateVersion, + round.measurementBatches, + roundDetailsCid, + { + 0: { successful: 2, total: 2 }, + 1: { successful: 0, total: 2 } + } + ]) + await providerRetrievalResultStats.publish({ + createPgClient, + storachaClient: { + uploadCAR: async () => {} + }, + rsrContract: { + addProviderRetrievalResultStats: async () => { + return { + wait: async () => {} + } + } + } + }) + const { rows } = await pgClient.query('SELECT * FROM unpublished_provider_retrieval_result_stats_rounds') + assert.strictEqual(rows.length, 1) + }) + }) +})