diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 478ea535cd..ecbb989641 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,3 +3,6 @@ contact_links: - name: Account Limit Increase Request url: https://mailxto.com/xy5ijb about: Submit a request to increase the size limit on your Web3.Storage account! + - name: Pinning Service Access Request + url: https://mailxto.com/h58kmo + about: Submit a request to access Web3.Storage's IPFS pinning service in your account! diff --git a/package-lock.json b/package-lock.json index 3a989c40ac..2a47e56249 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10622,6 +10622,8 @@ }, "node_modules/is-unicode-supported": { "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", "license": "MIT", "engines": { "node": ">=10" @@ -11338,6 +11340,8 @@ }, "node_modules/log-symbols": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "license": "MIT", "dependencies": { "chalk": "^4.1.0", @@ -28618,7 +28622,9 @@ "version": "1.0.0" }, "is-unicode-supported": { - "version": "0.1.0" + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==" }, "is-weakmap": { "version": "2.0.1", @@ -29127,6 +29133,8 @@ }, "log-symbols": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "requires": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" @@ -34421,4 +34429,4 @@ "version": "0.1.0" } } -} +} \ No newline at end of file diff --git a/packages/api/package.json b/packages/api/package.json index 7ab757e038..873195f73e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -14,6 +14,7 @@ "publish": "wrangler publish --env $(whoami)", "build": "WEBPACK_CLI_FORCE_LOAD_ESM_CONFIG=true webpack", "test": "npm-run-all -p -r mock:backup test:e2e -s test:size", + "test:debug": "npm-run-all -p -r mock:backup test:e2e:debug -s test:size", "test:e2e": "playwright-test \"test/**/*.spec.js\" --sw src/index.js -b webkit", "test:e2e:debug": "npm run test:e2e -- --debug", "test:size": "bundlesize", diff --git a/packages/api/src/car.js b/packages/api/src/car.js index fed570a7f1..cb81bea99d 100644 --- a/packages/api/src/car.js +++ b/packages/api/src/car.js @@ -10,7 +10,7 @@ import * as pb from '@ipld/dag-pb' import retry from 'p-retry' import { GATEWAY, LOCAL_ADD_THRESHOLD, MAX_BLOCK_SIZE } from './constants.js' import { JSONResponse } from './utils/json-response.js' -import { toPinStatusEnum } from './utils/pin.js' +import { getPins, PIN_OK_STATUS, waitAndUpdateOkPins } from './utils/pin.js' import { normalizeCid } from './utils/normalize-cid.js' /** @@ -19,12 +19,6 @@ import { normalizeCid } from './utils/normalize-cid.js' const decoders = [pb, raw, cbor] -// Duration between status check polls in ms. -const PIN_STATUS_CHECK_INTERVAL = 5000 -// Max time in ms to spend polling for an OK status. -const MAX_PIN_STATUS_CHECK_TIME = 30000 -// Pin statuses considered OK. -const PIN_OK_STATUS = ['Pinned', 'Pinning', 'PinQueued'] // Times to retry the transaction after the first failure. const CREATE_UPLOAD_RETRIES = 4 // Time in ms before starting the first retry. @@ -171,25 +165,12 @@ export async function handleCarUpload (request, env, ctx, car, uploadType = 'Car // Keep querying Cluster until one of the nodes reports something other than // Unpinned i.e. PinQueued or Pinning or Pinned. if (!pins.some(p => PIN_OK_STATUS.includes(p.status))) { - tasks.push(async () => { - const start = Date.now() - while (Date.now() - start > MAX_PIN_STATUS_CHECK_TIME) { - await new Promise(resolve => setTimeout(resolve, PIN_STATUS_CHECK_INTERVAL)) - const { peerMap } = await env.cluster.status(cid) - const pins = toPins(peerMap) - if (!pins.length) { // should not happen - throw new Error('not pinning on any node') - } - - const okPins = pins.filter(p => PIN_OK_STATUS.includes(p.status)) - if (!okPins.length) continue - - for (const pin of okPins) { - await env.db.upsertPin(normalizedCid, pin) - } - return - } - }) + tasks.push(waitAndUpdateOkPins.bind( + null, + normalizeCid, + env.cluster, + env.db) + ) } if (ctx.waitUntil) { @@ -235,12 +216,7 @@ async function addToCluster (car, env) { // will be done async by bitswap instead. local: car.size > LOCAL_ADD_THRESHOLD }) - - const { peerMap } = await env.cluster.status(cid) - const pins = toPins(peerMap) - if (!pins.length) { // should not happen - throw new Error('not pinning on any node') - } + const pins = await getPins(cid, env.cluster) return { cid, pins } } @@ -350,13 +326,3 @@ function cumulativeSize (pbNodeBytes, pbNode) { // This logic is the same as used by go/js-ipfs to display the cumulative size of a dag-pb dag. return pbNodeBytes.byteLength + pbNode.Links.reduce((acc, curr) => acc + (curr.Tsize || 0), 0) } - -/** - * @param {import('@nftstorage/ipfs-cluster').StatusResponse['peerMap']} peerMap - */ -function toPins (peerMap) { - return Object.entries(peerMap).map(([peerId, { peerName, status }]) => ({ - status: toPinStatusEnum(status), - location: { peerId, peerName } - })) -} diff --git a/packages/api/src/index.js b/packages/api/src/index.js index e0f5fb3d53..ed9077b264 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -8,6 +8,7 @@ import { statusGet } from './status.js' import { carHead, carGet, carPut, carPost } from './car.js' import { uploadPost } from './upload.js' import { userLoginPost, userTokensPost, userTokensGet, userTokensDelete, userUploadsGet, userUploadsDelete, userAccountGet, userUploadsRename, userInfoGet } from './user.js' +import { pinDelete, pinGet, pinPost, pinsGet } from './pins.js' import { metricsGet } from './metrics.js' import { versionGet } from './version.js' import { @@ -44,6 +45,12 @@ router.put('/car/:cid', mode['📝'](auth['🔒'](carPut))) router.post('/upload', mode['📝'](auth['🔒'](uploadPost))) router.get('/user/uploads', mode['👀'](auth['🔒'](userUploadsGet))) +router.post('/pins', mode['📝'](auth['🔒'](pinPost))) +router.post('/pins/:requestId', mode['📝'](auth['🔒'](pinPost))) +router.get('/pins/:requestId', mode['👀'](auth['🔒'](pinGet))) +router.get('/pins', mode['👀'](auth['🔒'](pinsGet))) +router.delete('/pins/:requestId', mode['📝'](auth['🔒'](pinDelete))) + router.get('/name/:key', mode['👀'](auth['🤲'](nameGet))) router.post('/name/:key', mode['📝'](auth['🔒'](namePost))) diff --git a/packages/api/src/pins.js b/packages/api/src/pins.js new file mode 100644 index 0000000000..87eee4d38f --- /dev/null +++ b/packages/api/src/pins.js @@ -0,0 +1,491 @@ +import { JSONResponse, notFound } from './utils/json-response.js' +import { normalizeCid } from './utils/normalize-cid.js' +import { getPins, PIN_OK_STATUS, waitAndUpdateOkPins } from './utils/pin.js' + +/** + * @typedef {'queued' | 'pinning' | 'failed' | 'pinned'} apiPinStatus + */ + +/** + * + * Service API Pin object definition + * @typedef {Object} PsaPin + * @property {string} cid + * @property {string} [name] + * @property {Array.} [origins] + * @property {object} [meta] + */ + +/** + * + * Service API Pin Status definition + * @typedef {Object} PsaPinStatusResponse + * @property {string} requestId + * @property {apiPinStatus} status + * @property {string} created + * @property {Array} delegates + * @property {string} [info] + * + * @property {PsaPin} pin + */ + +/** + * @typedef {{ error: { reason: string, details?: string } }} PinDataError + */ + +/** + * + * @param {import('../../db/db-client-types.js').PinItemOutput[]} pins + * @return {apiPinStatus} status + */ +export const getEffectivePinStatus = (pins) => { + const pinStatuses = pins.map((p) => p.status) + + // TODO what happens with Sharded? I'd assumed is pinned? + + if (pinStatuses.includes('Pinned')) { + return 'pinned' + } + + if (pinStatuses.includes('Pinning')) { + return 'pinning' + } + + if (pinStatuses.includes('PinQueued') || + pinStatuses.includes('Remote')) { + return 'queued' + } + + // TODO after some time if there are no pins we should give up and return a failed + // status instead + + return 'failed' +} + +// Error messages +// TODO: Refactor errors +export const ERROR_CODE = 400 +export const ERROR_STATUS = 'INVALID_PIN_DATA' +export const INVALID_CID = 'Invalid cid' +export const INVALID_MATCH = 'Match should be a string (i.e. "exact", "iexact", "partial", "ipartial")' +export const INVALID_META = 'Meta should be an object with string values' +export const INVALID_NAME = 'Name should be a string' +export const INVALID_ORIGINS = 'Origins should be an array of strings' +export const INVALID_REQUEST_ID = 'Request id should be a string containing digits only' +export const INVALID_REPLACE = 'Existing and replacement CID are the same' +export const INVALID_STATUS = 'Status should be a list of "queued", "pinning", "pinned", or "failed"' +export const INVALID_TIMESTAMP = 'Should be a valid timestamp' +export const INVALID_LIMIT = 'Limit should be a number' +export const REQUIRED_CID = 'CID is required' +export const REQUIRED_REQUEST_ID = 'Request id is required' +export const UNPERMITTED_MATCH = 'Match should be "exact", "iexact", "partial", or "ipartial"' + +const MATCH_OPTIONS = ['exact', 'iexact', 'partial', 'ipartial'] +const STATUS_OPTIONS = ['queued', 'pinning', 'pinned', 'failed'] + +/** + * @param {import('./user').AuthenticatedRequest} request + * @param {import('./env').Env} env + * @param {import('./index').Ctx} ctx + * @return {Promise} + */ +export async function pinPost (request, env, ctx) { + const pinData = await request.json() + const { cid, name, origins, meta } = pinData + let normalizedCid + + // Require cid + if (!cid) { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: REQUIRED_CID } }, + { status: ERROR_CODE } + ) + } + + // Validate cid + try { + normalizedCid = normalizeCid(cid) + } catch (err) { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_CID } }, + { status: ERROR_CODE } + ) + } + + // Validate name + if (name && typeof name !== 'string') { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_NAME } }, + { status: ERROR_CODE } + ) + } + + // Validate origins + if (origins && !Array.isArray(origins)) { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_ORIGINS } }, + { status: ERROR_CODE } + ) + } + + // Validate meta + if (meta) { + if (typeof meta !== 'object' || Array.isArray(meta) || Object.entries(meta).some(([, v]) => typeof v !== 'string')) { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_META } }, + { status: ERROR_CODE } + ) + } + } + + const { authToken } = request.auth + const requestId = request.params ? request.params.requestId : null + + if (requestId) { + if (typeof requestId !== 'string') { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_REQUEST_ID } }, + { status: ERROR_CODE } + ) + } + + return replacePin(pinData, requestId, authToken._id, env, ctx) + } + + return createPin(normalizedCid, pinData, authToken._id, env, ctx) +} + +/** + * @param {string} normalizedCid + * @param {Object} pinData + * @param {string} authTokenId + * @param {import('./env').Env} env + * @param {import('./index').Ctx} ctx + * @return {Promise} + */ +async function createPin (normalizedCid, pinData, authTokenId, env, ctx) { + const { cid, origins } = pinData + + // deal with empty strings + const pinName = pinData.name || undefined + const pinMeta = pinData.meta || undefined + + await env.cluster.pin(cid, { + name: pinName, + origins + }) + const pins = await getPins(cid, env.cluster) + + const pinRequestData = { + sourceCid: cid, + contentCid: normalizedCid, + authKey: authTokenId, + meta: pinMeta, + name: pinName, + pins + } + + const pinRequest = await env.db.createPsaPinRequest(pinRequestData) + + /** @type {PsaPinStatusResponse} */ + const pinStatus = getPinStatus(pinRequest) + + /** @type {(() => Promise)[]} */ + const tasks = [] + + if (!pins.some(p => PIN_OK_STATUS.includes(p.status))) { + tasks.push( + waitAndUpdateOkPins.bind( + null, + normalizeCid, + env.cluster, + env.db) + ) + } + + if (ctx.waitUntil) { + tasks.forEach(t => ctx.waitUntil(t())) + } + + return new JSONResponse(pinStatus) +} + +/** + * @param {import('./user').AuthenticatedRequest} request + * @param {import('./env').Env} env + * @param {import('./index').Ctx} ctx + */ +export async function pinGet (request, env, ctx) { + if (typeof request.params.requestId !== 'string') { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_REQUEST_ID } }, + { status: ERROR_CODE } + ) + } + + const { authToken } = request.auth + + /** @type { import('../../db/db-client-types.js').PsaPinRequestUpsertOutput } */ + let pinRequest + + try { + pinRequest = await env.db.getPsaPinRequest(authToken._id, request.params.requestId) + } catch (e) { + console.error(e) + // TODO catch different exceptions + // TODO notFound error paylod does not strictly comply to spec. + return notFound() + } + + /** @type { PsaPinStatusResponse } */ + const pin = getPinStatus(pinRequest) + return new JSONResponse(pin) +} + +/** + * List all the pins matching optional filters. + * When no filter is specified, only successful pins are returned. + * + * @param {import('./user').AuthenticatedRequest} request + * @param {import('./env').Env} env + * @param {import('./index').Ctx} ctx + */ +export async function pinsGet (request, env, ctx) { + const url = new URL(request.url) + const urlParams = new URLSearchParams(url.search) + const params = Object.fromEntries(urlParams) + + const result = parseSearchParams(params) + if (result.error) { + return new JSONResponse(result.error, { status: 400 }) + } + + let pinRequests + const opts = result.data + + try { + pinRequests = await env.db.listPsaPinRequests(request.auth.authToken._id, opts) + } catch (e) { + console.error(e) + return notFound() + } + + const pins = pinRequests.results.map((pinRequest) => getPinStatus(pinRequest)) + + return new JSONResponse({ + count: pinRequests.count, + results: pins + }) +} + +/** + * Parse the list options + * + * @param {*} params + * @returns + */ +function parseSearchParams (params) { + const opts = {} + const { + cid, + name, + match, + status, + before, + after, + limit + } = params + + if (cid) { + const cids = [] + try { + cid.split(',').forEach((c) => { + normalizeCid(c) + cids.push(c) + }) + } catch (err) { + return { + error: { reason: ERROR_STATUS, details: INVALID_CID }, + data: undefined + } + } + opts.cid = cids + } + + if (name) { + if (typeof name !== 'string') { + return { + error: { reason: ERROR_STATUS, details: INVALID_NAME }, + data: undefined + } + } + opts.name = name + } + + if (match) { + if (typeof match !== 'string') { + if (!MATCH_OPTIONS.includes(match)) { + return { + error: { reason: ERROR_STATUS, details: UNPERMITTED_MATCH }, + data: undefined + } + } + + return { + error: { reason: ERROR_STATUS, details: INVALID_MATCH }, + data: undefined + } + } + opts.match = match + } + + if (status) { + const statuses = status.split(',') + const isValidStatus = status.every(v => STATUS_OPTIONS.includes(v)) + + if (!isValidStatus) { + return { + error: { reason: ERROR_STATUS, details: INVALID_STATUS }, + data: undefined + } + } + // TODO(https://github.com/web3-storage/web3.storage/issues/797): statuses need to be mapped to db statuses + opts.status = statuses + } + + if (before) { + if (typeof before !== 'string' || !Date.parse(before)) { + return { + error: { reason: ERROR_STATUS, details: INVALID_TIMESTAMP }, + data: undefined + } + } + opts.before = before + } + + if (after) { + if (typeof after !== 'string' || !Date.parse(after)) { + return { + error: { reason: ERROR_STATUS, details: INVALID_TIMESTAMP }, + data: undefined + } + } + opts.after = after + } + + if (limit) { + if (!(/^\d+$/.test(limit))) { + return { + error: { reason: ERROR_STATUS, details: INVALID_LIMIT }, + data: undefined + } + } + opts.limit = limit + } + + return { error: undefined, data: opts } +} + +/** + * Transform a PinRequest into a PinStatus + * + * @param { Object } pinRequest + * @returns { PsaPinStatusResponse } + */ +function getPinStatus (pinRequest) { + return { + requestId: pinRequest._id.toString(), + status: getEffectivePinStatus(pinRequest.pins), + created: pinRequest.created, + pin: { + cid: pinRequest.sourceCid, + name: pinRequest.name, + origins: [], + meta: pinRequest.meta + }, + // TODO(https://github.com/web3-storage/web3.storage/issues/792) + delegates: [] + } +} + +/** + * @param {import('./user').AuthenticatedRequest} request + * @param {import('./env').Env} env + * @param {import('./index').Ctx} ctx + */ +export async function pinDelete (request, env, ctx) { + const requestId = request.params.requestId + // Don't delete pin requests that don't belong to the user + const { authToken } = request.auth + + if (!requestId) { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: REQUIRED_REQUEST_ID } }, + { status: ERROR_CODE } + ) + } + + if (typeof requestId !== 'string') { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_REQUEST_ID } }, + { status: ERROR_CODE } + ) + } + + try { + // Update deleted_at (and updated_at) timestamp for the pin request. + await env.db.deletePsaPinRequest(requestId, authToken._id) + } catch (e) { + console.error(e) + // TODO catch different exceptions + // TODO notFound error paylod does not strictly comply to spec. + return notFound() + } + + return new JSONResponse({}, { status: 202 }) +} + +/** + * @param {Object} newPinData + * @param {string} requestId + * @param {string} authTokenId + * @param {import('./env').Env} env + * @param {import('./index').Ctx} ctx + */ +async function replacePin (newPinData, requestId, authTokenId, env, ctx) { + let existingPinRequest + try { + existingPinRequest = await env.db.getPsaPinRequest(authTokenId, requestId) + } catch (e) { + return notFound() + } + + const existingCid = existingPinRequest.sourceCid + if (newPinData.cid === existingCid) { + return new JSONResponse( + { error: { reason: ERROR_STATUS, details: INVALID_REPLACE } }, + { status: ERROR_CODE } + ) + } + + let pinStatus + try { + pinStatus = await createPin(existingPinRequest.contentCid, newPinData, authTokenId, env, ctx) + } catch (e) { + return new JSONResponse( + { error: { reason: `DB Error: ${e}` } }, + { status: 501 } + ) + } + + try { + await env.db.deletePsaPinRequest(requestId, authTokenId) + } catch (e) { + return new JSONResponse( + { error: { reason: `DB Error: ${e}` } }, + { status: 501 } + ) + } + + return pinStatus +} diff --git a/packages/api/src/utils/json-response.js b/packages/api/src/utils/json-response.js index f4a51c9e2f..f1eeecac84 100644 --- a/packages/api/src/utils/json-response.js +++ b/packages/api/src/utils/json-response.js @@ -1,7 +1,7 @@ /* eslint-env serviceworker */ export class JSONResponse extends Response { /** - * @param {BodyInit} body + * @param {any} body * @param {ResponseInit} [init] */ constructor (body, init = {}) { diff --git a/packages/api/src/utils/pin.js b/packages/api/src/utils/pin.js index ea42a89122..90fb9549f6 100644 --- a/packages/api/src/utils/pin.js +++ b/packages/api/src/utils/pin.js @@ -30,6 +30,15 @@ const PinStatusMap = { sharded: 'Sharded' } +// Duration between status check polls in ms. +const PIN_STATUS_CHECK_INTERVAL = 5000 + +// Max time in ms to spend polling for an OK status. +const MAX_PIN_STATUS_CHECK_TIME = 30000 + +// Pin statuses considered OK. +export const PIN_OK_STATUS = ['Pinned', 'Pinning', 'PinQueued'] + /** * Converts from cluster status string to DB pin status enum string. * @param {TrackerStatus} trackerStatus @@ -44,3 +53,94 @@ export function toPinStatusEnum (trackerStatus) { } return status } + +/** + * Function that returns list of pins for given CID. + * + * @param {string} cid cid to be looked for + * @param {import('@nftstorage/ipfs-cluster').Cluster} cluster + * @param {import('@nftstorage/ipfs-cluster').StatusResponse['peerMap']} [peerMap] Optional list of peers, if not provided the fuctions queries the cluster. + * @return {Promise.} + */ +export async function getPins (cid, cluster, peerMap) { + if (!peerMap) { + peerMap = (await cluster.status(cid)).peerMap + } + + const pins = toPins(peerMap) + + if (!pins.length) { + throw new Error('not pinning on any node') + } + + return pins +} + +/** + * Function that returns list of pins with a PIN_OK_STATUS. + * + * @param {string} cid cid to be looked for + * @param {import('@nftstorage/ipfs-cluster').Cluster} cluster + * @param {import('@nftstorage/ipfs-cluster').StatusResponse['peerMap']} [peerMap] Optional list of peers, if not provided the fuctions queries the cluster. + * @return {Promise.} + */ +export async function getOKpins (cid, cluster, peerMap) { + const pins = await getPins(cid, cluster, peerMap) + + return pins.filter(p => PIN_OK_STATUS.includes(p.status)) +} + +/** + * @param {import('@nftstorage/ipfs-cluster').StatusResponse['peerMap']} peerMap + * @return {Array.} + */ +export function toPins (peerMap) { + return Object.entries(peerMap).map(([peerId, { peerName, status }]) => ({ + status: toPinStatusEnum(status), + location: { peerId, peerName } + })) +} + +/** + * + * waitOkPins checks the status of the given CID on the cluster + * every given `checkInterval` until at least one pin has a PIN_OK_STATUS. + * + * After a given maximum `waitTime`, if no OK pins are found the promise is resolved with an empty array. + * + * @param {string} cid + * @param {import('@nftstorage/ipfs-cluster').Cluster} cluster + * @param {number} waitTime + * @param {number} checkInterval + * @return {Promise.} + */ +export async function waitOkPins (cid, cluster, waitTime = MAX_PIN_STATUS_CHECK_TIME, checkInterval = PIN_STATUS_CHECK_INTERVAL) { + const start = Date.now() + while (Date.now() - start < waitTime) { + await new Promise(resolve => setTimeout(resolve, checkInterval)) + const okPins = await getOKpins(cid, cluster) + if (!okPins.length) continue + + return okPins + } + return [] +} + +/** + * Used to async wait for pins for the provided cid to have an + * OK_STATUS and update them in the db. + * + * @param {string} cid + * @param {import('@nftstorage/ipfs-cluster').Cluster} cluster + * @param {import('@web3-storage/db').DBClient} db + * @param {number} waitTime + * @param {number} checkInterval + * @return {Promise.} + */ +export async function waitAndUpdateOkPins (cid, cluster, db, waitTime = MAX_PIN_STATUS_CHECK_TIME, checkInterval = PIN_STATUS_CHECK_INTERVAL) { + const okPins = await waitOkPins(cid, cluster, waitTime, checkInterval) + for (const pin of okPins) { + await db.upsertPin(cid, pin) + } + return okPins +} diff --git a/packages/api/test/fixtures/init-data.sql b/packages/api/test/fixtures/init-data.sql index 55038110c2..75adbf16ab 100644 --- a/packages/api/test/fixtures/init-data.sql +++ b/packages/api/test/fixtures/init-data.sql @@ -39,8 +39,8 @@ VALUES (3, 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 'bafyb (3, 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'Car', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), (3, 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'Car', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'); -INSERT INTO pin_location (id, peer_id, peer_name, region) -VALUES (1, '12D3KooWR1Js', 'who?', 'where?'); +INSERT INTO pin_location (peer_id, peer_name, region) +VALUES ('12D3KooWR1Js', 'who?', 'where?'); INSERT INTO pin (status, content_cid, pin_location_id, inserted_at, updated_at) VALUES ('Pinned', 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 1, '2021-07-14T19:27:14.934572+00:00', '2021-07-14T19:27:14.934572+00:00'), @@ -68,3 +68,25 @@ VALUES ( 1, 1669394359626000000 ); + +-- /pins route testing +INSERT INTO content (cid) +VALUES ('bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4'), + ('bafybeig7yvw6a4uhio4pmg5gahyd2xumowkfljdukad7pmdsv5uk5zcseu'), + ('bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu'), + ('bafybeidw7pc6nvm7u4rfhpctac4qgtpmwxapw4duugvsl3ppivvzibdlgy'), + ('bafybeidrzt6t4k25qjeasydgi3fyh6ejos5x4d6tk2pdzxkb66bkomezy4'), + ('bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm'), + ('bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa'), + ('bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy'); + + +INSERT INTO psa_pin_request (id, auth_key_id, content_cid, source_cid, meta, name, inserted_at, updated_at) +VALUES ('ab62cf3c-c98d-494b-a756-b3a3fb6ddcab', 3, 'bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4', 'bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4', null, 'ReportDoc.pdf', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('bebd5f62-1381-4124-93a1-1e4eeed52635', 3, 'bafybeig7yvw6a4uhio4pmg5gahyd2xumowkfljdukad7pmdsv5uk5zcseu', 'bafybeig7yvw6a4uhio4pmg5gahyd2xumowkfljdukad7pmdsv5uk5zcseu', '{"app_id": "99986338-1113-4706-8302-4420da6158aa"}', 'reportdoc.pdf', '2021-01-01T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('d9670c3b-179c-4799-b4fa-66e2bb82c84f', 3, 'bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu', 'bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu', null, 'Data', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('63992f6e-5bbf-4d01-8a69-9e0561c38b04', 3, 'bafybeidw7pc6nvm7u4rfhpctac4qgtpmwxapw4duugvsl3ppivvzibdlgy', 'bafybeidw7pc6nvm7u4rfhpctac4qgtpmwxapw4duugvsl3ppivvzibdlgy', null, 'Image.jpeg', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('3fa630f2-f22c-486f-ad1c-b36d4d740e31', 3, 'bafybeidrzt6t4k25qjeasydgi3fyh6ejos5x4d6tk2pdzxkb66bkomezy4', 'bafybeidrzt6t4k25qjeasydgi3fyh6ejos5x4d6tk2pdzxkb66bkomezy4', null, 'Image.png', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('5c7e7885-7f68-462d-bdfb-3f0abfb367b5', 3, 'bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm', 'bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm', null, 'Image.jpg', '2021-07-20T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('3a19e48d-d6db-4f36-b686-fb8bc37c9d48', 2, 'bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa', 'bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa', null, 'Image.jpg', '2021-07-20T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'), + ('9be23b92-918e-44b8-98f4-6043c346fb4e', 2, 'bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', 'bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', null, 'Image.jpg', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'); diff --git a/packages/api/test/mocks/cluster/post_pins#@cid.js b/packages/api/test/mocks/cluster/post_pins#@cid.js new file mode 100644 index 0000000000..b93a74af59 --- /dev/null +++ b/packages/api/test/mocks/cluster/post_pins#@cid.js @@ -0,0 +1,32 @@ +/** + * @param {any} data + */ +async function run (data) { + const { query, params } = data + + return { + statusCode: 200, + headers: {}, + body: { + allocations: [], + cid: { + '/': params.cid + }, + expire_at: '0001-01-01T00:00:00Z', + max_depth: -1, + metadata: null, + mode: 'recursive', + name: query.name, + origins: [], + pin_update: null, + reference: null, + replication_factor_max: -1, + replication_factor_min: -1, + shard_size: 0, + type: 2, + user_allocations: null + } + } +} + +module.exports = run diff --git a/packages/api/test/pin.spec.js b/packages/api/test/pin.spec.js new file mode 100644 index 0000000000..35f51cc51f --- /dev/null +++ b/packages/api/test/pin.spec.js @@ -0,0 +1,747 @@ +/* eslint-env mocha, browser */ +import assert from 'assert' +import { endpoint } from './scripts/constants.js' +import { getTestJWT } from './scripts/helpers.js' +import { + ERROR_CODE, + ERROR_STATUS, + getEffectivePinStatus, + INVALID_CID, + INVALID_META, + INVALID_NAME, + INVALID_ORIGINS, + REQUIRED_CID, + INVALID_LIMIT, + INVALID_REPLACE +} from '../src/pins.js' + +/** + * + * @param {string} status + * @returns {import('../../db/db-client-types.js').PinItemOutput} + */ +const createPinWithStatus = (status) => { + const now = new Date().toISOString() + return { + _id: (Math.random() * 100).toString(), + status: status, + created: now, + updated: now, + peerId: (Math.random() * 100).toString(), + peerName: 'name', + region: 'region' + } +} + +/** + * + * @param {object} data + */ +const assertCorrectPinResponse = (data) => { + assert.ok(typeof data.requestId === 'string', 'requestId should be a string') + assert.doesNotThrow(() => parseInt(data.requestId, 10), 'requestId is a stringified number') + assert.ok(typeof data.status === 'string', 'status should be a string') + assert.ok(['queued', 'pinning', 'pinned', 'failed'].includes(data.status), 'it has a valid status') + assert.ok(Date.parse(data.created), 'created should be valid date string') + assert.ok(Array.isArray(data.delegates), 'delegates should be an array') + + if (data.info) { + assert.ok(typeof data.info === 'object', 'info should be an object') + } + + assert.ok(typeof data.pin === 'object', 'pin should be an object') + assert.ok(typeof data.pin.cid === 'string', 'pin.cid should be an string') + + if (data.pin.name) { + assert.ok(typeof data.pin.name === 'string', 'pin.name should be an string') + } + + if (data.pin.origins) { + assert.ok(Array.isArray(data.pin.origins), 'pin.origins should be an array') + } + + if (data.pin.meta) { + assert.ok(typeof data.pin.meta === 'object', 'pin.meta should be an object') + } +} + +/** + * + * @param {string} cid + * @param {string} token + * @return {Promise} + */ +const createPinRequest = async (cid, token) => { + return await (await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ cid }) + })).json() +} + +describe('Pinning APIs endpoints', () => { + let token = null + + before(async () => { + token = await getTestJWT('test-upload', 'test-upload') + }) + + describe('GET /pins', () => { + let baseUrl + + before(async () => { + baseUrl = new URL('pins', endpoint).toString() + }) + + it('validates filter values', async () => { + const opts = new URLSearchParams({ + limit: '3.14' + }) + const url = new URL(`${baseUrl}?${opts}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const error = await res.json() + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_LIMIT) + }) + + it('validates CID values passed as filter', async () => { + const cids = [ + 'notAValidCID', + 'bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu', + 'bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4' + ] + + const url = new URL(`${baseUrl}?cid=${cids.join(',')}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert.strictEqual(res.status, ERROR_CODE) + const error = await res.json() + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_CID) + }) + + it('returns the pins for this user with default filter values', async () => { + const res = await fetch( + baseUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 6) + }) + + it('filters pins on CID, for this user', async () => { + const cids = [ + 'bafybeig7yvw6a4uhio4pmg5gahyd2xumowkfljdukad7pmdsv5uk5zcseu', + 'bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu', + 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4' // Not exists + ] + + const url = new URL(`${baseUrl}?cid=${cids.join(',')}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 2) + }) + + it('filters case sensitive exact match on name', async () => { + const opts = new URLSearchParams({ + name: 'ReportDoc.pdf' + }) + const url = new URL(`${baseUrl}?${opts}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 1) + }) + + it('filters case insensitive partial match on name', async () => { + const opts = new URLSearchParams({ + name: 'image', + match: 'ipartial' + }) + const url = new URL(`${baseUrl}?${opts}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 3) + }) + + it('filters pins created before a date', async () => { + const opts = new URLSearchParams({ + before: '2021-07-01T00:00:00.000000Z' + }) + const url = new URL(`${baseUrl}?${opts}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 1) + }) + + it('filters pins created after a date', async () => { + const opts = new URLSearchParams({ + after: '2021-07-15T00:00:00.000000Z' + }) + const url = new URL(`${baseUrl}?${opts}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 1) + }) + + it('limits the number of pins returned for this user and includes the total', async () => { + const opts = new URLSearchParams({ + limit: '3' + }) + const url = new URL(`${baseUrl}?${opts}`).toString() + const res = await fetch( + url, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + const data = await res.json() + assert.strictEqual(data.count, 6) + assert.strictEqual(data.results.length, 3) + }) + }) + + describe('POST /pins', () => { + it('should receive pin data containing cid', async () => { + const cid = 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e' + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ cid }) + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response ok') + const data = await res.json() + assert.strictEqual(data.pin.cid, cid) + }) + + it('requires cid', async () => { + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({}) + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const data = await res.json() + const error = data.error + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, REQUIRED_CID) + }) + + it('throws error if cid is invalid', async () => { + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + cid: 'abc' + }) + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const data = await res.json() + const error = data.error + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_CID) + }) + + it('should receive pin data containing cid, name, origin, meta', async () => { + const cid = 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e' + const meta = { app_id: '99986338-1113-4706-8302-4420da6158aa' } + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + cid, + name: 'PreciousData.pdf', + origins: [ + '/ip6/2606:4700:60::6/tcp/4009/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N', + '/ip4/172.65.0.13/tcp/4009/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx4N' + ], + meta + }) + }) + + assert(res, 'Server responded') + assert(res.ok, 'Server response ok') + const data = await res.json() + assert.strictEqual(data.pin.cid, cid) + assert.deepStrictEqual(data.pin.meta, meta) + }) + + it('validates name', async () => { + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + cid: 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e', + name: 1 + }) + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const data = await res.json() + const error = data.error + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_NAME) + }) + + it('validates origins', async () => { + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + cid: 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e', + origins: 1 + }) + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const data = await res.json() + const error = data.error + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_ORIGINS) + }) + + it('validates meta', async () => { + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + cid: 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e', + meta: 1 + }) + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const data = await res.json() + const error = data.error + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_META) + }) + + it('validates meta values', async () => { + const res = await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + cid: 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e', + meta: { + app_id: 1 + } + }) + }) + + assert(res, 'Server responded') + assert.strictEqual(res.status, ERROR_CODE) + const data = await res.json() + const error = data.error + assert.strictEqual(error.reason, ERROR_STATUS) + assert.strictEqual(error.details, INVALID_META) + }) + }) + + describe('GET /pins/:requestId', () => { + let pinRequest + + before(async () => { + // Create token + token = await getTestJWT('test-upload', 'test-upload') + + const cid = 'bafybeihy6bymmfcdjdrkhaha2srphnhrewimtkdxdmcama2dpgvpyx4efu' + pinRequest = await (await fetch(new URL('pins', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ cid }) + })).json() + }) + + it('returns unauthorized if no token', async () => { + const res = await fetch(new URL('pins/1', endpoint).toString(), { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert(!res.ok) + assert.deepEqual(res.status, 401) + }) + + it('returns not found if the request does not exists', async () => { + const pinThatDoesNotExists = 100 + const res = await fetch(new URL(`pins/${pinThatDoesNotExists}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + + assert(res, 'Server responded') + assert.deepEqual(res.status, 404) + }) + + it('returns not found if the request does not belong to the user token', async () => { + const wrongToken = await getTestJWT() + const res = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'GET', + headers: { Authorization: `Bearer ${wrongToken}` } + }) + + assert(res, 'Server responded') + assert.deepEqual(res.status, 404) + }) + + it('returns the pin request', async () => { + const res = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + const data = await res.json() + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + assertCorrectPinResponse(data) + assert.deepEqual(data.requestId, pinRequest.requestId) + assert.deepEqual(data.status, 'pinning') + }) + + it('returns the pin request with specified name', async () => { + const requestId = 'bebd5f62-1381-4124-93a1-1e4eeed52635' + + const res = await fetch(new URL(`pins/${requestId}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + const data = await res.json() + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + assertCorrectPinResponse(data) + assert.strictEqual(data.requestId, requestId.toString()) + assert.strictEqual(data.pin.name, 'reportdoc.pdf') + }) + + it('returns the pin request with specified metadata', async () => { + const requestId = 'bebd5f62-1381-4124-93a1-1e4eeed52635' + const meta = { app_id: '99986338-1113-4706-8302-4420da6158aa' } + + const res = await fetch(new URL(`pins/${requestId}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + } + }) + const data = await res.json() + + assert(res, 'Server responded') + assert(res.ok, 'Server response is ok') + assertCorrectPinResponse(data) + assert.strictEqual(data.requestId, requestId.toString()) + assert.deepStrictEqual(data.pin.meta, meta) + }) + }) + + describe('getEffectivePinStatus', () => { + it('should return pinned if at it is pinned in at least a node', () => { + /** @type {import('../../db/db-client-types.js').PinItemOutput[]} */ + const pins = [ + createPinWithStatus('Pinned'), + createPinWithStatus('PinQueued'), + createPinWithStatus('PinError') + ] + assert.strictEqual(getEffectivePinStatus(pins), 'pinned') + }) + + it('should return queued if there are no pins yet', () => { + const pins = [] + assert.strictEqual(getEffectivePinStatus(pins), 'failed') + }) + + it('should return "queued" at least 1 pin has it queued', () => { + const pins = [ + createPinWithStatus('UnpinQueued'), + createPinWithStatus('PinError'), + createPinWithStatus('PinQueued') + ] + assert.strictEqual(getEffectivePinStatus(pins), 'queued') + }) + + it('should return "queued" at least 1 pin has remote status', () => { + const pins = [ + createPinWithStatus('UnpinQueued'), + createPinWithStatus('PinError'), + createPinWithStatus('PinQueued') + ] + + assert.strictEqual(getEffectivePinStatus(pins), 'queued') + }) + + it('should return failed pins have statuses other than Pinned, Pinning, PinQueued or Remote', () => { + const pins = [ + createPinWithStatus('UnpinQueued'), + createPinWithStatus('PinError') + ] + + assert.strictEqual(getEffectivePinStatus(pins), 'failed') + }) + }) + + describe('DELETE /pins/:requestId', () => { + it('fails to delete if there is no user token', async () => { + const res = await fetch(new URL('pins/ab62cf3c-c98d-494b-a756-b3a3fb6ddcab', endpoint).toString(), { + method: 'DELETE' + }) + + assert(res, 'Server responded') + assert(!res.ok) + assert.deepEqual(res.status, 401) + }) + + it('returns not found if the request does not exists', async () => { + const pinThatDoesNotExists = 'idThatDoesNotExists' + const res = await fetch(new URL(`pins/${pinThatDoesNotExists}`, endpoint).toString(), { + method: 'DELETE', + headers: { + Authorization: `Bearer ${token}` + } + }) + + assert(res, 'Server responded') + assert.deepEqual(res.status, 404) + }) + + it('deletes the pin request', async () => { + const cid = 'bafybeifzequu4ial7i4jdw4gxabi5xyx2qeci2o4scc65s2st5o7fsynqu' + const pinRequest = await createPinRequest(cid, token) + + const r = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}` + } + }) + + assert(r.ok, 'It did not create the request in the first place') + + const resD = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'DELETE', + headers: { + Authorization: `Bearer ${token}` + } + }) + + assert.equal(resD.status, 202, 'Delete request was not successful') + + const res = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}` + } + }) + + assert.equal(res.status, 404) + }) + + it('returns the pin request id that has been deleted', async () => { + const requestId = '5c7e7885-7f68-462d-bdfb-3f0abfb367b5' + const res = await fetch(new URL(`pins/${requestId}`, endpoint).toString(), { + method: 'DELETE', + headers: { + Authorization: `Bearer ${token}` + } + }) + + assert(res.ok, 'Server responded') + assert.equal(res.status, 202) + }) + }) + + describe('POST /pins/:requestId', () => { + it('should not replace a pin request that doesn\'t exist', async () => { + const res = await fetch(new URL('pins/100', endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + cid: 'bafybeibqmrg5e5bwhx2ny4kfcjx2mm3ohh2cd4i54wlygquwx7zbgwqs4e' + }) + }) + + assert(res, 'Server responded') + assert.equal(res.status, 404) + const { message } = await res.json() + assert.equal(message, 'Not Found') + }) + + it('should delete the pin request and replace it', async () => { + const cid = 'bafybeid3ka3b3f443kv2je3mfm4byk6qps3wipr7wzu5uli6tdo57crcke' + const newCid = 'bafybeid4f2r3zpnkjqrglkng265ttqg6zbdr75dpbiwellvlpcxq7pggjy' + + // Creates pin Requests + const pinRequest = await createPinRequest(cid, token) + + // It replaces it + const resR = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + cid: newCid + }) + }) + + assert(resR, 'Replace request did not respond') + assert(resR.ok, 'Replace request was not successful') + + const resG = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${token}` + } + }) + + assert(resG, 'Get request did not respond') + assert.strictEqual(resG.status, 404, 'Pin request was not deleted') + }) + + it('should not replace the same pin request', async () => { + const cid = 'bafybeieppxukl4i4acnjcdj2fueoa5oppuaciseggv2cvplj2iu6d7kx2e' + const pinRequest = await createPinRequest(cid, token) + const res = await fetch(new URL(`pins/${pinRequest.requestId}`, endpoint).toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + cid + }) + }) + + assert(res, 'Server responded') + assert(!res.ok) + assert.equal(res.status, 400) + const { error } = await res.json() + assert.equal(error.details, INVALID_REPLACE) + }) + }) +}) diff --git a/packages/db/db-client-types.ts b/packages/db/db-client-types.ts index 082d549a74..519df80e0c 100644 --- a/packages/db/db-client-types.ts +++ b/packages/db/db-client-types.ts @@ -2,7 +2,7 @@ import { definitions } from './postgres/pg-rest-api-types' // User export type UpsertUserInput = { - id: definitions['user']['id'], + id?: definitions['user']['id'], name: definitions['user']['name'], picture?: definitions['user']['picture'], email: definitions['user']['email'], @@ -62,7 +62,7 @@ export type AuthKeyItemOutput = { // Pin export type PinUpsertInput = { - id?: definitions['pin']['id'] + id?: definitions['pin']['id'], status: definitions['pin']['status'], location: Location, } @@ -129,6 +129,16 @@ export type Deal = { } // Content +export type ContentInput = { + cid: definitions['content']['cid'] + dagSize: definitions['content']['dag_size'] + pins: Array<{ + status: definitions['pin']['status'] + updated: definitions['pin']['updated_at'] + location: Location + }> +} + export type ContentItem = { cid: definitions['content']['cid'] dagSize: definitions['content']['dag_size'] @@ -178,15 +188,7 @@ export type UploadItem = { name?: definitions['upload']['name'] created?: definitions['upload']['inserted_at'] updated?: definitions['upload']['updated_at'] - content: { - cid: definitions['content']['cid'] - dagSize: definitions['content']['dag_size'] - pins: Array<{ - status: definitions['pin']['status'] - updated: definitions['pin']['updated_at'] - location: Location - }> - } + content: ContentItem } export type UploadItemOutput = { @@ -242,6 +244,81 @@ export type ListUploadsOptions = { sortOrder?: 'Asc' | 'Desc' } + +// Pinninng + +// PinRequest +export type PsaPinRequestUpsertInput = { + id?: string, + name?: definitions['psa_pin_request']['name'], + meta?: definitions['psa_pin_request']['meta'], + authKey: string, + sourceCid: definitions['psa_pin_request']['source_cid'], + contentCid: definitions['upload']['content_cid'], + dagSize?: definitions['content']['dag_size'], + pins: Array, + created?: definitions['upload']['inserted_at'], + updated?: definitions['upload']['updated_at'], +} + +export type PsaPinRequestItem = PsaPinRequestUpsertInput & { + _id: string, + contentCid: definitions['psa_pin_request']['content_cid'] + created: definitions['upload']['inserted_at'] + updated: definitions['upload']['updated_at'] + deleted?: definitions['upload']['deleted_at'] + content: ContentItem +} + +export type PsaPinRequestUpsertOutput = PsaPinRequestUpsertInput & { + _id: string, + contentCid: definitions['psa_pin_request']['content_cid'] + created: definitions['psa_pin_request']['inserted_at'] + updated: definitions['psa_pin_request']['updated_at'] + deleted?: definitions['psa_pin_request']['deleted_at'] + pins: Array +} + +export type ListPsaPinRequestOptions = { + /** + * Comma-separated list of CIDs to match + */ + cid?: string[] + /** + * Name to match + */ + name?: string + /** + * Match (default: exact) + */ + match?: "exact" | "iexact" | "partial" | "ipartial" + /** + * status to match + */ + status?: Array + /** + * Uploads created before a given timestamp. + */ + before?: string + /** + * Uploads created after a given timestamp. + */ + after?: string + /** + * Max records (default: 10). + */ + limit?: number + /** + * TODO. + */ + meta?: unknown, +} + +export type ListPsaPinRequestResults = { + count: number, + results: Array +} + export type NameItem = { record: definitions['name']['record'] } diff --git a/packages/db/index.d.ts b/packages/db/index.d.ts index b3423c0689..fb2e85cd56 100644 --- a/packages/db/index.d.ts +++ b/packages/db/index.d.ts @@ -20,7 +20,13 @@ import type { PinRequestItemOutput, PinSyncRequestOutput, PinUpsertInput, - BackupOutput + BackupOutput, + PsaPinRequestItem, + PsaPinRequestUpsertOutput, + PsaPinRequestUpsertInput, + ContentInput, + ListPsaPinRequestOptions, + ListPsaPinRequestResults, } from './db-client-types' export { gql } @@ -37,7 +43,7 @@ export class DBClient { renameUpload (userId: number, cid: string, name: string): Promise<{ name: string }> deleteUpload (userId: number, cid: string): Promise<{ _id: number }> getStatus (cid: string): Promise - getBackups(uploadId: number): Promise> + getBackups (uploadId: number): Promise> upsertPin (cid: string, pin: PinUpsertInput): Promise upsertPins (pins: Array): Promise getPins (cid: string): Promise> @@ -51,6 +57,11 @@ export class DBClient { createKey (key: CreateAuthKeyInput): Promise getKey (issuer: string, secret: string): Promise listKeys (userId: number): Promise> + createPsaPinRequest (pinRequest: PsaPinRequestUpsertInput): Promise + getPsaPinRequest (authKey: string, pinRequestId: string) : Promise + listPsaPinRequests (authKey: string, opts?: ListPsaPinRequestOptions ) : Promise + deletePsaPinRequest (pinRequestId: string, authKey: string) : Promise + createContent (content: ContentInput, opt?: {updatePinRequests?: boolean}) : Promise deleteKey (id: number): Promise query(document: RequestDocument, variables: V): Promise } diff --git a/packages/db/index.js b/packages/db/index.js index 14eca63668..25dc9766af 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -1,6 +1,6 @@ import { PostgrestClient } from '@supabase/postgrest-js' -import { normalizeUpload, normalizeContent, normalizePins, normalizeDeals } from './utils.js' +import { normalizeUpload, normalizeContent, normalizePins, normalizeDeals, normalizePsaPinRequest } from './utils.js' import { DBError } from './errors.js' import { getUserMetrics, @@ -19,6 +19,20 @@ const uploadQuery = ` updated:updated_at, content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, region))) ` + +const psaPinRequestTableName = 'psa_pin_request' +const pinRequestSelect = ` + _id:id::text, + sourceCid:source_cid, + contentCid:content_cid, + authKey:auth_key_id::text, + name, + meta, + deleted:deleted_at, + created:inserted_at, + updated:updated_at, + content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id::text, peerId:peer_id, peerName:peer_name, region))) ` + /** * @typedef {import('./postgres/pg-rest-api-types').definitions} definitions * @typedef {import('@supabase/postgrest-js').PostgrestError} PostgrestError @@ -349,10 +363,10 @@ export class DBClient { * * @param {string} cid * @param {import('./db-client-types').PinUpsertInput} pin - * @return {Promise} + * @return {Promise} */ async upsertPin (cid, pin) { - /** @type {{ data: number, error: PostgrestError }} */ + /** @type {{ data: string, error: PostgrestError }} */ const { data: pinId, error } = await this._client.rpc('upsert_pin', { data: { content_cid: cid, @@ -558,7 +572,7 @@ export class DBClient { * @return {Promise>} */ async getDealsForCids (cids = []) { - /** @type {{ data: Array, error: PostgrestError }} */ + /** @type {{ data: Array, error: PostgrestError }} */ const { data, error } = await this._client .rpc('find_deals_by_content_cids', { cids @@ -747,6 +761,169 @@ export class DBClient { } } + /** + * Creates a Pin Request. + * + * @param {import('./db-client-types').PsaPinRequestUpsertInput} pinRequestData + * @return {Promise} + */ + async createPsaPinRequest (pinRequestData) { + const now = new Date().toISOString() + + /** @type {{ data: string, error: PostgrestError }} */ + const { data: pinRequestId, error } = await this._client.rpc('create_psa_pin_request', { + data: { + auth_key_id: pinRequestData.authKey, + content_cid: pinRequestData.contentCid, + source_cid: pinRequestData.sourceCid, + name: pinRequestData.name, + meta: pinRequestData.meta, + dag_size: pinRequestData.dagSize, + inserted_at: pinRequestData.created || now, + updated_at: pinRequestData.updated || now, + pins: pinRequestData.pins.map(pin => ({ + status: pin.status, + location: { + peer_id: pin.location.peerId, + peer_name: pin.location.peerName, + region: pin.location.region + } + })) + } + }).single() + + if (error) { + throw new DBError(error) + } + + // TODO: this second request could be avoided by returning the right data + // from create_psa_pin_request remote procedure. (But to keep this DRY we need to refactor + // this a bit) + return await this.getPsaPinRequest(pinRequestData.authKey, pinRequestId) + } + + /** + * Get a Pin Request by id + * + * @param {string} authKey + * @param {number} pinRequestId + * @return {Promise} + */ + async getPsaPinRequest (authKey, pinRequestId) { + /** @type {{data: import('./db-client-types').PsaPinRequestItem, error: PostgrestError }} */ + const { data, error } = await this._client + .from(psaPinRequestTableName) + .select(pinRequestSelect) + .match({ auth_key_id: authKey, id: pinRequestId }) + .is('deleted_at', null) + .single() + + if (error) { + throw new DBError(error) + } + + return normalizePsaPinRequest(data) + } + + /** + * Get a filtered list of pin requests for a user + * + * @param {string} authKey + * @param {import('./db-client-types').ListPsaPinRequestOptions} opts + * @return {Promise }> } + */ + async listPsaPinRequests (authKey, opts = {}) { + const match = opts?.match || 'exact' + const limit = opts?.limit || 10 + + let query = this._client + .from(psaPinRequestTableName) + .select(pinRequestSelect) + .eq('auth_key_id', authKey) + .order('inserted_at', { ascending: false }) + + if (opts.status) { + query = query.in('content.pins.status', opts.status) + } + + if (opts.cid) { + query = query.in('source_cid', opts.cid) + } + + if (opts.name && match === 'exact') { + query = query.like('name', `${opts.name}`) + } + + if (opts.name && match === 'iexact') { + query = query.ilike('name', `${opts.name}`) + } + + if (opts.name && match === 'partial') { + query = query.like('name', `%${opts.name}%`) + } + + if (opts.name && match === 'ipartial') { + query = query.ilike('name', `%${opts.name}%`) + } + + if (opts.before) { + query = query.lte('inserted_at', opts.before) + } + + if (opts.after) { + query = query.gte('inserted_at', opts.after) + } + + // TODO(https://github.com/web3-storage/web3.storage/issues/798): filter by meta is missing + + /** @type {{ data: Array, error: Error }} */ + const { data, error } = (await query) + + if (error) { + throw new DBError(error) + } + + const count = data.length + + // TODO(https://github.com/web3-storage/web3.storage/issues/804): Not limiting the query might cause + // performance issues if a user created lots of requests with a token. We should improve this. + const pinRequests = data.slice(0, limit) + const pins = pinRequests.map(pinRequest => normalizePsaPinRequest(pinRequest)) + + return { + count, + results: pins + } + } + + /** + * Delete a user PA pin request. + * + * @param {number} requestId + * @param {string} authKey + */ + async deletePsaPinRequest (requestId, authKey) { + const date = new Date().toISOString() + /** @type {{ data: import('./db-client-types').PsaPinRequestItem, error: PostgrestError }} */ + const { data, error } = await this._client + .from(psaPinRequestTableName) + .update({ + deleted_at: date, + updated_at: date + }) + .match({ auth_key_id: authKey, id: requestId }) + .filter('deleted_at', 'is', null) + .single() + + if (error) { + throw new DBError(error) + } + + return { + _id: data.id + } + } + /** * Get the raw IPNS record for a given key. * diff --git a/packages/db/postgres/functions.sql b/packages/db/postgres/functions.sql index dba96a8ab1..29f262b301 100644 --- a/packages/db/postgres/functions.sql +++ b/packages/db/postgres/functions.sql @@ -41,18 +41,19 @@ BEGIN END $$; -CREATE OR REPLACE FUNCTION create_upload(data json) RETURNS TEXT + +-- Creates a content table, with relative pins and pin_requests +CREATE OR REPLACE FUNCTION create_content(data json) RETURNS TEXT LANGUAGE plpgsql volatile PARALLEL UNSAFE AS $$ DECLARE - backup_url TEXT; pin json; pin_result_id BIGINT; pin_location_result_id BIGINT; - inserted_upload_id BIGINT; + inserted_cid TEXT; BEGIN -- Set timeout as imposed by heroku SET LOCAL statement_timeout = '30s'; @@ -63,7 +64,8 @@ BEGIN (data ->> 'dag_size')::BIGINT, (data ->> 'updated_at')::timestamptz, (data ->> 'inserted_at')::timestamptz) - ON CONFLICT ( cid ) DO NOTHING; + ON CONFLICT ( cid ) DO NOTHING + returning cid into inserted_cid; -- Add to pin_request table if new insert into pin_request (content_cid, attempts, updated_at, inserted_at) @@ -107,6 +109,29 @@ BEGIN END IF; end loop; + return (inserted_cid); +END +$$; + +-- Creates an upload with relative content, pins, pin_requests and backups. +CREATE OR REPLACE FUNCTION create_upload(data json) RETURNS TEXT + LANGUAGE plpgsql + volatile + PARALLEL UNSAFE +AS +$$ +DECLARE + backup_url TEXT; + pin json; + pin_result_id BIGINT; + pin_location_result_id BIGINT; + inserted_upload_id BIGINT; +BEGIN + -- Set timeout as imposed by heroku + SET LOCAL statement_timeout = '30s'; + + PERFORM create_content(data); + insert into upload (user_id, auth_key_id, content_cid, @@ -145,6 +170,48 @@ BEGIN END $$; +-- Creates a pin request with relative content, pins, pin_requests and backups. +CREATE OR REPLACE FUNCTION create_psa_pin_request(data json) RETURNS TEXT + LANGUAGE plpgsql + volatile + PARALLEL UNSAFE +AS +$$ +DECLARE +-- TODO - Validate UUID type is available + inserted_pin_request_id TEXT; +BEGIN + -- Set timeout as imposed by heroku + SET LOCAL statement_timeout = '30s'; + + PERFORM create_content(data); + + insert into psa_pin_request ( + auth_key_id, + content_cid, + source_cid, + name, + meta, + inserted_at, + updated_at + ) + values ( + (data ->> 'auth_key_id')::BIGINT, + data ->> 'content_cid', + data ->> 'source_cid', + data ->> 'name', + (data ->> 'meta')::jsonb, + (data ->> 'inserted_at')::timestamptz, + (data ->> 'updated_at')::timestamptz + ) + + returning id into inserted_pin_request_id; + +-- TODO - Validate and use UUID type + return (inserted_pin_request_id)::TEXT; +END +$$; + CREATE OR REPLACE FUNCTION upsert_pin(data json) RETURNS TEXT LANGUAGE plpgsql volatile diff --git a/packages/db/postgres/pg-rest-api-types.d.ts b/packages/db/postgres/pg-rest-api-types.d.ts index 30711c65b9..cc59b145a0 100644 --- a/packages/db/postgres/pg-rest-api-types.d.ts +++ b/packages/db/postgres/pg-rest-api-types.d.ts @@ -3,7 +3,7 @@ * Do not make direct changes to the file. */ -export interface paths { + export interface paths { "/": { get: { responses: { @@ -906,6 +906,117 @@ export interface paths { }; }; }; + "/psa_pin_request": { + get: { + parameters: { + query: { + id?: parameters["rowFilter.psa_pin_request.id"]; + auth_key_id?: parameters["rowFilter.psa_pin_request.auth_key_id"]; + content_cid?: parameters["rowFilter.psa_pin_request.content_cid"]; + source_cid?: parameters["rowFilter.psa_pin_request.source_cid"]; + name?: parameters["rowFilter.psa_pin_request.name"]; + meta?: parameters["rowFilter.psa_pin_request.meta"]; + deleted_at?: parameters["rowFilter.psa_pin_request.deleted_at"]; + inserted_at?: parameters["rowFilter.psa_pin_request.inserted_at"]; + updated_at?: parameters["rowFilter.psa_pin_request.updated_at"]; + /** Filtering Columns */ + select?: parameters["select"]; + /** Ordering */ + order?: parameters["order"]; + /** Limiting and Pagination */ + offset?: parameters["offset"]; + /** Limiting and Pagination */ + limit?: parameters["limit"]; + }; + header: { + /** Limiting and Pagination */ + Range?: parameters["range"]; + /** Limiting and Pagination */ + "Range-Unit"?: parameters["rangeUnit"]; + /** Preference */ + Prefer?: parameters["preferCount"]; + }; + }; + responses: { + /** OK */ + 200: { + schema: definitions["psa_pin_request"][]; + }; + /** Partial Content */ + 206: unknown; + }; + }; + post: { + parameters: { + body: { + /** psa_pin_request */ + psa_pin_request?: definitions["psa_pin_request"]; + }; + query: { + /** Filtering Columns */ + select?: parameters["select"]; + }; + header: { + /** Preference */ + Prefer?: parameters["preferReturn"]; + }; + }; + responses: { + /** Created */ + 201: unknown; + }; + }; + delete: { + parameters: { + query: { + id?: parameters["rowFilter.psa_pin_request.id"]; + auth_key_id?: parameters["rowFilter.psa_pin_request.auth_key_id"]; + content_cid?: parameters["rowFilter.psa_pin_request.content_cid"]; + source_cid?: parameters["rowFilter.psa_pin_request.source_cid"]; + name?: parameters["rowFilter.psa_pin_request.name"]; + meta?: parameters["rowFilter.psa_pin_request.meta"]; + deleted_at?: parameters["rowFilter.psa_pin_request.deleted_at"]; + inserted_at?: parameters["rowFilter.psa_pin_request.inserted_at"]; + updated_at?: parameters["rowFilter.psa_pin_request.updated_at"]; + }; + header: { + /** Preference */ + Prefer?: parameters["preferReturn"]; + }; + }; + responses: { + /** No Content */ + 204: never; + }; + }; + patch: { + parameters: { + query: { + id?: parameters["rowFilter.psa_pin_request.id"]; + auth_key_id?: parameters["rowFilter.psa_pin_request.auth_key_id"]; + content_cid?: parameters["rowFilter.psa_pin_request.content_cid"]; + source_cid?: parameters["rowFilter.psa_pin_request.source_cid"]; + name?: parameters["rowFilter.psa_pin_request.name"]; + meta?: parameters["rowFilter.psa_pin_request.meta"]; + deleted_at?: parameters["rowFilter.psa_pin_request.deleted_at"]; + inserted_at?: parameters["rowFilter.psa_pin_request.inserted_at"]; + updated_at?: parameters["rowFilter.psa_pin_request.updated_at"]; + }; + body: { + /** psa_pin_request */ + psa_pin_request?: definitions["psa_pin_request"]; + }; + header: { + /** Preference */ + Prefer?: parameters["preferReturn"]; + }; + }; + responses: { + /** No Content */ + 204: never; + }; + }; + }; "/upload": { get: { parameters: { @@ -1165,6 +1276,29 @@ export interface paths { }; }; }; + + + // feat/pinning-api + "/rpc/create_content": { + post: { + parameters: { + body: { + args: { + data: string; + }; + }; + header: { + /** Preference */ + Prefer?: parameters["preferParams"]; + }; + }; + responses: { + /** OK */ + 200: unknown; + }; + }; + }; + "/rpc/user_auth_keys_list": { post: { parameters: { @@ -1184,6 +1318,7 @@ export interface paths { }; }; }; + "/rpc/pin_from_status_total": { post: { parameters: { @@ -1334,6 +1469,25 @@ export interface paths { }; }; }; + "/rpc/create_psa_pin_request": { + post: { + parameters: { + body: { + args: { + data: string; + }; + }; + header: { + /** Preference */ + Prefer?: parameters["preferParams"]; + }; + }; + responses: { + /** OK */ + 200: unknown; + }; + }; + }; "/rpc/user_used_storage": { post: { parameters: { @@ -1448,18 +1602,18 @@ export interface definitions { */ id: number; status: - | "Undefined" - | "ClusterError" - | "PinError" - | "UnpinError" - | "Pinned" - | "Pinning" - | "Unpinning" - | "Unpinned" - | "Remote" - | "PinQueued" - | "UnpinQueued" - | "Sharded"; + | "Undefined" + | "ClusterError" + | "PinError" + | "UnpinError" + | "Pinned" + | "Pinning" + | "Unpinning" + | "Unpinned" + | "Remote" + | "PinQueued" + | "UnpinQueued" + | "Sharded"; /** * Note: * This is a Foreign Key to `content.cid`. @@ -1511,6 +1665,29 @@ export interface definitions { pin_id: number; inserted_at: string; }; + psa_pin_request: { + /** + * Note: + * This is a Primary Key. + */ + id: string; // Create custom UUID v4 type + /** + * Note: + * This is a Foreign Key to `auth_key.id`. + */ + auth_key_id: number; + /** + * Note: + * This is a Foreign Key to `content.cid`. + */ + content_cid: string; + source_cid: string; + name?: string; + meta?: Record + deleted_at?: string; + inserted_at: string; + updated_at: string; + }; upload: { /** * Note: @@ -1641,6 +1818,17 @@ export interface parameters { "rowFilter.pin_sync_request.id": string; "rowFilter.pin_sync_request.pin_id": string; "rowFilter.pin_sync_request.inserted_at": string; + /** psa_pin_request */ + "body.psa_pin_request": definitions["psa_pin_request"]; + "rowFilter.psa_pin_request.id": string; + "rowFilter.psa_pin_request.auth_key_id": string; + "rowFilter.psa_pin_request.content_cid": string; + "rowFilter.psa_pin_request.source_cid": string; + "rowFilter.psa_pin_request.name": string; + "rowFilter.psa_pin_request.meta": Record; + "rowFilter.psa_pin_request.deleted_at": string; + "rowFilter.psa_pin_request.inserted_at": string; + "rowFilter.psa_pin_request.updated_at": string; /** upload */ "body.upload": definitions["upload"]; "rowFilter.upload.id": string; @@ -1666,6 +1854,6 @@ export interface parameters { "rowFilter.user.updated_at": string; } -export interface operations {} +export interface operations { } -export interface external {} +export interface external { } \ No newline at end of file diff --git a/packages/db/postgres/reset.sql b/packages/db/postgres/reset.sql index 498916e928..143973d297 100644 --- a/packages/db/postgres/reset.sql +++ b/packages/db/postgres/reset.sql @@ -5,6 +5,7 @@ DROP TABLE IF EXISTS pin CASCADE; DROP TABLE IF EXISTS pin_location; DROP TABLE IF EXISTS pin_request; DROP TABLE IF EXISTS pin_sync_request; +DROP TABLE IF EXISTS psa_pin_request; DROP TABLE IF EXISTS content; DROP TABLE IF EXISTS backup; DROP TABLE IF EXISTS auth_key; diff --git a/packages/db/postgres/tables.sql b/packages/db/postgres/tables.sql index bbe3520be0..dd3b0ce6c5 100644 --- a/packages/db/postgres/tables.sql +++ b/packages/db/postgres/tables.sql @@ -200,6 +200,29 @@ CREATE TABLE IF NOT EXISTS pin_sync_request CREATE INDEX IF NOT EXISTS pin_sync_request_pin_id_idx ON pin_sync_request (pin_id); + +-- Setting search_path to public scope for uuid function(s) +SET search_path TO public; +DROP extension IF EXISTS "uuid-ossp"; +CREATE extension "uuid-ossp" SCHEMA public; + +-- Tracks pinning requests from Pinning Service API +CREATE TABLE IF NOT EXISTS psa_pin_request +( + -- TODO - Vlaidate UUID type is available + id TEXT DEFAULT public.uuid_generate_v4() PRIMARY KEY, + -- Points to auth key used to pin the content. + auth_key_id BIGINT NOT NULL REFERENCES public.auth_key (id), + content_cid TEXT NOT NULL REFERENCES content (cid), + -- The id of the content being requested, it could not exist on IPFS (typo, node offline etc) + source_cid TEXT NOT NULL, + name TEXT, + meta jsonb, + deleted_at TIMESTAMP WITH TIME ZONE, + inserted_at TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL +); + -- A migration tracker. CREATE TABLE IF NOT EXISTS migration_tracker ( diff --git a/packages/db/test/pinning.spec.js b/packages/db/test/pinning.spec.js new file mode 100644 index 0000000000..c35f5fd333 --- /dev/null +++ b/packages/db/test/pinning.spec.js @@ -0,0 +1,413 @@ +/* eslint-env mocha, browser */ +import assert from 'assert' +import { normalizeCid } from '../../api/src/utils/normalize-cid' +import { DBClient } from '../index' +import { createUpload, createUser, createUserAuthKey, token } from './utils.js' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import * as pb from '@ipld/dag-pb' +/* global crypto */ + +const pinRequestTable = 'psa_pin_request' + +/** + * @param {number} code + * @returns {Promise} + */ +async function randomCid (code = pb.code) { + const bytes = crypto.getRandomValues(new Uint8Array(10)) + const hash = await sha256.digest(bytes) + return CID.create(1, code, hash).toString() +} + +/** + * + * @param {*} pinRequestOutput + * @param {object} opt + * @param {boolean} [opt.withContent] + */ +const assertCorrectPinRequestOutputTypes = (pinRequestOutput) => { + assert.ok(typeof pinRequestOutput._id === 'string', '_id should be a string') + assert.ok(typeof pinRequestOutput.sourceCid === 'string', 'sourceCid should be a string') + assert.ok(Array.isArray(pinRequestOutput.pins), 'pin should be an array') + assert.ok(Date.parse(pinRequestOutput.created), 'created should be valid date string') + assert.ok(Date.parse(pinRequestOutput.updated), 'updated should be valid date string') + assert.ok(typeof pinRequestOutput.contentCid === 'string', 'contentCid should be a string') +} + +describe('Pin Request', () => { + /** @type {DBClient & {_client: import('@supabase/postgrest-js').PostgrestClient }} */ + const client = (new DBClient({ + endpoint: 'http://127.0.0.1:3000', + token + })) + let user + let authKey + /** + * @type {import('../db-client-types').PsaPinRequestUpsertInput} + */ + let aPinRequestInput + + /** + * @type {import('../db-client-types').PsaPinRequestUpsertOutput} + */ + let aPinRequestOutput + + const cids = [ + 'QmdA5WkDNALetBn4iFeSepHjdLGJdxPBwZyY47ir1bZGAK', + 'QmNvTjdqEPjZVWCvRWsFJA1vK7TTw1g9JP6we1WBJTRADM' + ] + + const meta = { key: 'value' } + + const normalizedCids = cids.map(cid => normalizeCid(cid)) + + const pins = [ + { + status: 'Pinning', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', + peerName: 'web3-storage-sv15', + region: 'region' + } + }, + { + status: 'Pinning', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK7', + peerName: 'web3-storage-sv16', + region: 'region' + } + } + ] + + // Create user and auth key + before(async () => { + user = await createUser(client) + authKey = await createUserAuthKey(client, parseInt(user._id, 10)) + }) + + // Guarantee no Pin requests exist and create the ones needed for our tests + before(async () => { + // Make sure we don't have pinRequest and content + await client._client.from(pinRequestTable).delete() + const { count: countR } = await client._client.from(pinRequestTable).select('id', { + count: 'exact' + }) + assert.strictEqual(countR, 0, 'There are still requests in the db') + + aPinRequestInput = { + sourceCid: cids[0], + contentCid: normalizedCids[0], + meta, + pins, + authKey + } + + aPinRequestOutput = await client.createPsaPinRequest(aPinRequestInput) + }) + + describe('Create Pin', () => { + it('creates a Pin Request', async () => { + const savedPinRequest = await client.getPsaPinRequest(authKey, aPinRequestOutput._id) + assert.ok(savedPinRequest) + assert.strictEqual(savedPinRequest._id, aPinRequestOutput._id) + }) + + it('returns the right object', async () => { + assertCorrectPinRequestOutputTypes(aPinRequestOutput) + assert.deepStrictEqual(aPinRequestOutput.meta, meta, 'metadata is not the one provided') + assert.strictEqual(aPinRequestOutput.sourceCid, cids[0], 'sourceCid is not the one provided') + assert.strictEqual(aPinRequestOutput.authKey, authKey, 'auth key is not the one provided') + assert.strictEqual(aPinRequestOutput.contentCid, normalizedCids[0], 'contentCid is not the one provided') + }) + + it('creates content and pins', async () => { + const { count: countContent } = await client._client + .from('content') + .select('cid', { + count: 'exact' + }) + .match({ + cid: normalizedCids[0] + }) + assert.strictEqual(countContent, 1) + + const { count: countPins } = await client._client + .from('pin') + .select('id', { + count: 'exact' + }) + .match({ + content_cid: normalizedCids[0] + }) + assert.strictEqual(countPins, pins.length) + }) + + it('returns the right pins', async () => { + // Only checking statuses for simplicity + const statuses = aPinRequestOutput.pins + .map((p) => p.status) + assert.deepStrictEqual(statuses, [pins[0].status, pins[1].status]) + }) + }) + + describe('Get Pin', () => { + let savedPinRequest + + before(async () => { + savedPinRequest = await client.getPsaPinRequest(authKey, aPinRequestOutput._id) + }) + + it('gets a Pin Request, if it exists', async () => { + assert.ok(savedPinRequest) + }) + + it('returns the right object', async () => { + assertCorrectPinRequestOutputTypes(savedPinRequest) + assert.strictEqual(savedPinRequest.sourceCid, cids[0], 'sourceCid is not the one provided') + assert.strictEqual(savedPinRequest.authKey, authKey, 'auth key is not the one provided') + assert.strictEqual(savedPinRequest.contentCid, normalizedCids[0], 'contentCid is not the one provided') + }) + + it('returns the right pins', async () => { + // Only checking statuses for simplicity + const statuses = savedPinRequest.pins + .map((p) => p.status) + assert.deepStrictEqual(statuses, [pins[0].status, pins[1].status]) + }) + + it('throws if does not exists', async () => { + assert.rejects(client.getPsaPinRequest(authKey, 1000)) + }) + }) + + describe('Get Pins', () => { + const pins = [ + { + status: 'Pinning', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', + peerName: 'web3-storage-sv15', + region: 'region' + } + }, + { + status: 'Pinning', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK7', + peerName: 'web3-storage-sv16', + region: 'region' + } + } + ] + let pinRequestsInputs + + let userPinList + let authKeyPinList + let createdPinningRequests + let cidWithContent + let normalizeCidWithContent + + before(async () => { + userPinList = await createUser(client) + authKeyPinList = await createUserAuthKey(client, userPinList._id) + }) + + before(async () => { + cidWithContent = await randomCid() + normalizeCidWithContent = normalizeCid(cidWithContent) + await createUpload(client, userPinList._id, authKeyPinList, normalizeCidWithContent, { pins: pins }) + pinRequestsInputs = [ + { + name: 'horse', + date: [2020, 0, 1], + sourceCid: cidWithContent, + cid: normalizeCidWithContent + }, { + name: 'capybara', + date: [2020, 1, 1] + }, { + name: 'Camel', + date: [2020, 2, 1] + }, { + name: 'Giant Panda Bear', + date: [2020, 3, 1] + }, { + name: 'giant Schnoodle', + date: [2020, 4, 1] + }, { + name: 'giant worm', + date: [2020, 5, 1] + }, { + name: 'Zonkey Schnoodle', + date: [2020, 6, 1] + }, { + name: 'Zorse', + date: [2020, 7, 1] + }, { + date: [2020, 8, 1] + }, { + name: '', + date: [2020, 9, 1] + }, { + name: 'Bear', + date: [2020, 10, 1] + } + ] + createdPinningRequests = await Promise.all(pinRequestsInputs.map(async (item) => { + const sourceCid = item.sourceCid || await randomCid() + const normalizedCid = item.cid || normalizeCid(sourceCid) + + return client.createPsaPinRequest({ + ...(item.name) && { name: item.name }, + authKey: authKeyPinList, + sourceCid: sourceCid, + contentCid: normalizedCid, + pins + }) + })) + }) + + it('limits the results to 10', async () => { + const { results: prs } = await client.listPsaPinRequests(authKeyPinList) + assert.strictEqual(prs.length, 10) + }) + + it('limits the results to the provided limit', async () => { + const limit = 8 + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + limit + }) + assert.strictEqual(prs.length, limit) + }) + + it('returns only requests for the provided token', async () => { + const { results: prs } = await client.listPsaPinRequests('10') + assert.strictEqual(prs.length, 0) + }) + + it('sorts by date', async () => { + const { results: prs } = await client.listPsaPinRequests(authKeyPinList) + + const sorted = prs.reduce((n, item) => n !== null && item.created <= n.created && item) + assert(sorted) + }) + + it.skip('it filters items by provided status', async () => { + // TODO(https://github.com/web3-storage/web3.storage/issues/797): status filtering is currently not working + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + status: ['Pinning'] + }) + + assert.strictEqual(prs.length, 1) + assert.strictEqual(createdPinningRequests._id, prs[0]._id) + }) + + it('filters items by provided cid', async () => { + const cids = [createdPinningRequests[0].sourceCid, createdPinningRequests[1].sourceCid] + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + cid: cids + }) + + assert.strictEqual(prs.length, 2) + assert(prs.map(p => p.sourceCid).includes(cids[0])) + assert(prs.map(p => p.sourceCid).includes(cids[1])) + }) + + it('filters items by exact match by default', async () => { + const name = 'capybara' + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + name + }) + + assert.strictEqual(prs.length, 1) + prs.forEach(pr => { + assert.strictEqual(pr.name, name) + }) + }) + + it('filters items by iexact match', async () => { + const name = 'camel' + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + name, + match: 'iexact' + }) + + assert.strictEqual(prs.length, 1) + prs.forEach(pr => { + assert.strictEqual(pr.name.toLowerCase(), name.toLowerCase()) + }) + }) + + it('filters items by partial match', async () => { + const name = 'giant' + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + name, + match: 'partial' + }) + + assert.strictEqual(prs.length, 2) + prs.forEach(pr => { + assert(pr.name.includes(name)) + }) + }) + + it('filters items by ipartial match', async () => { + const name = 'giant' + const { results: prs } = await client.listPsaPinRequests(authKeyPinList, { + name, + match: 'ipartial' + }) + + assert.strictEqual(prs.length, 3) + prs.forEach(pr => { + assert(pr.name.toLowerCase().includes(name.toLowerCase())) + }) + }) + + it('filters items created before a date', async () => { + const { results: pins } = await client.listPsaPinRequests(authKeyPinList, { + before: '2021-01-01T00:00:00.000000Z' + }) + + assert.strictEqual(pins.length, 0) + }) + + it('filters items created after a date', async () => { + const { results: pins } = await client.listPsaPinRequests(authKeyPinList, { + after: '2021-01-01T00:00:00.000000Z', + limit: 20 + }) + + assert.strictEqual(pins.length, 11) + }) + }) + + describe('Delete Pin', () => { + it('throws if the request id does not exist', async () => { + assert.rejects(client.deletePsaPinRequest(1000, authKey)) + }) + + it('throws if the auth key does not belong to the pin request', async () => { + assert.rejects(client.deletePsaPinRequest(aPinRequestOutput._id, 'fakeAuth')) + }) + + it('returns the id of the deleted pin request', async () => { + const aPinRequestOutputId = aPinRequestOutput._id + const pinRequest = await client.getPsaPinRequest(authKey, aPinRequestOutputId) + assert.ok(!pinRequest.deleted, 'is null') + const deletedPinRequest = await client.deletePsaPinRequest(aPinRequestOutputId, authKey) + assert.ok(deletedPinRequest) + assert.equal(deletedPinRequest._id, pinRequest._id) + }) + + it('does not select pin request after deletion', async () => { + assert.rejects(client.getPsaPinRequest(authKey, aPinRequestOutput._id)) + }) + + it('cannot delete a pin request which is already deleted', async () => { + assert.rejects(client.deletePsaPinRequest(aPinRequestOutput._id, authKey)) + }) + }) +}) diff --git a/packages/db/test/utils.js b/packages/db/test/utils.js index 83a19fd17d..7a644f1c29 100644 --- a/packages/db/test/utils.js +++ b/packages/db/test/utils.js @@ -7,6 +7,7 @@ export const token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZ * @param {string} [options.email] * @param {string} [options.issuer] * @param {string} [options.publicAddress] + * @return {Promise.} */ export async function createUser (dbClient, options = {}) { const issuer = options.issuer || `issuer${Math.random()}` diff --git a/packages/db/utils.js b/packages/db/utils.js index 210ad9a75e..a6c87a6bdb 100644 --- a/packages/db/utils.js +++ b/packages/db/utils.js @@ -1,8 +1,8 @@ /** * Normalize upload item. * - * @param {import('../db-client-types').UploadItem} upload - * @return {import('../db-client-types').UploadItemOutput} + * @param {import('./db-client-types').UploadItem} upload + * @return {import('./db-client-types').UploadItemOutput} */ export function normalizeUpload (upload) { const nUpload = { ...upload } @@ -15,11 +15,27 @@ export function normalizeUpload (upload) { } } +/** + * Normalize pin request + * + * @param {object} psaPinRequest + * @return {import('./db-client-types').PsaPinRequestUpsertOutput} + */ +export function normalizePsaPinRequest (psaPinRequest) { + const nPsaPinRequest = { ...psaPinRequest } + delete nPsaPinRequest.content + + return { + ...nPsaPinRequest, + pins: psaPinRequest.content?.pins ? normalizePins(psaPinRequest.content.pins) : [] + } +} + /** * Normalize content item. * - * @param {import('../db-client-types').ContentItem} content - * @return {import('../db-client-types').ContentItemOutput} + * @param {import('./db-client-types').ContentItem} content + * @return {import('./db-client-types').ContentItemOutput} */ export function normalizeContent (content) { return { @@ -31,8 +47,8 @@ export function normalizeContent (content) { /** * Normalize pin items. * - * @param {Array} pins - * @return {Array} + * @param {Array} pins + * @return {Array} */ export function normalizePins (pins) { return pins.filter(pin => PIN_STATUS.has(pin.status))