From a86d7e2a7045d57793be28d7f78011581118daeb Mon Sep 17 00:00:00 2001 From: Paolo Chillari Date: Wed, 15 Jun 2022 14:36:39 +0100 Subject: [PATCH] fix: storage used should not count failed uploads (#1430) Co-authored-by: Adam Alton --- packages/db/postgres/functions.sql | 19 ++- ...16-user-used-storage-count-pinned-only.sql | 61 ++++++++++ packages/db/test/pin-sync-request.spec.js | 6 +- packages/db/test/user.spec.js | 42 ++++--- packages/db/test/utils.js | 109 +++++++++++++----- 5 files changed, 178 insertions(+), 59 deletions(-) create mode 100644 packages/db/postgres/migrations/016-user-used-storage-count-pinned-only.sql diff --git a/packages/db/postgres/functions.sql b/packages/db/postgres/functions.sql index 4abdb6fa37..9c7d073192 100644 --- a/packages/db/postgres/functions.sql +++ b/packages/db/postgres/functions.sql @@ -256,11 +256,20 @@ DECLARE BEGIN uploaded := ( - SELECT COALESCE(SUM(c.dag_size), 0) - FROM upload u - JOIN content c ON c.cid = u.content_cid - WHERE u.user_id = query_user_id::BIGINT - AND u.deleted_at is null + SELECT COALESCE(( + SELECT SUM(dag_size) + FROM ( + SELECT c.cid, + c.dag_size + FROM upload u + JOIN content c ON c.cid = u.content_cid + JOIN pin p ON p.content_cid = u.content_cid + WHERE u.user_id = query_user_id::BIGINT + AND u.deleted_at is null + AND p.status = 'Pinned' + GROUP BY c.cid, + c.dag_size + ) AS uploaded_content), 0) ); psa_pinned := diff --git a/packages/db/postgres/migrations/016-user-used-storage-count-pinned-only.sql b/packages/db/postgres/migrations/016-user-used-storage-count-pinned-only.sql new file mode 100644 index 0000000000..ae5fe70b70 --- /dev/null +++ b/packages/db/postgres/migrations/016-user-used-storage-count-pinned-only.sql @@ -0,0 +1,61 @@ +-- Due to the parameters differing from the old function definition, if we don't drop the old one then it's not replaced. +DROP FUNCTION IF EXISTS user_used_storage; + +CREATE OR REPLACE FUNCTION user_used_storage(query_user_id BIGINT) + RETURNS stored_bytes + LANGUAGE plpgsql +AS +$$ +DECLARE + used_storage stored_bytes; + uploaded BIGINT; + psa_pinned BIGINT; + total BIGINT; +BEGIN + uploaded := + ( + SELECT COALESCE(( + SELECT SUM(dag_size) + FROM ( + SELECT c.cid, + c.dag_size + FROM upload u + JOIN content c ON c.cid = u.content_cid + JOIN pin p ON p.content_cid = u.content_cid + WHERE u.user_id = query_user_id::BIGINT + AND u.deleted_at is null + AND p.status = 'Pinned' + GROUP BY c.cid, + c.dag_size + ) AS uploaded_content), 0) + ); + + psa_pinned := + ( + SELECT COALESCE(( + SELECT SUM(dag_size) + FROM ( + SELECT psa_pr.content_cid, + c.dag_size + FROM psa_pin_request psa_pr + JOIN content c ON c.cid = psa_pr.content_cid + JOIN pin p ON p.content_cid = psa_pr.content_cid + JOIN auth_key a ON a.id = psa_pr.auth_key_id + WHERE a.user_id = query_user_id::BIGINT + AND psa_pr.deleted_at is null + AND p.status = 'Pinned' + GROUP BY psa_pr.content_cid, + c.dag_size + ) AS pinned_content), 0) + ); + + total := uploaded + psa_pinned; + + SELECT uploaded::TEXT, + psa_pinned::TEXT, + total::TEXT + INTO used_storage; + + return used_storage; +END +$$; diff --git a/packages/db/test/pin-sync-request.spec.js b/packages/db/test/pin-sync-request.spec.js index 13c4361005..8588f2f5a3 100644 --- a/packages/db/test/pin-sync-request.spec.js +++ b/packages/db/test/pin-sync-request.spec.js @@ -2,7 +2,7 @@ import assert from 'assert' import { DBClient } from '../index.js' -import { createUser, createUserAuthKey, createUpload, defaultPinData, token } from './utils.js' +import { createUser, createUserAuthKey, createUpload, initialPinsNotPinned, token } from './utils.js' describe('pin-sync-request', () => { /** @type {DBClient} */ @@ -51,7 +51,7 @@ describe('pin-sync-request', () => { assert(pinSyncReqs, 'pin sync requests exist') // expect pin sync requests = added pins for each upload where status is not pinned - const expectedPinSyncReqs = defaultPinData.filter(pd => pd.status !== 'Pinned').length * 2 + const expectedPinSyncReqs = initialPinsNotPinned.filter(pd => pd.status !== 'Pinned').length * 2 assert.strictEqual(pinSyncReqs.length, expectedPinSyncReqs, 'created pin sync requests for non pinned entries') }) @@ -96,7 +96,7 @@ describe('pin-sync-request', () => { assert(pinSyncReqs, 'pin sync requests exist') // From the 2 setup uploads expected pins - const previousExpectedPinSyncReqs = defaultPinData.filter(pd => pd.status !== 'Pinned').length * 2 + const previousExpectedPinSyncReqs = initialPinsNotPinned.filter(pd => pd.status !== 'Pinned').length * 2 // Pins for the new upload const newPinSyncReqs = pins.filter(pd => pd.status !== 'Pinned').length assert.strictEqual(pinSyncReqs.length, newPinSyncReqs + previousExpectedPinSyncReqs, 'created pin sync requests for non pinned entries') diff --git a/packages/db/test/user.spec.js b/packages/db/test/user.spec.js index 3bdcfb1eef..f2e078826c 100644 --- a/packages/db/test/user.spec.js +++ b/packages/db/test/user.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha, browser */ import assert from 'assert' import { DBClient } from '../index.js' -import { token } from './utils.js' +import { createUpload, initialPinsNotPinned, pinsError, randomCid, token } from './utils.js' describe('user operations', () => { const name = 'test-name' @@ -166,15 +166,8 @@ describe('user operations', () => { // Create Upload 1 const cid1 = 'bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47fgf111' const dagSize1 = 10000 - await client.createUpload({ - user: user._id, - contentCid: cid1, - sourceCid: cid1, - authKey: authKey._id, - type: 'Upload', - dagSize: dagSize1, - pins: [], - backupUrls: [] + await createUpload(client, user._id, authKey._id, cid1, { + dagSize: dagSize1 }) const firstUsedStorage = await client.getStorageUsed(user._id) @@ -183,17 +176,28 @@ describe('user operations', () => { // Create Upload 2 const cid2 = 'bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47fgf112' const dagSize2 = 30000 - await client.createUpload({ - user: user._id, - contentCid: cid2, - sourceCid: cid2, - authKey: authKey._id, - type: 'Upload', - dagSize: dagSize2, - pins: [], - backupUrls: [] + + await createUpload(client, user._id, authKey._id, cid2, { + dagSize: dagSize2 + }) + + // Create "Failed" Upload. It should not be counted. + const cid3 = await randomCid() + const dagSize3 = 100000 + await createUpload(client, user._id, authKey._id, cid3, { + dagSize: dagSize3, + pins: pinsError }) + // Create Upload not pinned yet. It should not be counted yet. + await createUpload(client, user._id, authKey._id, await randomCid(), { + dagSize: 1000000, + pins: initialPinsNotPinned + }) + + const usedStorageWithFailed = await client.getStorageUsed(user._id) + assert.strictEqual(usedStorageWithFailed.uploaded, dagSize1 + dagSize2, 'used storage should not count unpinned') + const secondUsedStorage = await client.getStorageUsed(user._id) assert.strictEqual(secondUsedStorage.uploaded, dagSize1 + dagSize2, 'used storage with second upload') diff --git a/packages/db/test/utils.js b/packages/db/test/utils.js index 60c97ac0ef..c18edf2a7d 100644 --- a/packages/db/test/utils.js +++ b/packages/db/test/utils.js @@ -7,6 +7,54 @@ import { normalizeCid } from '../../api/src/utils/cid.js' export const token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoicG9zdGdyZXMifQ.oM0SXF31Vs1nfwCaDxjlczE237KcNKhTpKEYxMX-jEU' export const dbEndpoint = 'http://127.0.0.1:3000' +export const initialPinsNotPinned = [{ + status: 'Pinning', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', + peerName: 'web3-storage-sv15', + ipfsPeerId: '12D3KooWR19qPPiZH4khepNjS3CLXiB7AbrbAD4ZcDjN1UjGUNE1', + region: 'region' + } +}] + +export const pinsPinned = [ + { + status: 'Pinning', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', + peerName: 'web3-storage-sv15', + region: 'region' + } + }, + { + status: 'Pinned', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK7', + peerName: 'web3-storage-sv16', + region: 'region' + } + } +] + +export const pinsError = [ + { + status: 'PinError', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', + peerName: 'web3-storage-sv15', + region: 'region' + } + }, + { + status: 'PinError', + location: { + peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK7', + peerName: 'web3-storage-sv16', + region: 'region' + } + } +] + /** * @param {number} code * @returns {Promise} @@ -37,27 +85,10 @@ export async function createUser (dbClient, options = {}) { return dbClient.getUser(issuer) } -const defaultUserPinnedRequests = [ - { - status: 'Pinning', - location: { - peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', - peerName: 'web3-storage-sv15', - region: 'region' - } - }, - { - status: 'Pinned', - location: { - peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK7', - peerName: 'web3-storage-sv16', - region: 'region' - } - } -] - /** - * Create a user and files with a specified storage quota used + * Create a user and files with a specified storage quota used. + * + * It creates some failed and yet to be pinned content. * @param {import('../index').DBClient} dbClient * @param {Object} [options] * @param {string} [options.email] @@ -95,6 +126,18 @@ export async function createUserWithFiles (dbClient, options = {}) { const pinRequests = 3 const dagSize = Math.ceil(((percentStorageUsed / 100) * storageQuota) / (uploads + pinRequests)) + // Create a failed upload. + await createUpload(dbClient, Number(user._id), Number(authKey), await randomCid(), { + dagSize, + pins: pinsError + }) + + // Create a yet to be pinned upload. + await createUpload(dbClient, Number(user._id), Number(authKey), await randomCid(), { + dagSize, + pins: initialPinsNotPinned + }) + for (let i = 0; i < uploads; i++) { const cid = await randomCid() await createUpload(dbClient, Number(user._id), Number(authKey), cid, { @@ -102,11 +145,23 @@ export async function createUserWithFiles (dbClient, options = {}) { }) } + // Create a failed PinRequest. + await createPsaPinRequest(dbClient, authKey, await randomCid(), { + dagSize, + pins: pinsError + }) + + // Create a yet to be pinned PinRequest. + await createPsaPinRequest(dbClient, authKey, await randomCid(), { + dagSize, + pins: initialPinsNotPinned + }) + for (let i = 0; i < pinRequests; i++) { const cid = await randomCid() await createPsaPinRequest(dbClient, authKey, cid, { dagSize, - pins: pins || defaultUserPinnedRequests + pins: pins || pinsPinned }) } @@ -130,16 +185,6 @@ export async function createUserAuthKey (dbClient, user, options = {}) { return _id } -export const defaultPinData = [{ - status: 'Pinning', - location: { - peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK6', - peerName: 'web3-storage-sv15', - ipfsPeerId: '12D3KooWR19qPPiZH4khepNjS3CLXiB7AbrbAD4ZcDjN1UjGUNE1', - region: 'region' - } -}] - /** * @param {import('../index').DBClient} dbClient * @param {number} user @@ -163,7 +208,7 @@ export async function createUpload (dbClient, user, authKey, cid, options = {}) type: options.type || 'Upload', dagSize: options.dagSize === undefined ? 1000 : options.dagSize, name: options.name || `Upload_${new Date().toISOString()}`, - pins: options.pins || defaultPinData, + pins: options.pins || pinsPinned, backupUrls: options.backupUrls || [initialBackupUrl] })