diff --git a/packages/db/index.js b/packages/db/index.js index df5d7d042f..4454576842 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -22,6 +22,7 @@ const uploadQuery = ` created:inserted_at, updated:updated_at, backupUrls:backup_urls, + backup( url ), content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, ipfsPeerId:ipfs_peer_id, region))) ` diff --git a/packages/db/postgres/pg-rest-api-types.d.ts b/packages/db/postgres/pg-rest-api-types.d.ts index 162fb2e734..3d24e8b844 100644 --- a/packages/db/postgres/pg-rest-api-types.d.ts +++ b/packages/db/postgres/pg-rest-api-types.d.ts @@ -356,6 +356,102 @@ export interface paths { }; }; }; + "/backup": { + get: { + parameters: { + query: { + id?: parameters["rowFilter.backup.id"]; + upload_id?: parameters["rowFilter.backup.upload_id"]; + url?: parameters["rowFilter.backup.url"]; + inserted_at?: parameters["rowFilter.backup.inserted_at"]; + /** Filtering Columns */ + select?: parameters["select"]; + /** Ordering */ + order?: parameters["order"]; + /** Limiting and Pagination */ + offset?: parameters["offset"]; + /** Limiting and Pagination */ + limit?: parameters["limit"]; + }; + header: { + /** Limiting and Pagination */ + Range?: parameters["range"]; + /** Limiting and Pagination */ + "Range-Unit"?: parameters["rangeUnit"]; + /** Preference */ + Prefer?: parameters["preferCount"]; + }; + }; + responses: { + /** OK */ + 200: { + schema: definitions["backup"][]; + }; + /** Partial Content */ + 206: unknown; + }; + }; + post: { + parameters: { + body: { + /** backup */ + backup?: definitions["backup"]; + }; + query: { + /** Filtering Columns */ + select?: parameters["select"]; + }; + header: { + /** Preference */ + Prefer?: parameters["preferReturn"]; + }; + }; + responses: { + /** Created */ + 201: unknown; + }; + }; + delete: { + parameters: { + query: { + id?: parameters["rowFilter.backup.id"]; + upload_id?: parameters["rowFilter.backup.upload_id"]; + url?: parameters["rowFilter.backup.url"]; + inserted_at?: parameters["rowFilter.backup.inserted_at"]; + }; + header: { + /** Preference */ + Prefer?: parameters["preferReturn"]; + }; + }; + responses: { + /** No Content */ + 204: never; + }; + }; + patch: { + parameters: { + query: { + id?: parameters["rowFilter.backup.id"]; + upload_id?: parameters["rowFilter.backup.upload_id"]; + url?: parameters["rowFilter.backup.url"]; + inserted_at?: parameters["rowFilter.backup.inserted_at"]; + }; + body: { + /** backup */ + backup?: definitions["backup"]; + }; + header: { + /** Preference */ + Prefer?: parameters["preferReturn"]; + }; + }; + responses: { + /** No Content */ + 204: never; + }; + }; + }; "/content": { get: { parameters: { @@ -1592,13 +1688,23 @@ export interface paths { }; }; }; - "/rpc/upsert_pins": { + "/rpc/upsert_user": { post: { parameters: { body: { args: { - /** Format: json */ - data: string; + /** Format: text */ + _github: string; + /** Format: text */ + _email: string; + /** Format: text */ + _name: string; + /** Format: text */ + _picture: string; + /** Format: text */ + _issuer: string; + /** Format: text */ + _public_address: string; }; }; header: { @@ -1612,11 +1718,16 @@ export interface paths { }; }; }; - "/rpc/uuid_ns_url": { + "/rpc/user_auth_keys_list": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: bigint */ + query_user_id: number; + /** Format: boolean */ + include_deleted?: boolean; + }; }; header: { /** Preference */ @@ -1629,11 +1740,14 @@ export interface paths { }; }; }; - "/rpc/pgrst_watch": { + "/rpc/create_psa_pin_request": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: json */ + data: string; + }; }; header: { /** Preference */ @@ -1646,11 +1760,14 @@ export interface paths { }; }; }; - "/rpc/uuid_nil": { + "/rpc/user_used_storage": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: bigint */ + query_user_id: number; + }; }; header: { /** Preference */ @@ -1663,11 +1780,14 @@ export interface paths { }; }; }; - "/rpc/uuid_generate_v1": { + "/rpc/json_arr_to_json_element_array": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: json */ + _json: string; + }; }; header: { /** Preference */ @@ -1680,7 +1800,7 @@ export interface paths { }; }; }; - "/rpc/create_content": { + "/rpc/create_key": { post: { parameters: { body: { @@ -1700,11 +1820,14 @@ export interface paths { }; }; }; - "/rpc/uuid_ns_oid": { + "/rpc/json_arr_to_text_arr": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: json */ + _json: string; + }; }; header: { /** Preference */ @@ -1737,20 +1860,11 @@ export interface paths { }; }; }; - "/rpc/users_by_storage_used": { + "/rpc/uuid_generate_v1mc": { post: { parameters: { body: { - args: { - /** Format: integer */ - to_percent?: number; - /** Format: bigint */ - user_id_gt?: number; - /** Format: integer */ - from_percent: number; - /** Format: bigint */ - user_id_lte?: number; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -1763,16 +1877,11 @@ export interface paths { }; }; }; - "/rpc/user_auth_keys_list": { + "/rpc/uuid_generate_v1": { post: { parameters: { body: { - args: { - /** Format: boolean */ - include_deleted?: boolean; - /** Format: bigint */ - query_user_id: number; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -1785,14 +1894,11 @@ export interface paths { }; }; }; - "/rpc/find_deals_by_content_cids": { + "/rpc/uuid_generate_v4": { post: { parameters: { body: { - args: { - /** Format: text[] */ - cids: string; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -1805,15 +1911,13 @@ export interface paths { }; }; }; - "/rpc/uuid_generate_v5": { + "/rpc/create_upload": { post: { parameters: { body: { args: { - /** Format: uuid */ - namespace: string; - /** Format: text */ - name: string; + /** Format: json */ + data: string; }; }; header: { @@ -1847,11 +1951,14 @@ export interface paths { }; }; }; - "/rpc/uuid_ns_x500": { + "/rpc/find_deals_by_content_cids": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: text[] */ + cids: string; + }; }; header: { /** Preference */ @@ -1864,13 +1971,19 @@ export interface paths { }; }; }; - "/rpc/json_arr_to_text_arr": { + "/rpc/users_by_storage_used": { post: { parameters: { body: { args: { - /** Format: json */ - _json: string; + /** Format: integer */ + to_percent?: number; + /** Format: bigint */ + user_id_gt?: number; + /** Format: integer */ + from_percent: number; + /** Format: bigint */ + user_id_lte?: number; }; }; header: { @@ -1884,16 +1997,11 @@ export interface paths { }; }; }; - "/rpc/uuid_generate_v3": { + "/rpc/uuid_ns_url": { post: { parameters: { body: { - args: { - /** Format: uuid */ - namespace: string; - /** Format: text */ - name: string; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -1906,23 +2014,15 @@ export interface paths { }; }; }; - "/rpc/upsert_user": { + "/rpc/uuid_generate_v5": { post: { parameters: { body: { args: { /** Format: text */ - _issuer: string; - /** Format: text */ - _github: string; - /** Format: text */ - _name: string; - /** Format: text */ - _email: string; - /** Format: text */ - _public_address: string; - /** Format: text */ - _picture: string; + name: string; + /** Format: uuid */ + namespace: string; }; }; header: { @@ -1936,7 +2036,7 @@ export interface paths { }; }; }; - "/rpc/create_key": { + "/rpc/create_content_with_pin_sync_request": { post: { parameters: { body: { @@ -1956,11 +2056,14 @@ export interface paths { }; }; }; - "/rpc/uuid_generate_v4": { + "/rpc/create_content": { post: { parameters: { body: { - args: { [key: string]: unknown }; + args: { + /** Format: json */ + data: string; + }; }; header: { /** Preference */ @@ -1973,7 +2076,7 @@ export interface paths { }; }; }; - "/rpc/uuid_ns_dns": { + "/rpc/uuid_ns_oid": { post: { parameters: { body: { @@ -1990,13 +2093,13 @@ export interface paths { }; }; }; - "/rpc/json_arr_to_json_element_array": { + "/rpc/user_psa_storage": { post: { parameters: { body: { args: { - /** Format: json */ - _json: string; + /** Format: bigint */ + query_user_id: number; }; }; header: { @@ -2010,14 +2113,11 @@ export interface paths { }; }; }; - "/rpc/user_psa_storage": { + "/rpc/pgrst_watch": { post: { parameters: { body: { - args: { - /** Format: bigint */ - query_user_id: number; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -2030,14 +2130,11 @@ export interface paths { }; }; }; - "/rpc/create_psa_pin_request": { + "/rpc/uuid_ns_x500": { post: { parameters: { body: { - args: { - /** Format: json */ - data: string; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -2050,14 +2147,11 @@ export interface paths { }; }; }; - "/rpc/user_used_storage": { + "/rpc/uuid_nil": { post: { parameters: { body: { - args: { - /** Format: bigint */ - query_user_id: number; - }; + args: { [key: string]: unknown }; }; header: { /** Preference */ @@ -2070,7 +2164,7 @@ export interface paths { }; }; }; - "/rpc/uuid_generate_v1mc": { + "/rpc/uuid_ns_dns": { post: { parameters: { body: { @@ -2087,7 +2181,7 @@ export interface paths { }; }; }; - "/rpc/create_upload": { + "/rpc/upsert_pins": { post: { parameters: { body: { @@ -2107,6 +2201,28 @@ export interface paths { }; }; }; + "/rpc/uuid_generate_v3": { + post: { + parameters: { + body: { + args: { + /** Format: text */ + name: string; + /** Format: uuid */ + namespace: string; + }; + }; + header: { + /** Preference */ + Prefer?: parameters["preferParams"]; + }; + }; + responses: { + /** OK */ + 200: unknown; + }; + }; + }; } export interface definitions { @@ -2206,6 +2322,27 @@ export interface definitions { /** Format: timestamp with time zone */ deleted_at?: string; }; + backup: { + /** + * Format: bigint + * @description Note: + * This is a Primary Key. + */ + id: number; + /** + * Format: bigint + * @description Note: + * This is a Foreign Key to `upload.id`. + */ + upload_id: number; + /** Format: text */ + url: string; + /** + * Format: timestamp with time zone + * @default timezone('utc'::text, now()) + */ + inserted_at: string; + }; content: { /** * Format: text @@ -2639,6 +2776,16 @@ export interface parameters { "rowFilter.auth_key_history.inserted_at": string; /** Format: timestamp with time zone */ "rowFilter.auth_key_history.deleted_at": string; + /** @description backup */ + "body.backup": definitions["backup"]; + /** Format: bigint */ + "rowFilter.backup.id": string; + /** Format: bigint */ + "rowFilter.backup.upload_id": string; + /** Format: text */ + "rowFilter.backup.url": string; + /** Format: timestamp with time zone */ + "rowFilter.backup.inserted_at": string; /** @description content */ "body.content": definitions["content"]; /** Format: text */ diff --git a/packages/db/postgres/tables.sql b/packages/db/postgres/tables.sql index c46edf4cbc..83af9f2d52 100644 --- a/packages/db/postgres/tables.sql +++ b/packages/db/postgres/tables.sql @@ -286,6 +286,20 @@ CREATE INDEX IF NOT EXISTS upload_inserted_at_idx ON upload (inserted_at); CREATE INDEX IF NOT EXISTS upload_updated_at_idx ON upload (updated_at); CREATE INDEX IF NOT EXISTS upload_source_cid_idx ON upload (source_cid); +-- Details of the backups created for an upload. +CREATE TABLE IF NOT EXISTS backup +( + id BIGSERIAL PRIMARY KEY, + -- Upload that resulted in this backup. + upload_id BIGINT NOT NULL REFERENCES upload (id) ON DELETE CASCADE, + -- Backup url location. + url TEXT NOT NULL, + inserted_at TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL, + UNIQUE (upload_id, url) +); + +CREATE INDEX IF NOT EXISTS backup_upload_id_idx ON backup (upload_id); + -- A request to keep a Pin in sync with the nodes that are pinning it. CREATE TABLE IF NOT EXISTS pin_sync_request ( diff --git a/packages/db/test/upload.spec.js b/packages/db/test/upload.spec.js index 25f16e8edf..490a4a8b01 100644 --- a/packages/db/test/upload.spec.js +++ b/packages/db/test/upload.spec.js @@ -278,6 +278,42 @@ describe('upload', () => { assert.ok(userUploads.find(upload => upload.cid === sourceCid)) }) + it('lists user uploads with CAR links in parts', async () => { + const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' + const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR' + // note: `exampleCarParkUrl` and `exampleS3Url` are for same CAR. The s3 url is base32(multihash(car)) and the other is cid v1 + const exampleCarParkUrl = 'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car' + const exampleS3Url = 'https://dotstorage-dev-0.s3.us-east-1.amazonaws.com/raw/bafybeiao32xtnrlibcekpw3vyfi5txgrmvvrua4pccx3xik33ll3qhko2q/2/ciqplrl7tuebgpzbo5nqlqus5hj2kowxzz7ayr4z6ao2ftg7ibcr3ca.car' + const created = new Date().toISOString() + const name = `rand-${Math.random().toString().slice(2)}` + await client.createUpload({ + user: user._id, + contentCid, + sourceCid, + authKey: authKeys[0]._id, + type, + dagSize, + name, + pins: [initialPinData], + backupUrls: [`https://backup.cid/${created}`, exampleCarParkUrl, exampleS3Url], + created + }) + + // Default sort {inserted_at, Desc} + const { uploads } = await client.listUploads(user._id, { page: 1 }) + assert.ok(uploads.length > 0) + for (const upload of uploads) { + // backupUrls raw is private + assert.ok(!('backupUrls' in upload), 'upload does not have backupUrls property') + assert.ok(Array.isArray(upload.parts), 'upload.parts is an array') + } + const namedUpload = uploads.find(u => u.name === name) + assert.deepEqual(namedUpload.parts, [ + // this corresponds to `exampleCarParkUrl` + 'bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea' + ]) + }) + it('lists user uploads with CAR links in parts', async () => { const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR' diff --git a/packages/db/utils.js b/packages/db/utils.js index b040f9d7a6..902bea1fd6 100644 --- a/packages/db/utils.js +++ b/packages/db/utils.js @@ -10,13 +10,21 @@ import { fromString } from 'uint8arrays' */ export function normalizeUpload (upload) { const nUpload = { ...upload } - const backupUrls = nUpload.backupUrls ?? [] - delete nUpload.backupUrls + delete nUpload.content delete nUpload.sourceCid + // get hash links to CARs that contain parts of this upload /** @type {import('./db-client-types').UploadItemOutput['parts']} */ - const parts = [...carCidV1Base32sFromBackupUrls(backupUrls)] + const parts = [ + // from upload table 'backup_urls' column + ...carCidV1Base32sFromBackupUrls(nUpload.backupUrls ?? []), + // there is also a backup table that maybe have been joined in + // each backup has a url column + ...carCidV1Base32sFromBackupUrls((nUpload.backup ?? []).map(o => o.url)) + ] + delete nUpload.backupUrls + delete nUpload.backup return { ...nUpload,