diff --git a/packages/db/index.js b/packages/db/index.js
index df5d7d042f..4454576842 100644
--- a/packages/db/index.js
+++ b/packages/db/index.js
@@ -22,6 +22,7 @@ const uploadQuery = `
created:inserted_at,
updated:updated_at,
backupUrls:backup_urls,
+ backup( url ),
content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, ipfsPeerId:ipfs_peer_id, region)))
`
diff --git a/packages/db/postgres/pg-rest-api-types.d.ts b/packages/db/postgres/pg-rest-api-types.d.ts
index 162fb2e734..3d24e8b844 100644
--- a/packages/db/postgres/pg-rest-api-types.d.ts
+++ b/packages/db/postgres/pg-rest-api-types.d.ts
@@ -356,6 +356,102 @@ export interface paths {
};
};
};
+ "/backup": {
+ get: {
+ parameters: {
+ query: {
+ id?: parameters["rowFilter.backup.id"];
+ upload_id?: parameters["rowFilter.backup.upload_id"];
+ url?: parameters["rowFilter.backup.url"];
+ inserted_at?: parameters["rowFilter.backup.inserted_at"];
+ /** Filtering Columns */
+ select?: parameters["select"];
+ /** Ordering */
+ order?: parameters["order"];
+ /** Limiting and Pagination */
+ offset?: parameters["offset"];
+ /** Limiting and Pagination */
+ limit?: parameters["limit"];
+ };
+ header: {
+ /** Limiting and Pagination */
+ Range?: parameters["range"];
+ /** Limiting and Pagination */
+ "Range-Unit"?: parameters["rangeUnit"];
+ /** Preference */
+ Prefer?: parameters["preferCount"];
+ };
+ };
+ responses: {
+ /** OK */
+ 200: {
+ schema: definitions["backup"][];
+ };
+ /** Partial Content */
+ 206: unknown;
+ };
+ };
+ post: {
+ parameters: {
+ body: {
+ /** backup */
+ backup?: definitions["backup"];
+ };
+ query: {
+ /** Filtering Columns */
+ select?: parameters["select"];
+ };
+ header: {
+ /** Preference */
+ Prefer?: parameters["preferReturn"];
+ };
+ };
+ responses: {
+ /** Created */
+ 201: unknown;
+ };
+ };
+ delete: {
+ parameters: {
+ query: {
+ id?: parameters["rowFilter.backup.id"];
+ upload_id?: parameters["rowFilter.backup.upload_id"];
+ url?: parameters["rowFilter.backup.url"];
+ inserted_at?: parameters["rowFilter.backup.inserted_at"];
+ };
+ header: {
+ /** Preference */
+ Prefer?: parameters["preferReturn"];
+ };
+ };
+ responses: {
+ /** No Content */
+ 204: never;
+ };
+ };
+ patch: {
+ parameters: {
+ query: {
+ id?: parameters["rowFilter.backup.id"];
+ upload_id?: parameters["rowFilter.backup.upload_id"];
+ url?: parameters["rowFilter.backup.url"];
+ inserted_at?: parameters["rowFilter.backup.inserted_at"];
+ };
+ body: {
+ /** backup */
+ backup?: definitions["backup"];
+ };
+ header: {
+ /** Preference */
+ Prefer?: parameters["preferReturn"];
+ };
+ };
+ responses: {
+ /** No Content */
+ 204: never;
+ };
+ };
+ };
"/content": {
get: {
parameters: {
@@ -1592,13 +1688,23 @@ export interface paths {
};
};
};
- "/rpc/upsert_pins": {
+ "/rpc/upsert_user": {
post: {
parameters: {
body: {
args: {
- /** Format: json */
- data: string;
+ /** Format: text */
+ _github: string;
+ /** Format: text */
+ _email: string;
+ /** Format: text */
+ _name: string;
+ /** Format: text */
+ _picture: string;
+ /** Format: text */
+ _issuer: string;
+ /** Format: text */
+ _public_address: string;
};
};
header: {
@@ -1612,11 +1718,16 @@ export interface paths {
};
};
};
- "/rpc/uuid_ns_url": {
+ "/rpc/user_auth_keys_list": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: bigint */
+ query_user_id: number;
+ /** Format: boolean */
+ include_deleted?: boolean;
+ };
};
header: {
/** Preference */
@@ -1629,11 +1740,14 @@ export interface paths {
};
};
};
- "/rpc/pgrst_watch": {
+ "/rpc/create_psa_pin_request": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: json */
+ data: string;
+ };
};
header: {
/** Preference */
@@ -1646,11 +1760,14 @@ export interface paths {
};
};
};
- "/rpc/uuid_nil": {
+ "/rpc/user_used_storage": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: bigint */
+ query_user_id: number;
+ };
};
header: {
/** Preference */
@@ -1663,11 +1780,14 @@ export interface paths {
};
};
};
- "/rpc/uuid_generate_v1": {
+ "/rpc/json_arr_to_json_element_array": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: json */
+ _json: string;
+ };
};
header: {
/** Preference */
@@ -1680,7 +1800,7 @@ export interface paths {
};
};
};
- "/rpc/create_content": {
+ "/rpc/create_key": {
post: {
parameters: {
body: {
@@ -1700,11 +1820,14 @@ export interface paths {
};
};
};
- "/rpc/uuid_ns_oid": {
+ "/rpc/json_arr_to_text_arr": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: json */
+ _json: string;
+ };
};
header: {
/** Preference */
@@ -1737,20 +1860,11 @@ export interface paths {
};
};
};
- "/rpc/users_by_storage_used": {
+ "/rpc/uuid_generate_v1mc": {
post: {
parameters: {
body: {
- args: {
- /** Format: integer */
- to_percent?: number;
- /** Format: bigint */
- user_id_gt?: number;
- /** Format: integer */
- from_percent: number;
- /** Format: bigint */
- user_id_lte?: number;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -1763,16 +1877,11 @@ export interface paths {
};
};
};
- "/rpc/user_auth_keys_list": {
+ "/rpc/uuid_generate_v1": {
post: {
parameters: {
body: {
- args: {
- /** Format: boolean */
- include_deleted?: boolean;
- /** Format: bigint */
- query_user_id: number;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -1785,14 +1894,11 @@ export interface paths {
};
};
};
- "/rpc/find_deals_by_content_cids": {
+ "/rpc/uuid_generate_v4": {
post: {
parameters: {
body: {
- args: {
- /** Format: text[] */
- cids: string;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -1805,15 +1911,13 @@ export interface paths {
};
};
};
- "/rpc/uuid_generate_v5": {
+ "/rpc/create_upload": {
post: {
parameters: {
body: {
args: {
- /** Format: uuid */
- namespace: string;
- /** Format: text */
- name: string;
+ /** Format: json */
+ data: string;
};
};
header: {
@@ -1847,11 +1951,14 @@ export interface paths {
};
};
};
- "/rpc/uuid_ns_x500": {
+ "/rpc/find_deals_by_content_cids": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: text[] */
+ cids: string;
+ };
};
header: {
/** Preference */
@@ -1864,13 +1971,19 @@ export interface paths {
};
};
};
- "/rpc/json_arr_to_text_arr": {
+ "/rpc/users_by_storage_used": {
post: {
parameters: {
body: {
args: {
- /** Format: json */
- _json: string;
+ /** Format: integer */
+ to_percent?: number;
+ /** Format: bigint */
+ user_id_gt?: number;
+ /** Format: integer */
+ from_percent: number;
+ /** Format: bigint */
+ user_id_lte?: number;
};
};
header: {
@@ -1884,16 +1997,11 @@ export interface paths {
};
};
};
- "/rpc/uuid_generate_v3": {
+ "/rpc/uuid_ns_url": {
post: {
parameters: {
body: {
- args: {
- /** Format: uuid */
- namespace: string;
- /** Format: text */
- name: string;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -1906,23 +2014,15 @@ export interface paths {
};
};
};
- "/rpc/upsert_user": {
+ "/rpc/uuid_generate_v5": {
post: {
parameters: {
body: {
args: {
/** Format: text */
- _issuer: string;
- /** Format: text */
- _github: string;
- /** Format: text */
- _name: string;
- /** Format: text */
- _email: string;
- /** Format: text */
- _public_address: string;
- /** Format: text */
- _picture: string;
+ name: string;
+ /** Format: uuid */
+ namespace: string;
};
};
header: {
@@ -1936,7 +2036,7 @@ export interface paths {
};
};
};
- "/rpc/create_key": {
+ "/rpc/create_content_with_pin_sync_request": {
post: {
parameters: {
body: {
@@ -1956,11 +2056,14 @@ export interface paths {
};
};
};
- "/rpc/uuid_generate_v4": {
+ "/rpc/create_content": {
post: {
parameters: {
body: {
- args: { [key: string]: unknown };
+ args: {
+ /** Format: json */
+ data: string;
+ };
};
header: {
/** Preference */
@@ -1973,7 +2076,7 @@ export interface paths {
};
};
};
- "/rpc/uuid_ns_dns": {
+ "/rpc/uuid_ns_oid": {
post: {
parameters: {
body: {
@@ -1990,13 +2093,13 @@ export interface paths {
};
};
};
- "/rpc/json_arr_to_json_element_array": {
+ "/rpc/user_psa_storage": {
post: {
parameters: {
body: {
args: {
- /** Format: json */
- _json: string;
+ /** Format: bigint */
+ query_user_id: number;
};
};
header: {
@@ -2010,14 +2113,11 @@ export interface paths {
};
};
};
- "/rpc/user_psa_storage": {
+ "/rpc/pgrst_watch": {
post: {
parameters: {
body: {
- args: {
- /** Format: bigint */
- query_user_id: number;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -2030,14 +2130,11 @@ export interface paths {
};
};
};
- "/rpc/create_psa_pin_request": {
+ "/rpc/uuid_ns_x500": {
post: {
parameters: {
body: {
- args: {
- /** Format: json */
- data: string;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -2050,14 +2147,11 @@ export interface paths {
};
};
};
- "/rpc/user_used_storage": {
+ "/rpc/uuid_nil": {
post: {
parameters: {
body: {
- args: {
- /** Format: bigint */
- query_user_id: number;
- };
+ args: { [key: string]: unknown };
};
header: {
/** Preference */
@@ -2070,7 +2164,7 @@ export interface paths {
};
};
};
- "/rpc/uuid_generate_v1mc": {
+ "/rpc/uuid_ns_dns": {
post: {
parameters: {
body: {
@@ -2087,7 +2181,7 @@ export interface paths {
};
};
};
- "/rpc/create_upload": {
+ "/rpc/upsert_pins": {
post: {
parameters: {
body: {
@@ -2107,6 +2201,28 @@ export interface paths {
};
};
};
+ "/rpc/uuid_generate_v3": {
+ post: {
+ parameters: {
+ body: {
+ args: {
+ /** Format: text */
+ name: string;
+ /** Format: uuid */
+ namespace: string;
+ };
+ };
+ header: {
+ /** Preference */
+ Prefer?: parameters["preferParams"];
+ };
+ };
+ responses: {
+ /** OK */
+ 200: unknown;
+ };
+ };
+ };
}
export interface definitions {
@@ -2206,6 +2322,27 @@ export interface definitions {
/** Format: timestamp with time zone */
deleted_at?: string;
};
+ backup: {
+ /**
+ * Format: bigint
+ * @description Note:
+ * This is a Primary Key.
+ */
+ id: number;
+ /**
+ * Format: bigint
+ * @description Note:
+ * This is a Foreign Key to `upload.id`.
+ */
+ upload_id: number;
+ /** Format: text */
+ url: string;
+ /**
+ * Format: timestamp with time zone
+ * @default timezone('utc'::text, now())
+ */
+ inserted_at: string;
+ };
content: {
/**
* Format: text
@@ -2639,6 +2776,16 @@ export interface parameters {
"rowFilter.auth_key_history.inserted_at": string;
/** Format: timestamp with time zone */
"rowFilter.auth_key_history.deleted_at": string;
+ /** @description backup */
+ "body.backup": definitions["backup"];
+ /** Format: bigint */
+ "rowFilter.backup.id": string;
+ /** Format: bigint */
+ "rowFilter.backup.upload_id": string;
+ /** Format: text */
+ "rowFilter.backup.url": string;
+ /** Format: timestamp with time zone */
+ "rowFilter.backup.inserted_at": string;
/** @description content */
"body.content": definitions["content"];
/** Format: text */
diff --git a/packages/db/postgres/tables.sql b/packages/db/postgres/tables.sql
index c46edf4cbc..83af9f2d52 100644
--- a/packages/db/postgres/tables.sql
+++ b/packages/db/postgres/tables.sql
@@ -286,6 +286,20 @@ CREATE INDEX IF NOT EXISTS upload_inserted_at_idx ON upload (inserted_at);
CREATE INDEX IF NOT EXISTS upload_updated_at_idx ON upload (updated_at);
CREATE INDEX IF NOT EXISTS upload_source_cid_idx ON upload (source_cid);
+-- Details of the backups created for an upload.
+CREATE TABLE IF NOT EXISTS backup
+(
+ id BIGSERIAL PRIMARY KEY,
+ -- Upload that resulted in this backup.
+ upload_id BIGINT NOT NULL REFERENCES upload (id) ON DELETE CASCADE,
+ -- Backup url location.
+ url TEXT NOT NULL,
+ inserted_at TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL,
+ UNIQUE (upload_id, url)
+);
+
+CREATE INDEX IF NOT EXISTS backup_upload_id_idx ON backup (upload_id);
+
-- A request to keep a Pin in sync with the nodes that are pinning it.
CREATE TABLE IF NOT EXISTS pin_sync_request
(
diff --git a/packages/db/test/upload.spec.js b/packages/db/test/upload.spec.js
index 25f16e8edf..490a4a8b01 100644
--- a/packages/db/test/upload.spec.js
+++ b/packages/db/test/upload.spec.js
@@ -278,6 +278,42 @@ describe('upload', () => {
assert.ok(userUploads.find(upload => upload.cid === sourceCid))
})
+ it('lists user uploads with CAR links in parts', async () => {
+ const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi'
+ const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR'
+ // note: `exampleCarParkUrl` and `exampleS3Url` are for same CAR. The s3 url is base32(multihash(car)) and the other is cid v1
+ const exampleCarParkUrl = 'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car'
+ const exampleS3Url = 'https://dotstorage-dev-0.s3.us-east-1.amazonaws.com/raw/bafybeiao32xtnrlibcekpw3vyfi5txgrmvvrua4pccx3xik33ll3qhko2q/2/ciqplrl7tuebgpzbo5nqlqus5hj2kowxzz7ayr4z6ao2ftg7ibcr3ca.car'
+ const created = new Date().toISOString()
+ const name = `rand-${Math.random().toString().slice(2)}`
+ await client.createUpload({
+ user: user._id,
+ contentCid,
+ sourceCid,
+ authKey: authKeys[0]._id,
+ type,
+ dagSize,
+ name,
+ pins: [initialPinData],
+ backupUrls: [`https://backup.cid/${created}`, exampleCarParkUrl, exampleS3Url],
+ created
+ })
+
+ // Default sort {inserted_at, Desc}
+ const { uploads } = await client.listUploads(user._id, { page: 1 })
+ assert.ok(uploads.length > 0)
+ for (const upload of uploads) {
+ // backupUrls raw is private
+ assert.ok(!('backupUrls' in upload), 'upload does not have backupUrls property')
+ assert.ok(Array.isArray(upload.parts), 'upload.parts is an array')
+ }
+ const namedUpload = uploads.find(u => u.name === name)
+ assert.deepEqual(namedUpload.parts, [
+ // this corresponds to `exampleCarParkUrl`
+ 'bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea'
+ ])
+ })
+
it('lists user uploads with CAR links in parts', async () => {
const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi'
const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR'
diff --git a/packages/db/utils.js b/packages/db/utils.js
index b040f9d7a6..902bea1fd6 100644
--- a/packages/db/utils.js
+++ b/packages/db/utils.js
@@ -10,13 +10,21 @@ import { fromString } from 'uint8arrays'
*/
export function normalizeUpload (upload) {
const nUpload = { ...upload }
- const backupUrls = nUpload.backupUrls ?? []
- delete nUpload.backupUrls
+
delete nUpload.content
delete nUpload.sourceCid
+ // get hash links to CARs that contain parts of this upload
/** @type {import('./db-client-types').UploadItemOutput['parts']} */
- const parts = [...carCidV1Base32sFromBackupUrls(backupUrls)]
+ const parts = [
+ // from upload table 'backup_urls' column
+ ...carCidV1Base32sFromBackupUrls(nUpload.backupUrls ?? []),
+ // there is also a backup table that maybe have been joined in
+ // each backup has a url column
+ ...carCidV1Base32sFromBackupUrls((nUpload.backup ?? []).map(o => o.url))
+ ]
+ delete nUpload.backupUrls
+ delete nUpload.backup
return {
...nUpload,