diff --git a/packages/api/scripts/add-mock-deals.js b/packages/api/scripts/add-mock-deals.js index b10fcf9e4b..cdb986c9a5 100644 --- a/packages/api/scripts/add-mock-deals.js +++ b/packages/api/scripts/add-mock-deals.js @@ -5,10 +5,8 @@ * FAUNA_KEY= node add-mock-deals.js */ import { DBClient, gql } from '@web3-storage/db' -import { CID } from 'multiformats/cid' -import { sha256 } from 'multiformats/hashes/sha2' import * as pb from '@ipld/dag-pb' -import crypto from 'crypto' +import { randomCid } from '../test/scripts/helpers' const FIND_UPLOADS = gql` query FindUploads($cursor: String, $size: Int) { @@ -59,16 +57,6 @@ function randomInt (min, max) { return Math.floor(Math.random() * (max - min) + min) } -/** - * @param {number} code - * @returns {Promise} - */ -async function randomCid (code) { - const bytes = crypto.randomBytes(10) - const hash = await sha256.digest(bytes) - return CID.create(1, code, hash).toString() -} - async function mockAggregate () { const dataCid = await randomCid(pb.code) const pieceCid = await randomCid(0xf101) diff --git a/packages/api/test/fixtures/init-data.sql b/packages/api/test/fixtures/init-data.sql index b9ec733d7d..74f8ed5651 100644 --- a/packages/api/test/fixtures/init-data.sql +++ b/packages/api/test/fixtures/init-data.sql @@ -1,18 +1,18 @@ -- Mock DB data for API tests -INSERT INTO public.user (id, name, email, issuer, public_address) -VALUES (1, 'test-user', 'test@user.com', 'test-magic-issuer', 'test-magic'); +INSERT INTO public.user ( name, email, issuer, public_address) +VALUES ('test-user', 'test@user.com', 'test-magic-issuer', 'test-magic'); -INSERT INTO public.user (id, name, email, issuer, public_address) -VALUES (2, 'test-upload-user', 'test-upload@user.com', 'test-upload', 'test-upload'); +INSERT INTO public.user ( name, email, issuer, public_address) +VALUES ('test-upload-user', 'test-upload@user.com', 'test-upload', 'test-upload'); -INSERT INTO public.user (id, name, email, issuer, public_address) -VALUES (3, 'test-status-user', 'test-status@user.com', 'test-status', 'test-status'); +INSERT INTO public.user ( name, email, issuer, public_address) +VALUES ('test-status-user', 'test-status@user.com', 'test-status', 'test-status'); -INSERT INTO public.user (id, name, email, issuer, public_address) -VALUES (4, 'test-pinning-user', 'test-pinning@user.com', 'test-pinning', 'test-pinning'); +INSERT INTO public.user ( name, email, issuer, public_address) +VALUES ('test-pinning-user', 'test-pinning@user.com', 'test-pinning', 'test-pinning'); -INSERT INTO public.user (id, name, email, issuer, public_address) -VALUES (5, 'test-pinning-2-user', 'test-pinning2@user.com', 'test-pinning-2', 'test-pinning-2'); +INSERT INTO public.user ( name, email, issuer, public_address) +VALUES ('test-pinning-2-user', 'test-pinning2@user.com', 'test-pinning-2', 'test-pinning-2'); INSERT INTO auth_key (name, secret, user_id) VALUES ('test-key', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0LW1hZ2ljLWlzc3VlciIsImlzcyI6IndlYjMtc3RvcmFnZSIsImlhdCI6MTYzMzk1NzM4OTg3MiwibmFtZSI6InRlc3QtbWFnaWMtaXNzdWVyIn0.p2nD1Q4X4Z6DtJ0vxk35hhZOqSPVymhN5uyXrXth1zs', 1); diff --git a/packages/api/test/hooks.js b/packages/api/test/hooks.js index bd7ab35df3..f2142991b8 100644 --- a/packages/api/test/hooks.js +++ b/packages/api/test/hooks.js @@ -67,6 +67,10 @@ export const mochaHooks = () => { console.log('🛑 Stopping PostgreSQL and PostgREST') execa(dbCli, ['db', '--stop', '--clean', '--project', projectDb]) } + }, + + async beforeEach () { + await execa(dbCli, ['db-sql', '--skipCreate', '--truncate', `--customSqlPath=${initScript}`]) } } } diff --git a/packages/api/test/scripts/helpers.js b/packages/api/test/scripts/helpers.js index d3e4e1f91e..f052bf66f9 100644 --- a/packages/api/test/scripts/helpers.js +++ b/packages/api/test/scripts/helpers.js @@ -8,6 +8,8 @@ import { keys } from 'libp2p-crypto' import * as JWT from '../../src/utils/jwt.js' import { JWT_ISSUER } from '../../src/constants.js' import { SALT } from './worker-globals.js' +import { sha256 } from 'multiformats/hashes/sha2' +import * as pb from '@ipld/dag-pb' const libp2pKeyCode = 0x72 const lifetime = 1000 * 60 * 60 @@ -52,3 +54,12 @@ export async function updateNameRecord (privKey, existingRecord, newValue) { export function getTestJWT (sub = 'test-magic-issuer', name = 'test-magic-issuer') { return JWT.sign({ sub, iss: JWT_ISSUER, iat: 1633957389872, name }, SALT) } + +/** + * @param {number} code + * @returns {Promise} + */ +export async function randomCid (code = pb.code) { + const hash = await sha256.digest(Buffer.from(`${Math.random()}`)) + return CID.create(1, code, hash).toString() +} diff --git a/packages/db/index.js b/packages/db/index.js index 29ce140b94..fdb166338b 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -64,7 +64,7 @@ export class DBClient { this._client = new PostgrestClient(endpoint, { headers: { Authorization: `Bearer ${token}`, - accept: '*/*' + Accept: '*/*' } }) } diff --git a/packages/db/package.json b/packages/db/package.json index 144f1948c5..8aeee7d85c 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "author": "Alan Shaw", "license": "(Apache-2.0 OR MIT)", "dependencies": { - "@supabase/postgrest-js": "^0.34.0", + "@supabase/postgrest-js": "^0.37.0", "p-retry": "^4.6.1" }, "devDependencies": { @@ -36,5 +36,9 @@ "playwright-test": "^7.0.1", "sade": "^1.7.4", "standard": "^16.0.3" + }, + "exports": { + ".": "./index.js", + "./test-utils": "./test/utils.js" } } diff --git a/packages/db/postgres/docker/docker-compose.yml b/packages/db/postgres/docker/docker-compose.yml index d5b1a51376..192d6ddd14 100644 --- a/packages/db/postgres/docker/docker-compose.yml +++ b/packages/db/postgres/docker/docker-compose.yml @@ -13,6 +13,7 @@ services: PGRST_DB_SCHEMA: public PGRST_DB_ANON_ROLE: postgres PGRST_JWT_SECRET: super-secret-jwt-token-with-at-least-32-characters-long + PGRST_LOG_LEVEL: info db: build: context: ./postgres diff --git a/packages/db/postgres/truncate.sql b/packages/db/postgres/truncate.sql new file mode 100644 index 0000000000..f03fc5c8b9 --- /dev/null +++ b/packages/db/postgres/truncate.sql @@ -0,0 +1,17 @@ +CREATE OR REPLACE FUNCTION truncate_tables(username IN VARCHAR, schema_n in VARCHAR) RETURNS void AS $$ +DECLARE + statements CURSOR FOR + SELECT tablename FROM pg_tables + WHERE tableowner = username AND schemaname = schema_n; +BEGIN + + FOR stmt IN statements LOOP + EXECUTE 'TRUNCATE TABLE ' || schema_n || '.' || quote_ident(stmt.tablename) || ' RESTART IDENTITY CASCADE;'; + END LOOP; +END; +$$ LANGUAGE plpgsql; + + +SELECT truncate_tables('postgres', 'public'); +SELECT truncate_tables('postgres', 'cargo'); + diff --git a/packages/db/scripts/cli.js b/packages/db/scripts/cli.js index d4c1d87a63..8c1b68df0d 100755 --- a/packages/db/scripts/cli.js +++ b/packages/db/scripts/cli.js @@ -18,6 +18,8 @@ prog .command('db-sql') .describe('Database scripts') .option('--reset', 'Reset db before running SQL.', false) + .option('--truncate', 'Truncates existing data before running SQL', false) + .option('--skipCreate', 'Skip creating tables from schema', false) .option('--cargo', 'Import cargo data.', false) .option('--testing', 'Tweak schema for testing.', false) .action(dbSqlCmd) diff --git a/packages/db/scripts/cmds/db-sql.js b/packages/db/scripts/cmds/db-sql.js index 357c36e744..0fef6850c9 100644 --- a/packages/db/scripts/cmds/db-sql.js +++ b/packages/db/scripts/cmds/db-sql.js @@ -16,9 +16,11 @@ const { Client } = pg * @param {boolean} [opts.reset] * @param {boolean} [opts.cargo] * @param {boolean} [opts.testing] + * @param {boolean} [opts.truncate] * @param {string} [opts.customSqlPath] + * @param {boolean} [opts.skipCreate] */ -export async function dbSqlCmd ({ reset, cargo, testing, customSqlPath } = {}) { +export async function dbSqlCmd ({ reset, cargo, testing, truncate, customSqlPath, skipCreate } = {}) { // read all the SQL files const configSql = fs.readFileSync(path.join(__dirname, '../../postgres/config.sql'), 'utf-8') @@ -28,6 +30,7 @@ export async function dbSqlCmd ({ reset, cargo, testing, customSqlPath } = {}) { const cargoSql = fs.readFileSync(path.join(__dirname, '../../postgres/cargo.sql'), 'utf-8') const cargoTesting = fs.readFileSync(path.join(__dirname, '../../postgres/cargo.testing.sql'), 'utf-8') let fdwSql = fs.readFileSync(path.join(__dirname, '../../postgres/fdw.sql'), 'utf-8') + const truncateSql = fs.readFileSync(path.join(__dirname, '../../postgres/truncate.sql'), 'utf-8') // Ready custom const customSql = customSqlPath ? fs.readFileSync(customSqlPath, 'utf-8') : undefined @@ -46,25 +49,31 @@ export async function dbSqlCmd ({ reset, cargo, testing, customSqlPath } = {}) { await client.query(resetSql) } - await client.query(configSql) - await client.query(tablesSql) + if (truncate) { + await client.query(truncateSql) + } - // if testing or cargo fdw flag not set, you just get the schema, no fdw connection to dagcargo - if (testing && cargo) { - await client.query(cargoTesting) - } else if (!testing && cargo) { - // Replace secrets in the FDW sql file - fdwSql = fdwSql.replace(":'DAG_CARGO_HOST'", `'${process.env.DAG_CARGO_HOST}'`) - fdwSql = fdwSql.replace(":'DAG_CARGO_DATABASE'", `'${process.env.DAG_CARGO_DATABASE}'`) - fdwSql = fdwSql.replaceAll(":'DAG_CARGO_USER'", `'${process.env.DAG_CARGO_USER}'`) - fdwSql = fdwSql.replaceAll(":'DAG_CARGO_PASSWORD'", `'${process.env.DAG_CARGO_PASSWORD}'`) - fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_USER', `${process.env.WEB3_STORAGE_USER || 'CURRENT_USER'}`) - fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_STATS_USER', `${process.env.WEB3_STORAGE_STATS_USER || 'CURRENT_USER'}`) - await client.query(fdwSql) - await client.query(cargoSql) + if (!skipCreate) { + console.log('creating') + await client.query(configSql) + await client.query(tablesSql) + // if testing or cargo fdw flag not set, you just get the schema, no fdw connection to dagcargo + if (testing && cargo) { + await client.query(cargoTesting) + } else if (!testing && cargo) { + // Replace secrets in the FDW sql file + fdwSql = fdwSql.replace(":'DAG_CARGO_HOST'", `'${process.env.DAG_CARGO_HOST}'`) + fdwSql = fdwSql.replace(":'DAG_CARGO_DATABASE'", `'${process.env.DAG_CARGO_DATABASE}'`) + fdwSql = fdwSql.replaceAll(":'DAG_CARGO_USER'", `'${process.env.DAG_CARGO_USER}'`) + fdwSql = fdwSql.replaceAll(":'DAG_CARGO_PASSWORD'", `'${process.env.DAG_CARGO_PASSWORD}'`) + fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_USER', `${process.env.WEB3_STORAGE_USER || 'CURRENT_USER'}`) + fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_STATS_USER', `${process.env.WEB3_STORAGE_STATS_USER || 'CURRENT_USER'}`) + await client.query(fdwSql) + await client.query(cargoSql) + } + await client.query(functionsSql) } - await client.query(functionsSql) customSql && await client.query(customSql) await client.end() }