chore: flush data between tests and expose utils (#1109)

- flush DB at every test
- expose DB utils so that we can share factories to populate DB from client packages
- fixed a nasty bug where multiple results were returned from .single because of inconsistent versions of fetch
- moved randomCid and, refactored to use `Math.random`

packages/api/scripts/add-mock-deals.js

@@ -5,10 +5,8 @@
  *     FAUNA_KEY=<SECRET> node add-mock-deals.js
  */
 import { DBClient, gql } from '@web3-storage/db'
-import { CID } from 'multiformats/cid'
-import { sha256 } from 'multiformats/hashes/sha2'
 import * as pb from '@ipld/dag-pb'
-import crypto from 'crypto'
+import { randomCid } from '../test/scripts/helpers'
 
 const FIND_UPLOADS = gql`
   query FindUploads($cursor: String, $size: Int) {
@@ -59,16 +57,6 @@ function randomInt (min, max) {
   return Math.floor(Math.random() * (max - min) + min)
 }
 
-/**
- * @param {number} code
- * @returns {Promise<string>}
- */
-async function randomCid (code) {
-  const bytes = crypto.randomBytes(10)
-  const hash = await sha256.digest(bytes)
-  return CID.create(1, code, hash).toString()
-}
-
 async function mockAggregate () {
   const dataCid = await randomCid(pb.code)
   const pieceCid = await randomCid(0xf101)

packages/api/test/fixtures/init-data.sql

@@ -1,18 +1,18 @@
 -- Mock DB data for API tests
-INSERT INTO public.user (id, name, email, issuer, public_address)
-VALUES (1, 'test-user', 'test@user.com', 'test-magic-issuer', 'test-magic');
+INSERT INTO public.user ( name, email, issuer, public_address)
+VALUES ('test-user', 'test@user.com', 'test-magic-issuer', 'test-magic');
 
-INSERT INTO public.user (id, name, email, issuer, public_address)
-VALUES (2, 'test-upload-user', 'test-upload@user.com', 'test-upload', 'test-upload');
+INSERT INTO public.user ( name, email, issuer, public_address)
+VALUES ('test-upload-user', 'test-upload@user.com', 'test-upload', 'test-upload');
 
-INSERT INTO public.user (id, name, email, issuer, public_address)
-VALUES (3, 'test-status-user', 'test-status@user.com', 'test-status', 'test-status');
+INSERT INTO public.user ( name, email, issuer, public_address)
+VALUES ('test-status-user', 'test-status@user.com', 'test-status', 'test-status');
 
-INSERT INTO public.user (id, name, email, issuer, public_address)
-VALUES (4, 'test-pinning-user', 'test-pinning@user.com', 'test-pinning', 'test-pinning');
+INSERT INTO public.user ( name, email, issuer, public_address)
+VALUES ('test-pinning-user', 'test-pinning@user.com', 'test-pinning', 'test-pinning');
 
-INSERT INTO public.user (id, name, email, issuer, public_address)
-VALUES (5, 'test-pinning-2-user', 'test-pinning2@user.com', 'test-pinning-2', 'test-pinning-2');
+INSERT INTO public.user ( name, email, issuer, public_address)
+VALUES ('test-pinning-2-user', 'test-pinning2@user.com', 'test-pinning-2', 'test-pinning-2');
 
 INSERT INTO auth_key (name, secret, user_id)
 VALUES ('test-key', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0LW1hZ2ljLWlzc3VlciIsImlzcyI6IndlYjMtc3RvcmFnZSIsImlhdCI6MTYzMzk1NzM4OTg3MiwibmFtZSI6InRlc3QtbWFnaWMtaXNzdWVyIn0.p2nD1Q4X4Z6DtJ0vxk35hhZOqSPVymhN5uyXrXth1zs', 1);

packages/api/test/hooks.js

@@ -67,6 +67,10 @@ export const mochaHooks = () => {
         console.log('🛑 Stopping PostgreSQL and PostgREST')
         execa(dbCli, ['db', '--stop', '--clean', '--project', projectDb])
       }
+    },
+
+    async beforeEach () {
+      await execa(dbCli, ['db-sql', '--skipCreate', '--truncate', `--customSqlPath=${initScript}`])
     }
   }
 }

packages/api/test/scripts/helpers.js

@@ -8,6 +8,8 @@ import { keys } from 'libp2p-crypto'
 import * as JWT from '../../src/utils/jwt.js'
 import { JWT_ISSUER } from '../../src/constants.js'
 import { SALT } from './worker-globals.js'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as pb from '@ipld/dag-pb'
 
 const libp2pKeyCode = 0x72
 const lifetime = 1000 * 60 * 60
@@ -52,3 +54,12 @@ export async function updateNameRecord (privKey, existingRecord, newValue) {
 export function getTestJWT (sub = 'test-magic-issuer', name = 'test-magic-issuer') {
   return JWT.sign({ sub, iss: JWT_ISSUER, iat: 1633957389872, name }, SALT)
 }
+
+/**
+ * @param {number} code
+ * @returns {Promise<string>}
+ */
+export async function randomCid (code = pb.code) {
+  const hash = await sha256.digest(Buffer.from(`${Math.random()}`))
+  return CID.create(1, code, hash).toString()
+}

packages/db/index.js

@@ -64,7 +64,7 @@ export class DBClient {
     this._client = new PostgrestClient(endpoint, {
       headers: {
         Authorization: `Bearer ${token}`,
-        accept: '*/*'
+        Accept: '*/*'
       }
     })
   }

packages/db/package.json

@@ -21,7 +21,7 @@
   "author": "Alan Shaw",
   "license": "(Apache-2.0 OR MIT)",
   "dependencies": {
-    "@supabase/postgrest-js": "^0.34.0",
+    "@supabase/postgrest-js": "^0.37.0",
     "p-retry": "^4.6.1"
   },
   "devDependencies": {
@@ -36,5 +36,9 @@
     "playwright-test": "^7.0.1",
     "sade": "^1.7.4",
     "standard": "^16.0.3"
+  },
+  "exports": {
+    ".": "./index.js",
+    "./test-utils": "./test/utils.js"
   }
 }

packages/db/postgres/docker/docker-compose.yml

@@ -13,6 +13,7 @@ services:
       PGRST_DB_SCHEMA: public
       PGRST_DB_ANON_ROLE: postgres
       PGRST_JWT_SECRET: super-secret-jwt-token-with-at-least-32-characters-long
+      PGRST_LOG_LEVEL: info
   db:
     build:
       context: ./postgres

packages/db/postgres/truncate.sql

@@ -0,0 +1,17 @@
+CREATE OR REPLACE FUNCTION truncate_tables(username IN VARCHAR, schema_n in VARCHAR) RETURNS void AS $$
+DECLARE
+    statements CURSOR FOR
+        SELECT tablename FROM pg_tables
+        WHERE tableowner = username AND schemaname = schema_n;
+BEGIN
+
+    FOR stmt IN statements LOOP
+        EXECUTE 'TRUNCATE TABLE ' || schema_n || '.' || quote_ident(stmt.tablename) || ' RESTART IDENTITY CASCADE;';
+    END LOOP;
+END;
+$$ LANGUAGE plpgsql;
+
+
+SELECT truncate_tables('postgres', 'public');
+SELECT truncate_tables('postgres', 'cargo');
+

packages/db/scripts/cli.js

@@ -18,6 +18,8 @@ prog
   .command('db-sql')
   .describe('Database scripts')
   .option('--reset', 'Reset db before running SQL.', false)
+  .option('--truncate', 'Truncates existing data before running SQL', false)
+  .option('--skipCreate', 'Skip creating tables from schema', false)
   .option('--cargo', 'Import cargo data.', false)
   .option('--testing', 'Tweak schema for testing.', false)
   .action(dbSqlCmd)

packages/db/scripts/cmds/db-sql.js

@@ -16,9 +16,11 @@ const { Client } = pg
  * @param {boolean} [opts.reset]
  * @param {boolean} [opts.cargo]
  * @param {boolean} [opts.testing]
+ * @param {boolean} [opts.truncate]
  * @param {string} [opts.customSqlPath]
+ * @param {boolean} [opts.skipCreate]
  */
-export async function dbSqlCmd ({ reset, cargo, testing, customSqlPath } = {}) {
+export async function dbSqlCmd ({ reset, cargo, testing, truncate, customSqlPath, skipCreate } = {}) {
   // read all the SQL files
   const configSql = fs.readFileSync(path.join(__dirname, '../../postgres/config.sql'), 'utf-8')
 
@@ -28,6 +30,7 @@ export async function dbSqlCmd ({ reset, cargo, testing, customSqlPath } = {}) {
   const cargoSql = fs.readFileSync(path.join(__dirname, '../../postgres/cargo.sql'), 'utf-8')
   const cargoTesting = fs.readFileSync(path.join(__dirname, '../../postgres/cargo.testing.sql'), 'utf-8')
   let fdwSql = fs.readFileSync(path.join(__dirname, '../../postgres/fdw.sql'), 'utf-8')
+  const truncateSql = fs.readFileSync(path.join(__dirname, '../../postgres/truncate.sql'), 'utf-8')
   // Ready custom
   const customSql = customSqlPath ? fs.readFileSync(customSqlPath, 'utf-8') : undefined
 
@@ -46,25 +49,31 @@ export async function dbSqlCmd ({ reset, cargo, testing, customSqlPath } = {}) {
     await client.query(resetSql)
   }
 
-  await client.query(configSql)
-  await client.query(tablesSql)
+  if (truncate) {
+    await client.query(truncateSql)
+  }
 
-  // if testing or cargo fdw flag not set, you just get the schema, no fdw connection to dagcargo
-  if (testing && cargo) {
-    await client.query(cargoTesting)
-  } else if (!testing && cargo) {
-    // Replace secrets in the FDW sql file
-    fdwSql = fdwSql.replace(":'DAG_CARGO_HOST'", `'${process.env.DAG_CARGO_HOST}'`)
-    fdwSql = fdwSql.replace(":'DAG_CARGO_DATABASE'", `'${process.env.DAG_CARGO_DATABASE}'`)
-    fdwSql = fdwSql.replaceAll(":'DAG_CARGO_USER'", `'${process.env.DAG_CARGO_USER}'`)
-    fdwSql = fdwSql.replaceAll(":'DAG_CARGO_PASSWORD'", `'${process.env.DAG_CARGO_PASSWORD}'`)
-    fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_USER', `${process.env.WEB3_STORAGE_USER || 'CURRENT_USER'}`)
-    fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_STATS_USER', `${process.env.WEB3_STORAGE_STATS_USER || 'CURRENT_USER'}`)
-    await client.query(fdwSql)
-    await client.query(cargoSql)
+  if (!skipCreate) {
+    console.log('creating')
+    await client.query(configSql)
+    await client.query(tablesSql)
+    // if testing or cargo fdw flag not set, you just get the schema, no fdw connection to dagcargo
+    if (testing && cargo) {
+      await client.query(cargoTesting)
+    } else if (!testing && cargo) {
+      // Replace secrets in the FDW sql file
+      fdwSql = fdwSql.replace(":'DAG_CARGO_HOST'", `'${process.env.DAG_CARGO_HOST}'`)
+      fdwSql = fdwSql.replace(":'DAG_CARGO_DATABASE'", `'${process.env.DAG_CARGO_DATABASE}'`)
+      fdwSql = fdwSql.replaceAll(":'DAG_CARGO_USER'", `'${process.env.DAG_CARGO_USER}'`)
+      fdwSql = fdwSql.replaceAll(":'DAG_CARGO_PASSWORD'", `'${process.env.DAG_CARGO_PASSWORD}'`)
+      fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_USER', `${process.env.WEB3_STORAGE_USER || 'CURRENT_USER'}`)
+      fdwSql = fdwSql.replaceAll(':WEB3_STORAGE_STATS_USER', `${process.env.WEB3_STORAGE_STATS_USER || 'CURRENT_USER'}`)
+      await client.query(fdwSql)
+      await client.query(cargoSql)
+    }
+    await client.query(functionsSql)
   }
 
-  await client.query(functionsSql)
   customSql && await client.query(customSql)
   await client.end()
 }