From 768e3b50b7bd4d65bb1fa8dcceb0da8b4d34ac79 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Tue, 12 Dec 2023 09:59:11 -0500 Subject: [PATCH] trpc test setup (#6872) --- packages/trpc-server/.prettierrc | 6 + packages/trpc-server/README.md | 4 + packages/trpc-server/docker-compose.yml | 30 +++ packages/trpc-server/package.json | 12 +- packages/trpc-server/src/index.ts | 50 +---- .../trpc-server/src/routers/search-router.ts | 109 +++++++++++ packages/trpc-server/src/server.ts | 40 ++++ packages/trpc-server/test.sh | 23 +++ packages/trpc-server/test/_fixtures.ts | 175 ++++++++++++++++++ packages/trpc-server/test/_test_helpers.ts | 15 ++ packages/trpc-server/test/router.test.ts | 57 ++++++ packages/trpc-server/test/search.test.ts | 36 ++++ 12 files changed, 510 insertions(+), 47 deletions(-) create mode 100644 packages/trpc-server/.prettierrc create mode 100644 packages/trpc-server/README.md create mode 100644 packages/trpc-server/docker-compose.yml create mode 100644 packages/trpc-server/src/routers/search-router.ts create mode 100644 packages/trpc-server/src/server.ts create mode 100644 packages/trpc-server/test.sh create mode 100644 packages/trpc-server/test/_fixtures.ts create mode 100644 packages/trpc-server/test/_test_helpers.ts create mode 100644 packages/trpc-server/test/router.test.ts create mode 100644 packages/trpc-server/test/search.test.ts diff --git a/packages/trpc-server/.prettierrc b/packages/trpc-server/.prettierrc new file mode 100644 index 00000000000..66e7e941cb9 --- /dev/null +++ b/packages/trpc-server/.prettierrc @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": false, + "singleQuote": true +} \ No newline at end of file diff --git a/packages/trpc-server/README.md b/packages/trpc-server/README.md new file mode 100644 index 00000000000..b9e859856e1 --- /dev/null +++ b/packages/trpc-server/README.md @@ -0,0 +1,4 @@ +## Tests + +* run once: `bash test.sh run` +* start watch mode: `bash test.sh` diff --git a/packages/trpc-server/docker-compose.yml b/packages/trpc-server/docker-compose.yml new file mode 100644 index 00000000000..8962bd9d223 --- /dev/null +++ b/packages/trpc-server/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3.1' + +services: + + db: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: testing + ports: + - 35764:5432 + volumes: + - ../discovery-provider/ddl:/ddl + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2 + environment: + - network.host=0.0.0.0 + - discovery.type=single-node + - cluster.name=docker-cluster + - node.name=cluster1-node1 + - xpack.license.self_generated.type=basic + - xpack.security.enabled=false + - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + ulimits: + memlock: + soft: -1 + hard: -1 + ports: + - 35765:9200 diff --git a/packages/trpc-server/package.json b/packages/trpc-server/package.json index bcd88cbbb0a..acf0e10d057 100644 --- a/packages/trpc-server/package.json +++ b/packages/trpc-server/package.json @@ -5,11 +5,13 @@ "main": "src/index.ts", "scripts": { "build": "tsc", - "dev": "DEBUG=audius:* tsx watch src/index.ts", + "dev": "DEBUG=audius:* tsx watch src/server.ts", "db:gen": "tsx ./tools/db-gen.ts", - "start": "tsx src/index.ts" + "start": "tsx src/server.ts", + "test": "vitest" }, "dependencies": { + "@elastic/elasticsearch": "^8.10.0", "@trpc/server": "10.38.4", "cors": "2.8.5", "dataloader": "2.2.2", @@ -28,6 +30,8 @@ "@types/swagger-ui-express": "^4.1.4", "eslint": "^8.40.0", "tsx": "^3.12.7", - "typescript": "^5.1.3" + "typescript": "^5.1.3", + "vite-node": "^0.34.6", + "vitest": "^0.34.6" } -} +} \ No newline at end of file diff --git a/packages/trpc-server/src/index.ts b/packages/trpc-server/src/index.ts index 1c9416c98e9..e21ad0e4f34 100644 --- a/packages/trpc-server/src/index.ts +++ b/packages/trpc-server/src/index.ts @@ -1,29 +1,18 @@ -import 'dotenv/config' - -import { createExpressMiddleware } from '@trpc/server/adapters/express' -import cors from 'cors' -import express from 'express' -import swaggerUi from 'swagger-ui-express' -import { - createOpenApiExpressMiddleware, - generateOpenApiDocument -} from 'trpc-openapi' -import { userRouter } from './routers/user-router' -import { createContext, publicProcedure, router } from './trpc' +import { inferRouterInputs, inferRouterOutputs } from '@trpc/server' import { meRouter } from './routers/me-router' -import { trackRouter } from './routers/track-router' import { playlistRouter } from './routers/playlist-router' -import { inferRouterInputs, inferRouterOutputs } from '@trpc/server' - -const app = express() -app.use(cors()) +import { searchRouter } from './routers/search-router' +import { trackRouter } from './routers/track-router' +import { userRouter } from './routers/user-router' +import { publicProcedure, router } from './trpc' // AppRouter -const appRouter = router({ +export const appRouter = router({ me: meRouter, users: userRouter, tracks: trackRouter, playlists: playlistRouter, + search: searchRouter, version: publicProcedure.query(() => ({ version: '0.0.2' @@ -34,28 +23,3 @@ export type AppRouter = typeof appRouter export type RouterInput = inferRouterInputs export type RouterOutput = inferRouterOutputs - -// endpoints -app.use('/trpc', createExpressMiddleware({ router: appRouter, createContext })) -app.use( - '/rest', - createOpenApiExpressMiddleware({ router: appRouter, createContext }) -) - -// OpenAPI schema document -export const openApiDocument = generateOpenApiDocument(appRouter, { - title: 'Example CRUD API', - description: 'OpenAPI compliant REST API built using tRPC with Express', - version: '1.0.0', - baseUrl: 'http://localhost:2022/rest', - docsUrl: 'https://docs.audius.co' -}) - -// Swagger UI -app.use('/', swaggerUi.serve) -app.get('/', swaggerUi.setup(openApiDocument)) - -const port = 2022 -app.listen(port, () => { - console.log('listening on ', port) -}) diff --git a/packages/trpc-server/src/routers/search-router.ts b/packages/trpc-server/src/routers/search-router.ts new file mode 100644 index 00000000000..b0298ef8170 --- /dev/null +++ b/packages/trpc-server/src/routers/search-router.ts @@ -0,0 +1,109 @@ +import { z } from 'zod' +import { publicProcedure, router } from '../trpc' +import { TRPCError } from '@trpc/server' +import { Client as ES } from '@elastic/elasticsearch' + +const esc = new ES({ node: process.env.audius_elasticsearch_url }) + +export const searchRouter = router({ + users: publicProcedure + .input( + z.object({ + q: z.string(), + onlyFollowed: z.boolean().default(false), + limit: z.number().default(20), + cursor: z.string().default('0'), + }) + ) + .query(async ({ ctx, input }) => { + const followedBy = input.onlyFollowed ? ctx.currentUserId : undefined + const found = await esc.search({ + index: 'users', + query: userSearchDSL(input.q, followedBy) as any, + _source: false, + }) + const ids = found.hits.hits.map((h) => h._id) + return ids + }), + + tracks: publicProcedure + .input( + z.object({ + q: z.string(), + onlySaved: z.boolean().default(false), + limit: z.number().default(20), + }) + ) + .query(async ({ ctx, input }) => { + const savedBy = input.onlySaved ? ctx.currentUserId : undefined + const found = await esc.search({ + index: 'tracks', + query: trackSearchDSL(input.q, savedBy) as any, + _source: false, + }) + const ids = found.hits.hits.map((h) => h._id) + return ids + }), +}) + +export function userSearchDSL(q: string, followedBy?: number) { + const dsl: any = { + bool: { + must: [suggestDSL(q), { term: { is_deactivated: { value: false } } }], + must_not: [{ exists: { field: 'stem_of' } }], + should: [], + }, + } + + if (followedBy) { + dsl.bool.must.push({ + terms: { + _id: { + index: 'users', + id: followedBy.toString(), + path: 'following_ids', + }, + }, + }) + } + + return dsl +} + +function trackSearchDSL(q: string, savedByUserId?: number) { + const dsl: any = { + bool: { + must: [ + suggestDSL(q), + { term: { is_unlisted: { value: false } } }, + { term: { is_delete: false } }, + ], + must_not: [], + should: [{ term: { is_verified: { value: true } } }], + }, + } + + if (savedByUserId) { + dsl.bool.must.push({ + term: { saved_by: { value: savedByUserId, boost: 1.2 } }, + }) + } + + return dsl +} + +function suggestDSL( + q: string, + operator: string = 'or', + extraFields: string[] = [] +) { + return { + multi_match: { + query: q, + fields: ['suggest', 'suggest._2gram', 'suggest._3gram', ...extraFields], + operator: operator as any, + type: 'bool_prefix', + fuzziness: 'AUTO', + }, + } +} diff --git a/packages/trpc-server/src/server.ts b/packages/trpc-server/src/server.ts new file mode 100644 index 00000000000..f1a28891359 --- /dev/null +++ b/packages/trpc-server/src/server.ts @@ -0,0 +1,40 @@ +import 'dotenv/config' + +import { createExpressMiddleware } from '@trpc/server/adapters/express' +import cors from 'cors' +import express from 'express' +import swaggerUi from 'swagger-ui-express' +import { + createOpenApiExpressMiddleware, + generateOpenApiDocument +} from 'trpc-openapi' +import { appRouter } from './index' +import { createContext } from './trpc' + +const app = express() +app.use(cors()) + +// endpoints +app.use('/trpc', createExpressMiddleware({ router: appRouter, createContext })) +app.use( + '/rest', + createOpenApiExpressMiddleware({ router: appRouter, createContext }) +) + +// OpenAPI schema document +export const openApiDocument = generateOpenApiDocument(appRouter, { + title: 'Example CRUD API', + description: 'OpenAPI compliant REST API built using tRPC with Express', + version: '1.0.0', + baseUrl: 'http://localhost:2022/rest', + docsUrl: 'https://docs.audius.co' +}) + +// Swagger UI +app.use('/', swaggerUi.serve) +app.get('/', swaggerUi.setup(openApiDocument)) + +const port = 2022 +app.listen(port, () => { + console.log('listening on ', port) +}) diff --git a/packages/trpc-server/test.sh b/packages/trpc-server/test.sh new file mode 100644 index 00000000000..b6007017904 --- /dev/null +++ b/packages/trpc-server/test.sh @@ -0,0 +1,23 @@ +#! /bin/bash +set -e + +# start postgres + elasticsearch +docker compose up -d + +# set env variables +export audius_db_url='postgres://postgres:testing@localhost:35764' +export audius_elasticsearch_url='http://localhost:35765' +export DB_URL="$audius_db_url" + +# run pg_migrate +# cd ../discovery-provider/ddl && ./pg_migrate.sh && cd - || exit +docker exec -w '/ddl' trpc-server-db-1 './pg_migrate.sh' + +# populate db fixtures +npx vite-node test/_fixtures.ts + +# run es-indexer +cd ../es-indexer && npm run catchup:ci && cd - || exit + +# run tests +npx vitest "$1" diff --git a/packages/trpc-server/test/_fixtures.ts b/packages/trpc-server/test/_fixtures.ts new file mode 100644 index 00000000000..590734071b5 --- /dev/null +++ b/packages/trpc-server/test/_fixtures.ts @@ -0,0 +1,175 @@ +/* + +ES Indexer uses blocknumber for cursor. +So records should have a blocknumber to get indexed. + + +*/ + +import { sql } from '../src/db' +import { FollowRow, PlaylistRow, SaveRow, TrackRow } from '../src/db-tables' + +type TableFixture = { + common: RowType + rows: RowType[] +} + +const fixtures = { + blocks: { + common: {}, + rows: [ + { blockhash: '0x1', number: 1 }, + { blockhash: '0x2', number: 2 }, + ], + }, + + users: { + common: { + blocknumber: 1, + is_current: true, + is_verified: true, + created_at: new Date(), + updated_at: new Date(), + has_collectibles: false, + txhash: '0x123', + is_deactivated: false, + is_available: true, + is_storage_v2: true, + allow_ai_attribution: false, + }, + rows: [ + { + user_id: 101, + handle: 'steve', + }, + { + user_id: 102, + handle: 'dave', + }, + { + user_id: 103, + handle: 'dave again', + }, + ], + }, + + tracks: { + common: { + blocknumber: 1, + isCurrent: true, + isDelete: false, + createdAt: new Date(), + updatedAt: new Date(), + isUnlisted: false, + txhash: '0x123', + isAvailable: true, + isPremium: false, + isPlaylistUpload: false, + trackSegments: '[]', + }, + rows: [ + { + ownerId: 101, + trackId: 201, + title: 'Who let the dogs out', + }, + { + ownerId: 101, + trackId: 202, + title: "Steve's unlisted dogs track", + isUnlisted: true, + }, + { + ownerId: 101, + trackId: 203, + title: 'Dogs remix', + }, + ], + } as TableFixture, + + playlists: { + common: { + blocknumber: 1, + createdAt: new Date(), + updatedAt: new Date(), + isCurrent: true, + isDelete: false, + isAlbum: false, + isPrivate: false, + txhash: '0x123', + isImageAutogenerated: false, + playlistContents: { track_ids: [] }, + }, + rows: [ + { + playlistOwnerId: 101, + playlistId: 301, + playlistName: "Steve's Playlist", + }, + { + playlistOwnerId: 101, + playlistId: 302, + playlistName: "Steve's private Playlist", + isPrivate: true, + }, + ], + } as TableFixture, + + follows: { + common: { + blocknumber: 1, + createdAt: new Date(), + isCurrent: true, + isDelete: false, + txhash: '0x123', + }, + rows: [ + { + followerUserId: 101, + followeeUserId: 102, + }, + ], + } as TableFixture, + + saves: { + common: { + blocknumber: 1, + createdAt: new Date(), + isCurrent: true, + isDelete: false, + txhash: '0x123', + isSaveOfRepost: false, + }, + rows: [ + { + userId: 102, + saveType: 'track', + saveItemId: 201, + }, + ], + } as TableFixture, +} + +async function main() { + // truncate tables + for (const table of Object.keys(fixtures)) { + await sql`truncate ${sql(table)} cascade` + } + await sql`truncate aggregate_user cascade` + await sql`truncate aggregate_track cascade` + await sql`truncate aggregate_playlist cascade` + + // create records + for (const [table, fixture] of Object.entries(fixtures)) { + await Promise.all( + fixture.rows.map( + (r) => + sql`insert into ${sql(table)} ${sql({ ...fixture.common, ...r })}` + ) + ) + } + + await sql.end() +} + +main() diff --git a/packages/trpc-server/test/_test_helpers.ts b/packages/trpc-server/test/_test_helpers.ts new file mode 100644 index 00000000000..f462090c0c3 --- /dev/null +++ b/packages/trpc-server/test/_test_helpers.ts @@ -0,0 +1,15 @@ +import { appRouter } from '../src' +import { createContext } from '../src/trpc' + +export async function testRouter(userId?: number) { + const ctx = await createContext({ + req: { + headers: { + 'x-current-user-id': userId + }, + query: {} + } + }) + + return appRouter.createCaller(ctx) +} diff --git a/packages/trpc-server/test/router.test.ts b/packages/trpc-server/test/router.test.ts new file mode 100644 index 00000000000..20693e9d2c2 --- /dev/null +++ b/packages/trpc-server/test/router.test.ts @@ -0,0 +1,57 @@ +/* +example test from: +https://github.com/trpc/examples-next-prisma-starter/blob/main/src/server/routers/post.test.ts +*/ +import { expect, test } from 'vitest' +import { testRouter } from './_test_helpers' + +test('version', async () => { + const caller = await testRouter() + const ver = await caller.version() + expect(ver).toMatchObject({ version: '0.0.2' }) +}) + +test('get user', async () => { + const caller = await testRouter(101) + + const steve = await caller.users.get('101') + expect(steve.handle).toEqual('steve') + expect(steve.trackCount).toEqual(2) + expect(steve.playlistCount).toEqual(1) + expect(steve.followerCount).toEqual(0) + expect(steve.followingCount).toEqual(1) + + const dave = await caller.users.get('102') + expect(dave.handle).toEqual('dave') + expect(dave.followerCount).toEqual(1) + expect(dave.followingCount).toEqual(0) + + // steve follows dave + { + const rel = await caller.me.userRelationship({ theirId: '102' }) + expect(rel.followed).toEqual(true) + expect(rel.followsMe).toEqual(false) + } + + // dave followed by steve + { + const caller = await testRouter(102) + const rel = await caller.me.userRelationship({ theirId: '101' }) + expect(rel.followed).toEqual(false) + expect(rel.followsMe).toEqual(true) + } +}) + +test('get track', async () => { + const caller = await testRouter(101) + const t = await caller.tracks.get('201') + expect(t.title).toEqual('Who let the dogs out') +}) + +test("dave tries to get steve's private track", async () => { + // TODO: this should probably 404... + const daveRouter = await testRouter(201) + const t = await daveRouter.tracks.get('202') + expect(t.title).toEqual("Steve's unlisted dogs track") + expect(t.isUnlisted).toEqual(true) +}) diff --git a/packages/trpc-server/test/search.test.ts b/packages/trpc-server/test/search.test.ts new file mode 100644 index 00000000000..22d46ab3ee8 --- /dev/null +++ b/packages/trpc-server/test/search.test.ts @@ -0,0 +1,36 @@ +import { expect, test } from 'vitest' +import { testRouter } from './_test_helpers' + +test('search user', async () => { + const caller = await testRouter(101) + { + const userIds = await caller.search.users({ q: 'steve' }) + expect(userIds).toEqual(['101']) + } + + { + const userIds = await caller.search.users({ q: 'dave' }) + expect(userIds).toEqual(['102', '103']) + } + + { + const userIds = await caller.search.users({ q: 'dave', onlyFollowed: true }) + expect(userIds).toEqual(['102']) + } +}) + +test('search tracks', async () => { + const caller = await testRouter(102) + + { + const trackIds = await caller.search.tracks({ q: 'dogs' }) + expect(trackIds).length(2) + expect(trackIds).toContain('201') + expect(trackIds).toContain('203') + } + + { + const trackIds = await caller.search.tracks({ q: 'dogs', onlySaved: true }) + expect(trackIds).toEqual(['201']) + } +})