From ddf9e2edb0168b01365b4faca1892146f0881169 Mon Sep 17 00:00:00 2001 From: tudor <7089284+tudddorrr@users.noreply.github.com> Date: Sat, 17 Aug 2024 17:04:27 +0100 Subject: [PATCH] migrate events from mysql to clickhouse --- clickhouse/Dockerfile | 10 ++ clickhouse/entrypoint.sh | 21 ++++ .../migrations/001CreateEventsTable.sql | 16 +++ .../migrations/002CreateEventPropsTable.sql | 6 + docker-compose.ci.yml | 38 +----- docker-compose.dev.yml | 5 +- docker-compose.test.yml | 14 +++ docker-compose.yml | 18 ++- envs/.dockerignore | 0 envs/.env.dev | 6 + envs/.env.test | 6 + package-lock.json | 17 +++ package.json | 1 + src/entities/event.ts | 86 +++++++++++--- src/entities/index.ts | 2 - src/entities/prop.ts | 2 +- src/lib/clickhouse/createClient.ts | 8 ++ src/lib/clickhouse/formatDateTime.ts | 10 ++ src/lib/demo-data/generateDemoEvents.ts | 66 ++++++++--- src/services/api/event-api.service.ts | 69 ++++++++--- src/services/data-export.service.ts | 24 +++- src/services/event.service.ts | 108 ++++++++++-------- src/services/headline.service.ts | 81 +++++++------ src/services/player.service.ts | 59 ++++++---- tests/fixtures/EventFactory.ts | 2 - tests/run-tests.sh | 2 +- tests/services/_public/demo/post.test.ts | 36 +++--- .../data-export/included-data.test.ts | 15 ++- tests/services/event/get.test.ts | 101 +++++++++------- tests/services/headline/get.test.ts | 76 ++++++++---- tests/services/player/events.test.ts | 36 ++++-- tests/setupTest.ts | 12 ++ 32 files changed, 655 insertions(+), 298 deletions(-) create mode 100644 clickhouse/Dockerfile create mode 100755 clickhouse/entrypoint.sh create mode 100644 clickhouse/migrations/001CreateEventsTable.sql create mode 100644 clickhouse/migrations/002CreateEventPropsTable.sql create mode 100644 envs/.dockerignore create mode 100644 src/lib/clickhouse/createClient.ts create mode 100644 src/lib/clickhouse/formatDateTime.ts diff --git a/clickhouse/Dockerfile b/clickhouse/Dockerfile new file mode 100644 index 00000000..bcddcf73 --- /dev/null +++ b/clickhouse/Dockerfile @@ -0,0 +1,10 @@ +FROM clickhouse/clickhouse-server:24-alpine + +RUN apk add --no-cache gettext + +COPY /clickhouse/migrations /docker-entrypoint-initdb.d/ +COPY /clickhouse/entrypoint.sh /usr/local/bin/entrypoint.sh + +RUN chmod +x /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/clickhouse/entrypoint.sh b/clickhouse/entrypoint.sh new file mode 100755 index 00000000..d301480d --- /dev/null +++ b/clickhouse/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +INIT_DIR="/docker-entrypoint-initdb.d" +FLAG_FILE="$INIT_DIR/.migrated" + +if [ ! -f "$FLAG_FILE" ]; then + for file in ${INIT_DIR}/*.sql; do + if [ -f "$file" ]; then + envsubst < "$file" > "$file.processed" && mv "$file.processed" "$file" + echo "Processed migration $file" + fi + done + + echo "Creating migrations lock" + touch "$FLAG_FILE" +else + echo "Migrations lock set; Skipping envsubst processing" +fi + +exec /entrypoint.sh "$@" diff --git a/clickhouse/migrations/001CreateEventsTable.sql b/clickhouse/migrations/001CreateEventsTable.sql new file mode 100644 index 00000000..fc3a126e --- /dev/null +++ b/clickhouse/migrations/001CreateEventsTable.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.events +( + id String, + name String, + game_id UInt32, + player_alias_id UInt32, + dev_build Boolean, + created_at DateTime, + updated_at DateTime, + PRIMARY KEY (id), + INDEX name_idx (name) TYPE bloom_filter(0.01) GRANULARITY 64, + INDEX game_id_idx (game_id) TYPE minmax GRANULARITY 64, + INDEX player_alias_id_idx (player_alias_id) TYPE minmax GRANULARITY 64 +) +ENGINE = MergeTree() +ORDER BY (id, created_at, game_id, player_alias_id); diff --git a/clickhouse/migrations/002CreateEventPropsTable.sql b/clickhouse/migrations/002CreateEventPropsTable.sql new file mode 100644 index 00000000..377f29cf --- /dev/null +++ b/clickhouse/migrations/002CreateEventPropsTable.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.event_props ( + event_id String, + prop_key String, + prop_value String +) ENGINE = MergeTree() +ORDER BY (event_id, prop_key); diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index f15cbab6..806afb0e 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,45 +1,9 @@ services: - test-db: - image: mysql:8.4 - command: --mysql-native-password=ON - environment: - - MYSQL_DATABASE=${DB_NAME} - - MYSQL_ROOT_PASSWORD=${DB_PASS} - restart: always - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"] - interval: 2s - timeout: 2s - retries: 10 - ports: - - ${DB_PORT}:3306 - volumes: - - test-data:/var/lib/mysql - networks: - - test-network - test-redis: image: bitnami/redis:7.2 + command: environment: - REDIS_PASSWORD=${REDIS_PASSWORD} - ports: - - ${REDIS_PORT}:6379 - depends_on: - test-db: - condition: service_healthy - networks: - - test-network stripe-api: image: stripe/stripe-mock:latest - ports: - - 12111:12111 - - 12112:12112 - networks: - - test-network - -volumes: - test-data: - -networks: - test-network: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0119a518..a22b1eab 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,10 +4,11 @@ services: context: . target: dev image: backend + depends_on: + - db ports: - 3000:80 volumes: - ./src:/usr/backend/src - ./tests:/usr/backend/tests - depends_on: - - db + \ No newline at end of file diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 478c11f4..beca9f2b 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -29,6 +29,20 @@ services: networks: - test-network + test-clickhouse: + build: + context: . + dockerfile: ./clickhouse/Dockerfile + environment: + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DB: ${CLICKHOUSE_DB} + restart: unless-stopped + ports: + - ${CLICKHOUSE_PORT}:8123 + networks: + - test-network + stripe-api: image: stripe/stripe-mock:latest-arm64 ports: diff --git a/docker-compose.yml b/docker-compose.yml index fdbb4161..55103f56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: depends_on: - db - redis + - clickhouse db: image: mysql:8.4 @@ -12,7 +13,7 @@ services: environment: - MYSQL_DATABASE=${DB_NAME} - MYSQL_ROOT_PASSWORD=${DB_PASS} - restart: always + restart: unless-stopped ports: - 3306:3306 volumes: @@ -24,5 +25,20 @@ services: ports: - 6379:6379 + clickhouse: + build: + context: . + dockerfile: ./clickhouse/Dockerfile + environment: + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DB: ${CLICKHOUSE_DB} + restart: unless-stopped + ports: + - ${CLICKHOUSE_PORT}:8123 + volumes: + - clickhouse-data:/var/lib/clickhouse + volumes: data: + clickhouse-data: diff --git a/envs/.dockerignore b/envs/.dockerignore new file mode 100644 index 00000000..e69de29b diff --git a/envs/.env.dev b/envs/.env.dev index db0ed112..b2b196e8 100644 --- a/envs/.env.dev +++ b/envs/.env.dev @@ -9,6 +9,12 @@ DB_PASS=password REDIS_PASSWORD=password +CLICKHOUSE_HOST=clickhouse +CLICKHOUSE_PORT=8123 +CLICKHOUSE_USER=gs_ch +CLICKHOUSE_PASSWORD=password +CLICKHOUSE_DB=gs_ch_dev + DEMO_ORGANISATION_NAME=Talo Demo SENDGRID_KEY= diff --git a/envs/.env.test b/envs/.env.test index 96e351b5..73717368 100644 --- a/envs/.env.test +++ b/envs/.env.test @@ -3,6 +3,12 @@ DB_PORT=3307 DB_NAME=gs_test DB_PASS=password +CLICKHOUSE_HOST=127.0.0.1 +CLICKHOUSE_PORT=8124 +CLICKHOUSE_USER= +CLICKHOUSE_PASSWORD= +CLICKHOUSE_DB=gs_ch_test + SENDGRID_KEY= SENTRY_DSN= diff --git a/package-lock.json b/package-lock.json index 639c7c13..221dc46d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.38.0", "license": "MIT", "dependencies": { + "@clickhouse/client": "^1.4.1", "@dinero.js/currencies": "^2.0.0-alpha.14", "@koa/cors": "^5.0.0", "@mikro-orm/core": "^6.3.2", @@ -156,6 +157,22 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@clickhouse/client": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.4.1.tgz", + "integrity": "sha512-12iV+MeykxdQySRFHwaVU+hKUv3JP6kdwOI+z3zzyfPVYHynTlV8emJjjGZR0+VfRaj3PCMuQfryfsJ82nh9WQ==", + "dependencies": { + "@clickhouse/client-common": "1.4.1" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@clickhouse/client-common": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@clickhouse/client-common/-/client-common-1.4.1.tgz", + "integrity": "sha512-f5eoTrUSDplrMoi3ddeZ0MzGTn0iGMByEQ8j63eVMoBSOI2+F6jEIPcW2tWofT79Rvnn3RRlveYcShiaIiCJyw==" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", diff --git a/package.json b/package.json index 8057c371..160d9395 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "vitest": "^1.5.2" }, "dependencies": { + "@clickhouse/client": "^1.4.1", "@dinero.js/currencies": "^2.0.0-alpha.14", "@koa/cors": "^5.0.0", "@mikro-orm/core": "^6.3.2", diff --git a/src/entities/event.ts b/src/entities/event.ts index 2e459ec8..80884458 100644 --- a/src/entities/event.ts +++ b/src/entities/event.ts @@ -1,35 +1,41 @@ -import { Entity, Embedded, ManyToOne, PrimaryKey, Property, Cascade } from '@mikro-orm/mysql' +import { v4 } from 'uuid' import sanitiseProps from '../lib/props/sanitiseProps' import Game from './game' import PlayerAlias from './player-alias' import Prop from './prop' +import { formatDateForClickHouse } from '../lib/clickhouse/formatDateTime' +import { EntityManager } from '@mikro-orm/mysql' +import createClickhouseClient from '../lib/clickhouse/createClient' const eventMetaProps = ['META_OS', 'META_GAME_VERSION', 'META_WINDOW_MODE', 'META_SCREEN_WIDTH', 'META_SCREEN_HEIGHT'] -@Entity() -export default class Event { - @PrimaryKey() - id: number - - @Property() +export type ClickhouseEvent = { + id: string name: string + game_id: number + player_alias_id: number + dev_build: boolean + created_at: string + updated_at: string +} - @Embedded(() => Prop, { array: true }) - props: Prop[] = [] +export type ClickhouseEventProp = { + event_id: string + prop_key: string + prop_value: string +} - @ManyToOne(() => Game) +export default class Event { + id: string + name: string + props: Prop[] = [] game: Game - - @ManyToOne(() => PlayerAlias, { cascade: [Cascade.REMOVE] }) playerAlias: PlayerAlias - - @Property() - createdAt: Date = new Date() - - @Property({ onUpdate: () => new Date() }) + createdAt: Date updatedAt: Date = new Date() constructor(name: string, game: Game) { + this.id = v4() this.name = name this.game = game } @@ -51,6 +57,26 @@ export default class Event { }) } + getInsertableData(): ClickhouseEvent { + return { + id: this.id, + name: this.name, + game_id: this.game.id, + player_alias_id: this.playerAlias.id, + dev_build: this.playerAlias.player.isDevBuild(), + created_at: formatDateForClickHouse(this.createdAt), + updated_at: formatDateForClickHouse(this.updatedAt) + } + } + + getInsertableProps(): ClickhouseEventProp[] { + return this.props.map((prop) => ({ + event_id: this.id, + prop_key: prop.key, + prop_value: prop.value + })) + } + toJSON() { return { id: this.id, @@ -62,3 +88,29 @@ export default class Event { } } } + +export async function createEventFromClickhouse(em: EntityManager, data: ClickhouseEvent, loadProps = false): Promise { + const game = await em.getRepository(Game).findOne(data.game_id) + const playerAlias = await em.getRepository(PlayerAlias).findOne(data.player_alias_id, { populate: ['player'] }) + + const event = new Event(data.name, game) + event.id = data.id + event.playerAlias = playerAlias + event.createdAt = new Date(data.created_at) + event.updatedAt = new Date(data.updated_at) + + if (loadProps) { + const clickhouse = createClickhouseClient() + + const props = await clickhouse.query({ + query: `SELECT * FROM event_props WHERE event_id = '${data.id}'`, + format: 'JSONEachRow' + }).then((res) => res.json()) + + event.props = props.map((prop) => new Prop(prop.prop_key, prop.prop_value)) + + clickhouse.close() + } + + return event +} diff --git a/src/entities/index.ts b/src/entities/index.ts index 8695d346..2e4da0d6 100644 --- a/src/entities/index.ts +++ b/src/entities/index.ts @@ -10,7 +10,6 @@ import Leaderboard from './leaderboard' import LeaderboardEntry from './leaderboard-entry' import APIKey from './api-key' import DataExport from './data-export' -import Event from './event' import FailedJob from './failed-job' import Game from './game' import Organisation from './organisation' @@ -58,7 +57,6 @@ export default [ LeaderboardEntry, DataExport, APIKey, - Event, Game, FailedJob, Organisation, diff --git a/src/entities/prop.ts b/src/entities/prop.ts index ef1434d2..9d9b3107 100644 --- a/src/entities/prop.ts +++ b/src/entities/prop.ts @@ -6,7 +6,7 @@ export default class Prop { key: string @Property() - value: string|null + value: string | null constructor(key: string, value?: string) { this.key = key diff --git a/src/lib/clickhouse/createClient.ts b/src/lib/clickhouse/createClient.ts new file mode 100644 index 00000000..e6fc39d7 --- /dev/null +++ b/src/lib/clickhouse/createClient.ts @@ -0,0 +1,8 @@ +import { createClient } from '@clickhouse/client' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' + +export default function createClickhouseClient(): NodeClickHouseClient { + return createClient({ + url: `http://${process.env.CLICKHOUSE_USER}:${process.env.CLICKHOUSE_PASSWORD}@${process.env.CLICKHOUSE_HOST}:${process.env.CLICKHOUSE_PORT}/${process.env.CLICKHOUSE_DB}` + }) +} diff --git a/src/lib/clickhouse/formatDateTime.ts b/src/lib/clickhouse/formatDateTime.ts new file mode 100644 index 00000000..797c4865 --- /dev/null +++ b/src/lib/clickhouse/formatDateTime.ts @@ -0,0 +1,10 @@ +export function formatDateForClickHouse(date: Date): string { + const year = date.getFullYear() + const month = String(date.getMonth() + 1).padStart(2, '0') + const day = String(date.getDate()).padStart(2, '0') + const hours = String(date.getHours()).padStart(2, '0') + const minutes = String(date.getMinutes()).padStart(2, '0') + const seconds = String(date.getSeconds()).padStart(2, '0') + + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}` +} diff --git a/src/lib/demo-data/generateDemoEvents.ts b/src/lib/demo-data/generateDemoEvents.ts index abb0d21c..66c1a1bf 100644 --- a/src/lib/demo-data/generateDemoEvents.ts +++ b/src/lib/demo-data/generateDemoEvents.ts @@ -7,6 +7,9 @@ import Prop from '../../entities/prop' import Game from '../../entities/game' import randomDate from '../dates/randomDate' import PlayerAlias from '../../entities/player-alias' +import createClickhouseClient from '../clickhouse/createClient' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' +import { formatDateForClickHouse } from '../clickhouse/formatDateTime' type DemoEvent = { name: string @@ -76,8 +79,33 @@ export function generateEventData(date: Date): Partial { } } +async function getEventCount(clickhouse: NodeClickHouseClient, game: Game, startDate: Date): Promise { + const startDateFormatted = formatDateForClickHouse(startDate) + + const query = ` + SELECT count() as count + FROM events + WHERE game_id = ${game.id} + AND created_at >= '${startDateFormatted}' + ` + + try { + const result = await clickhouse.query({ + query, + format: 'JSONEachRow' + }).then((res) => res.json<{ count: string }>()) + + return Number(result[0].count) + } catch (err) { + console.error('Error fetching event count from ClickHouse:', err) + return 0 + } + +} + export async function generateDemoEvents(req: Request): Promise { const em: EntityManager = req.ctx.em + const clickhouse = createClickhouseClient() const games = await em.getRepository(Game).find({ organisation: { @@ -88,18 +116,11 @@ export async function generateDemoEvents(req: Request): Promise { const startDate = subMonths(new Date(), 1) for (const game of games) { - const events = await em.getRepository(Event).find({ - playerAlias: { - player: { - game - } - }, - createdAt: { - $gte: startDate - } - }) + const eventCount = await getEventCount(clickhouse, game, startDate) + + if (eventCount === 0) { + const eventsToInsert: Event[] = [] - if (events.length === 0) { const prev: { [key: string]: number } = {} const playerAliases = await em.getRepository(PlayerAlias).find({ @@ -122,16 +143,25 @@ export async function generateDemoEvents(req: Request): Promise { prev[demoEvent.name] = numToGenerate for (let i = 0; i < numToGenerate; i++) { - const event = new Event(demoEvent.name, game) - event.setProps(getDemoEventProps(demoEvent)) - event.playerAlias = casual.random_element(playerAliases) - event.createdAt = randomDate(startOfDay(day), endOfDay(day)) - em.persist(event) + eventsToInsert.push(new Event(demoEvent.name, game)) + eventsToInsert.at(-1).setProps(getDemoEventProps(demoEvent)) + eventsToInsert.at(-1).playerAlias = casual.random_element(playerAliases) + eventsToInsert.at(-1).createdAt = randomDate(startOfDay(day), endOfDay(day)) } } } + + await clickhouse.insert({ + table: 'events', + values: eventsToInsert.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) + await clickhouse.insert({ + table: 'event_props', + values: eventsToInsert.flatMap((event) => event.getInsertableProps()), + format: 'JSONEachRow' + }) + clickhouse.close() } } - - await em.flush() } diff --git a/src/services/api/event-api.service.ts b/src/services/api/event-api.service.ts index 5888f226..0b9ba8e1 100644 --- a/src/services/api/event-api.service.ts +++ b/src/services/api/event-api.service.ts @@ -1,10 +1,18 @@ import { EntityManager } from '@mikro-orm/mysql' import { HasPermission, Request, Response, Validate, ValidationCondition, Docs } from 'koa-clay' -import Event from '../../entities/event' import EventAPIPolicy from '../../policies/api/event-api.policy' import APIService from './api-service' import EventAPIDocs from '../../docs/event-api.docs' +import createClickhouseClient from '../../lib/clickhouse/createClient' import Player from '../../entities/player' +import Event from '../../entities/event' +import Prop from '../../entities/prop' + +type EventData = { + name: string + timestamp: number + props?: { key: string, value: string }[] +} export default class EventAPIService extends APIService { @Validate({ @@ -24,43 +32,72 @@ export default class EventAPIService extends APIService { @HasPermission(EventAPIPolicy, 'post') @Docs(EventAPIDocs.post) async post(req: Request): Promise { - const { events } = req.body + const { events: items } = req.body const em: EntityManager = req.ctx.em - const errors = [...new Array(events.length)].map(() => []) - const items: Event[] = [] + const clickhouse = createClickhouseClient() + + const events: EventData[] = [] + const errors = Array.from({ length: items.length }).map(() => []) + + const player: Player = req.ctx.state.player + const playerAlias = player.aliases.getItems().find((alias) => alias.id === req.ctx.state.currentAliasId) - for (let i = 0; i < events.length; i++) { - const item = events[i] + for (let i = 0; i < items.length; i++) { + const item = items[i] for (const key of ['name', 'timestamp']) { - if (!item[key]) errors[i].push(`Event is missing the key: ${key}`) + if (!item[key]) { + errors[i].push(`Event is missing the key: ${key}`) + } } if (errors[i].length === 0) { - const event = new Event(item.name, req.ctx.state.key.game) - event.playerAlias = (req.ctx.state.player as Player).aliases.getItems().find((alias) => alias.id === req.ctx.state.currentAliasId) + const event = new Event(item.name, player.game) + event.playerAlias = playerAlias event.createdAt = new Date(item.timestamp) - if (item.props) { + try { + await clickhouse.insert({ + table: 'events', + values: [event.getInsertableData()], + format: 'JSONEachRow' + }) + } catch (err) { + errors[i].push(`Failed to insert event: ${err.message}`) + continue + } + + if (Array.isArray(item.props)) { + event.setProps(item.props.map((prop) => new Prop(prop.key, prop.value))) + try { - if (!Array.isArray(item.props)) throw new Error('Props must be an array') - event.setProps(item.props) + await clickhouse.insert({ + table: 'event_props', + values: event.getInsertableProps(), + format: 'JSONEachRow' + }) } catch (err) { - errors[i].push(err.message) + errors[i].push(`Failed to insert props': ${err.message}`) } + } else if (item.props) { + errors[i].push('Props must be an array') } - if (errors[i].length === 0) items.push(event) + if (errors[i].length === 0) { + events.push(item) + } } } - await em.persistAndFlush(items) + clickhouse.close() + + await em.flush() return { status: 200, body: { - events: items, + events, errors } } diff --git a/src/services/data-export.service.ts b/src/services/data-export.service.ts index 382f2dab..95c8dcbe 100644 --- a/src/services/data-export.service.ts +++ b/src/services/data-export.service.ts @@ -2,7 +2,7 @@ import { Collection, FilterQuery, MikroORM, EntityManager } from '@mikro-orm/mys import { HasPermission, Routes, Service, Request, Response, Validate, ValidationCondition } from 'koa-clay' import DataExport, { DataExportAvailableEntities, DataExportStatus } from '../entities/data-export' import DataExportPolicy from '../policies/data-export.policy' -import Event from '../entities/event' +import Event, { ClickhouseEvent, createEventFromClickhouse } from '../entities/event' import AdmZip from 'adm-zip' import get from 'lodash.get' import Prop from '../entities/prop' @@ -27,6 +27,7 @@ import PlayerProp from '../entities/player-prop' import { Job, Queue } from 'bullmq' import createEmailQueue from '../lib/queues/createEmailQueue' import { EmailConfig } from '../lib/messaging/sendEmail' +import createClickhouseClient from '../lib/clickhouse/createClient' type PropCollection = Collection @@ -136,15 +137,26 @@ export default class DataExportService extends Service { } private async getEvents(dataExport: DataExport, em: EntityManager, includeDevData: boolean): Promise { - const where: FilterQuery = { game: dataExport.game } + const clickhouse = createClickhouseClient() + + let query = ` + SELECT * + FROM events + WHERE game_id = ${dataExport.game.id} + ` if (!includeDevData) { - where.playerAlias = { - player: devDataPlayerFilter(em) - } + query += 'AND dev_build = false' } - return await em.getRepository(Event).find(where, { populate: ['playerAlias'] }) + const events = await clickhouse.query({ + query, + format: 'JSONEachRow' + }).then((res) => res.json()) + + clickhouse.close() + + return await Promise.all(events.map((data) => createEventFromClickhouse(em, data))) } private async getPlayers(dataExport: DataExport, em: EntityManager, includeDevData: boolean): Promise { diff --git a/src/services/event.service.ts b/src/services/event.service.ts index ffa9a6fc..5cea00f7 100644 --- a/src/services/event.service.ts +++ b/src/services/event.service.ts @@ -1,11 +1,9 @@ -import { FilterQuery, EntityManager } from '@mikro-orm/mysql' import { HasPermission, Service, Request, Response, Validate } from 'koa-clay' -import Event from '../entities/event' import EventPolicy from '../policies/event.policy' -import groupBy from 'lodash.groupby' -import { isSameDay, endOfDay } from 'date-fns' +import { endOfDay } from 'date-fns' import dateValidationSchema from '../lib/dates/dateValidationSchema' -import { devDataPlayerFilter } from '../middlewares/dev-data-middleware' +import { formatDateForClickHouse } from '../lib/clickhouse/formatDateTime' +import createClickhouseClient from '../lib/clickhouse/createClient' type EventData = { name: string @@ -14,6 +12,22 @@ type EventData = { change: number } +type AggregatedClickhouseEvent = { + name: string + date: string + count: string +} + +// events: { +// 'Zone explored': [ +// { name: 'Zone explored', date: 1577836800000, count: 3 }, +// { name: 'Zone explored', date: 1577923200000, count: 1 } +// ], +// 'Loot item': [ +// { name: 'Loot item', date: '1577923200000, count: 2 } +// ] +// } + export default class EventService extends Service { @Validate({ query: dateValidationSchema @@ -21,60 +35,55 @@ export default class EventService extends Service { @HasPermission(EventPolicy, 'index') async index(req: Request): Promise { const { startDate: startDateQuery, endDate: endDateQuery } = req.query - const em: EntityManager = req.ctx.em - const startDate = new Date(startDateQuery) - const endDate = new Date(endDateQuery) + const clickhouse = createClickhouseClient() - const where: FilterQuery = { - game: req.ctx.state.game, - createdAt: { - $gte: startDate, - $lte: endOfDay(endDate) - } - } + const startDate = formatDateForClickHouse(new Date(startDateQuery)) + const endDate = formatDateForClickHouse(endOfDay(new Date(endDateQuery))) + + let query = ` + SELECT + name, + toUnixTimestamp(toStartOfDay(created_at)) * 1000 AS date, + count() AS count + FROM events + WHERE created_at BETWEEN '${startDate}' AND '${endDate}' + AND game_id = ${req.ctx.state.game.id} + ` if (!req.ctx.state.includeDevData) { - where.playerAlias = { - player: devDataPlayerFilter(em) - } + query += 'AND dev_build = false' } - const events = await em.getRepository(Event).find(where) + query += ` + GROUP BY name, date + ORDER BY name, date + ` - // events: { - // 'Zone explored': [ - // { name: 'Zone explored', date: 1577836800000, count: 3 }, - // { name: 'Zone explored', date: 1577923200000, count: 1 } - // ], - // 'Loot item': [ - // { name: 'Loot item', date: '1577836800000, count: 0 } - // { name: 'Loot item', date: '1577923200000, count: 2 } - // ] - // } + const events = await clickhouse.query({ + query, + format: 'JSONEachRow' + }).then((res) => res.json()) - const data = groupBy(events, 'name') - - for (const name in data) { - const processed: EventData[] = [] - - for (let time = startDate.getTime(); time <= endDate.getTime(); time += 86400000 /* 24 hours in ms */) { - const dateFromTime = new Date(time) - - const count = data[name].filter((event: Event) => isSameDay(dateFromTime, event.createdAt)).length - const change = processed.length > 0 ? this.calculateChange(count, processed[processed.length - 1]) : 0 - - processed.push({ - name, - date: time, - count, - change - }) + const data: Record = {} + for (const event of events) { + if (!data[event.name]) { + data[event.name] = [] } - data[name] = processed + const lastEvent = data[event.name].at(-1) + const change = this.calculateChange(Number(event.count), lastEvent) + + data[event.name].push({ + name: event.name, + date: Number(event.date), + count: Number(event.count), + change + }) } + clickhouse.close() + return { status: 200, body: { @@ -84,8 +93,9 @@ export default class EventService extends Service { } } - calculateChange(count: number, lastEvent: EventData): number { - if (lastEvent.count === 0) return 1 + private calculateChange(count: number, lastEvent: EventData | undefined): number { + if ((lastEvent?.count ?? 0) === 0) return count || 1 + return (count - lastEvent.count) / lastEvent.count } } diff --git a/src/services/headline.service.ts b/src/services/headline.service.ts index 38098733..38daa9d4 100644 --- a/src/services/headline.service.ts +++ b/src/services/headline.service.ts @@ -1,11 +1,12 @@ import { FilterQuery, EntityManager } from '@mikro-orm/mysql' -import { endOfDay, isSameDay } from 'date-fns' +import { endOfDay, isSameDay, startOfDay } from 'date-fns' import { Service, Request, Response, Validate, HasPermission, Routes } from 'koa-clay' -import Event from '../entities/event' import Player from '../entities/player' import HeadlinePolicy from '../policies/headline.policy' import dateValidationSchema from '../lib/dates/dateValidationSchema' import { devDataPlayerFilter } from '../middlewares/dev-data-middleware' +import createClickhouseClient from '../lib/clickhouse/createClient' +import { formatDateForClickHouse } from '../lib/clickhouse/formatDateTime' @Routes([ { @@ -39,7 +40,7 @@ export default class HeadlineService extends Service { let where: FilterQuery = { game: req.ctx.state.game, createdAt: { - $gte: new Date(startDate), + $gte: startOfDay(new Date(startDate)), $lte: endOfDay(new Date(endDate)) } } @@ -67,11 +68,11 @@ export default class HeadlineService extends Service { let where: FilterQuery = { game: req.ctx.state.game, lastSeenAt: { - $gte: new Date(startDate), + $gte: startOfDay(new Date(startDate)), $lte: endOfDay(new Date(endDate)) }, createdAt: { - $lt: new Date(startDate) + $lt: startOfDay(new Date(startDate)) } } @@ -93,29 +94,35 @@ export default class HeadlineService extends Service { @Validate({ query: dateValidationSchema }) @HasPermission(HeadlinePolicy, 'index') async events(req: Request): Promise { - const { startDate, endDate } = req.query - const em: EntityManager = req.ctx.em + const { startDate: startDateQuery, endDate: endDateQuery } = req.query - const where: FilterQuery = { - game: req.ctx.state.game, - createdAt: { - $gte: new Date(startDate), - $lte: endOfDay(new Date(endDate)) - } - } + const clickhouse = createClickhouseClient() + + const startDate = formatDateForClickHouse(startOfDay(new Date(startDateQuery))) + const endDate = formatDateForClickHouse(endOfDay(new Date(endDateQuery))) + + let query = ` + SELECT count() AS count + FROM events + WHERE created_at BETWEEN '${startDate}' AND '${endDate}' + AND game_id = ${req.ctx.state.game.id} + ` if (!req.ctx.state.includeDevData) { - where.playerAlias = { - player: devDataPlayerFilter(em) - } + query += 'AND dev_build = false' } - const events = await em.getRepository(Event).find(where) + const result = await clickhouse.query({ + query, + format: 'JSONEachRow' + }).then((res) => res.json<{ count: string }>()) + + clickhouse.close() return { status: 200, body: { - count: events.length + count: Number(result[0].count) } } } @@ -123,31 +130,35 @@ export default class HeadlineService extends Service { @Validate({ query: dateValidationSchema }) @HasPermission(HeadlinePolicy, 'index') async uniqueEventSubmitters(req: Request): Promise { - const { startDate, endDate } = req.query - const em: EntityManager = req.ctx.em + const { startDate: startDateQuery, endDate: endDateQuery } = req.query + + const clickhouse = createClickhouseClient() - const query = em.qb(Event, 'e') - .join('e.playerAlias', 'pa') - .count('pa.player_id', true) - .where({ - game: req.ctx.state.game.id, - createdAt: { $gte: new Date(startDate), $lte: endOfDay(new Date(endDate)) } - }) + const startDate = formatDateForClickHouse(startOfDay(new Date(startDateQuery))) + const endDate = formatDateForClickHouse(endOfDay(new Date(endDateQuery))) + + let query = ` + SELECT COUNT(DISTINCT player_alias_id) AS uniqueSubmitters + FROM events + WHERE created_at BETWEEN '${startDate}' AND '${endDate}' + AND game_id = ${req.ctx.state.game.id} + ` if (!req.ctx.state.includeDevData) { - query.andWhere({ - playerAlias: { - player: devDataPlayerFilter(em) - } - }) + query += 'AND dev_build = false' } - const result = await query.execute('get') + const result = await clickhouse.query({ + query, + format: 'JSONEachRow' + }).then((res) => res.json<{ uniqueSubmitters: string }>()) + + clickhouse.close() return { status: 200, body: { - count: result.count + count: Number(result[0].uniqueSubmitters) } } } diff --git a/src/services/player.service.ts b/src/services/player.service.ts index 9cd6ebc1..00fab06d 100644 --- a/src/services/player.service.ts +++ b/src/services/player.service.ts @@ -4,7 +4,7 @@ import Player from '../entities/player' import PlayerPolicy from '../policies/player.policy' import PlayerAlias, { PlayerAliasService } from '../entities/player-alias' import sanitiseProps from '../lib/props/sanitiseProps' -import Event from '../entities/event' +import { ClickhouseEvent, createEventFromClickhouse } from '../entities/event' import { EntityManager } from '@mikro-orm/mysql' import { QueryOrder } from '@mikro-orm/mysql' import uniqWith from 'lodash.uniqwith' @@ -17,6 +17,7 @@ import PlayerGroup from '../entities/player-group' import GameSave from '../entities/game-save' import { PlayerAuthErrorCode } from '../entities/player-auth' import PlayerAuthActivity from '../entities/player-auth-activity' +import createClickhouseClient from '../lib/clickhouse/createClient' const propsValidation = async (val: unknown): Promise => [ { @@ -249,43 +250,51 @@ export default class PlayerService extends Service { } } + @Validate({ query: ['page'] }) @HasPermission(PlayerPolicy, 'getEvents') async events(req: Request): Promise { const itemsPerPage = 50 - const { search, page } = req.query - const em: EntityManager = req.ctx.em const player: Player = req.ctx.state.player // set in the policy - const query = em.createQueryBuilder(Event, 'e') - .select('e.*') - .orderBy({ createdAt: QueryOrder.DESC }) - .limit(itemsPerPage) - .offset(Number(page) * itemsPerPage) + const em: EntityManager = req.ctx.em + const clickhouse = createClickhouseClient() - if (search) { - query - .where('json_extract(props, \'$[*].value\') like ?', [`%${search}%`]) - .orWhere({ - name: { - $like: `%${search}%` - } - }) - } + const aliases = player.aliases.getItems().map((alias) => alias.id).join(',') - const [events, count] = await query - .andWhere({ - playerAlias: { - player - } - }) - .getResultAndCount() + const searchQuery = search ? `AND (name ILIKE '%${search}%' OR prop_value ILIKE '%${search}%')` : '' + const baseQuery = `FROM events + LEFT JOIN event_props ON events.id = event_props.event_id + WHERE player_alias_id IN (${aliases}) + ${searchQuery}` + + const query = ` + SELECT DISTINCT events.* + ${baseQuery} + ORDER BY created_at DESC + LIMIT ${itemsPerPage} + OFFSET ${Number(page) * itemsPerPage} + ` + + const items = await clickhouse.query({ query, format: 'JSONEachRow' }).then((res) => res.json()) + const events = await Promise.all(items.map((item) => createEventFromClickhouse(em, item, true))) + + const countQuery = ` + SELECT count(DISTINCT events.id) AS count + ${baseQuery}` + + const count = await clickhouse.query({ + query: countQuery, + format: 'JSONEachRow' + }).then((res) => res.json<{ count: string }>()) + + clickhouse.close() return { status: 200, body: { events, - count, + count: Number(count[0].count), itemsPerPage } } diff --git a/tests/fixtures/EventFactory.ts b/tests/fixtures/EventFactory.ts index 40774804..af61344f 100644 --- a/tests/fixtures/EventFactory.ts +++ b/tests/fixtures/EventFactory.ts @@ -8,12 +8,10 @@ import { generateEventData } from '../../src/lib/demo-data/generateDemoEvents' export default class EventFactory extends Factory { private availablePlayers: Player[] - private eventTitles: string[] constructor(availablePlayers: Player[]) { super(Event) this.availablePlayers = availablePlayers - this.eventTitles = ['Zone Explored', 'Item Looted', 'Treasure Discovered', 'Levelled up', 'Potion Used', 'Item Crafted', 'Secret Discovered', 'Item Bought', 'Talked to NPC'] } protected definition(): void { diff --git a/tests/run-tests.sh b/tests/run-tests.sh index ccb5f6b7..9a085589 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -5,7 +5,7 @@ export $(cat envs/.env.test | xargs) if [ -z "$CI" ]; then alias dc="docker compose -f docker-compose.test.yml" else - alias dc="docker compose -f docker-compose.ci.yml" + alias dc="docker compose -f docker-compose.test.yml -f docker-compose.ci.yml" fi trap "cleanup" EXIT diff --git a/tests/services/_public/demo/post.test.ts b/tests/services/_public/demo/post.test.ts index 1572b60c..022ac72f 100644 --- a/tests/services/_public/demo/post.test.ts +++ b/tests/services/_public/demo/post.test.ts @@ -1,6 +1,5 @@ import { EntityManager } from '@mikro-orm/mysql' import request from 'supertest' -import Event from '../../../../src/entities/event' import Organisation from '../../../../src/entities/organisation' import OrganisationFactory from '../../../fixtures/OrganisationFactory' import User, { UserType } from '../../../../src/entities/user' @@ -9,6 +8,8 @@ import PlayerFactory from '../../../fixtures/PlayerFactory' import GameFactory from '../../../fixtures/GameFactory' import { sub } from 'date-fns' import randomDate from '../../../../src/lib/dates/randomDate' +import { formatDateForClickHouse } from '../../../../src/lib/clickhouse/formatDateTime' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' describe('Demo service - post', () => { let demoOrg: Organisation @@ -36,30 +37,37 @@ describe('Demo service - post', () => { it('should insert events if there arent any for the last month', async () => { const game = await new GameFactory(demoOrg).one() const players = await new PlayerFactory([game]).many(2) + await (global.em).persistAndFlush(players) - let eventsThisMonth = await (global.em).getRepository(Event).find({ - createdAt: { - $gte: sub(new Date(), { months: 1 }) - } - }) + const date = formatDateForClickHouse(sub(new Date(), { months: 1 })) + + let eventsThisMonth = await (global.clickhouse).query({ + query: `SELECT count() as count FROM events WHERE game_id = ${game.id} AND created_at >= '${date}'`, + format: 'JSONEachRow' + }).then((res) => res.json<{ count: string }>()) + .then((res) => Number(res[0].count)) - expect(eventsThisMonth).toHaveLength(0) + expect(eventsThisMonth).toEqual(0) const randomEvents = await new EventFactory(players).state(() => ({ createdAt: randomDate(sub(new Date(), { years: 1 }), sub(new Date(), { months: 2 })) })).many(20) - await (global.em).persistAndFlush(randomEvents) + await (global.clickhouse).insert({ + table: 'events', + values: randomEvents.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) await request(global.app) .post('/public/demo') .expect(200) - eventsThisMonth = await (global.em).getRepository(Event).find({ - createdAt: { - $gte: sub(new Date(), { months: 1 }) - } - }) + eventsThisMonth = await (global.clickhouse).query({ + query: `SELECT count() as count FROM events WHERE game_id = ${game.id} AND created_at >= '${date}'`, + format: 'JSONEachRow' + }).then((res) => res.json<{ count: string }>()) + .then((res) => Number(res[0].count)) - expect(eventsThisMonth.length).toBeGreaterThan(0) + expect(eventsThisMonth).toBeGreaterThan(0) }) }) diff --git a/tests/services/data-export/included-data.test.ts b/tests/services/data-export/included-data.test.ts index 0a5db5bd..9f07cd33 100644 --- a/tests/services/data-export/included-data.test.ts +++ b/tests/services/data-export/included-data.test.ts @@ -8,6 +8,7 @@ import LeaderboardEntryFactory from '../../fixtures/LeaderboardEntryFactory' import GameStatFactory from '../../fixtures/GameStatFactory' import PlayerGameStatFactory from '../../fixtures/PlayerGameStatFactory' import createOrganisationAndGame from '../../utils/createOrganisationAndGame' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' describe('Data export service - included data', () => { it('should not include events from dev build players without the dev data header', async () => { @@ -19,7 +20,12 @@ describe('Data export service - included data', () => { const player = await new PlayerFactory([game]).devBuild().one() const event = await new EventFactory([player]).one() const dataExport = await new DataExportFactory(game).one() - await (global.em).persistAndFlush([event, dataExport]) + await (global.em).persistAndFlush(dataExport) + await (global.clickhouse).insert({ + table: 'events', + values: [event.getInsertableData()], + format: 'JSONEachRow' + }) const items = await proto.getEvents(dataExport, global.em, false) expect(items).toHaveLength(0) @@ -34,7 +40,12 @@ describe('Data export service - included data', () => { const player = await new PlayerFactory([game]).devBuild().one() const event = await new EventFactory([player]).one() const dataExport = await new DataExportFactory(game).one() - await (global.em).persistAndFlush([event, dataExport]) + await (global.em).persistAndFlush(dataExport) + await (global.clickhouse).insert({ + table: 'events', + values: [event.getInsertableData()], + format: 'JSONEachRow' + }) const items = await proto.getEvents(dataExport, global.em, true) expect(items).toHaveLength(1) diff --git a/tests/services/event/get.test.ts b/tests/services/event/get.test.ts index 7f79b117..8ab08515 100644 --- a/tests/services/event/get.test.ts +++ b/tests/services/event/get.test.ts @@ -1,11 +1,11 @@ import { EntityManager } from '@mikro-orm/mysql' import request from 'supertest' -import Event from '../../../src/entities/event' import EventFactory from '../../fixtures/EventFactory' import PlayerFactory from '../../fixtures/PlayerFactory' -import { sub } from 'date-fns' +import { addDays, sub } from 'date-fns' import createUserAndToken from '../../utils/createUserAndToken' import createOrganisationAndGame from '../../utils/createOrganisationAndGame' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' describe('Event service - get', () => { it('should return a list of events', async () => { @@ -15,14 +15,17 @@ describe('Event service - get', () => { const player = await new PlayerFactory([game]).one() const now = new Date('2021-01-01') - const dayInMs = 86400000 - - const events: Event[] = await new EventFactory([player]).state((event, idx) => ({ + const events = await new EventFactory([player]).state((event, idx) => ({ name: 'Open inventory', - createdAt: new Date(now.getTime() + (dayInMs * idx)) + createdAt: addDays(now, idx) })).many(2) - await (global.em).persistAndFlush(events) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/events`) @@ -30,18 +33,18 @@ describe('Event service - get', () => { .auth(token, { type: 'bearer' }) .expect(200) - expect(res.body.events['Open inventory']).toHaveLength(3) + expect(res.body.events['Open inventory']).toHaveLength(2) expect(res.body.events['Open inventory'][0]).toEqual({ name: 'Open inventory', - date: new Date(now.getTime()).getTime(), + date: now.getTime(), count: 1, - change: 0 + change: 1 }) expect(res.body.events['Open inventory'][1]).toEqual({ name: 'Open inventory', - date: new Date(now.getTime() + dayInMs).getTime(), + date: addDays(now, 1).getTime(), count: 1, change: 0 }) @@ -139,35 +142,38 @@ describe('Event service - get', () => { const player = await new PlayerFactory([game]).one() const now = new Date('2021-01-01') - const dayInMs = 86400000 - const eventFactory = new EventFactory([player]) - const firstEvent: Event = await eventFactory.state(() => ({ + const firstEvent = await eventFactory.state(() => ({ name: 'Open inventory', createdAt: now })).one() - const moreEvents: Event[] = await eventFactory.state(() => ({ + const moreEvents = await eventFactory.state(() => ({ name: 'Open inventory', - createdAt: new Date(now.getTime() + dayInMs) + createdAt: addDays(now, 1) })).many(2) - const evenMoreEvents: Event[] = await eventFactory.state(() => ({ + const evenMoreEvents = await eventFactory.state(() => ({ name: 'Open inventory', - createdAt: new Date(now.getTime() + dayInMs * 2) + createdAt: addDays(now, 2) })).many(3) - const lastEvent: Event = await eventFactory.state(() => ({ + const lastEvent = await eventFactory.state(() => ({ name: 'Open inventory', - createdAt: new Date(now.getTime() + dayInMs * 3) + createdAt: addDays(now, 3) })).one() - await (global.em).persistAndFlush([ - firstEvent, - ...moreEvents, - ...evenMoreEvents, - lastEvent - ]) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: [ + firstEvent, + ...moreEvents, + ...evenMoreEvents, + lastEvent + ].map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/events`) @@ -175,7 +181,7 @@ describe('Event service - get', () => { .auth(token, { type: 'bearer' }) .expect(200) - expect(res.body.events['Open inventory'][0].change).toBe(0) + expect(res.body.events['Open inventory'][0].change).toBe(1) expect(res.body.events['Open inventory'][1].change).toBe(1) expect(res.body.events['Open inventory'][2].change).toBe(0.5) expect(res.body.events['Open inventory'][3].change.toFixed(2)).toBe('-0.67') @@ -188,16 +194,19 @@ describe('Event service - get', () => { const player = await new PlayerFactory([game]).one() const now = new Date('2021-01-01') - const dayInMs = 86400000 - const eventFactory = new EventFactory([player]) - const event: Event = await eventFactory.state(() => ({ + const event = await eventFactory.state((_, idx) => ({ name: 'Join guild', - createdAt: new Date(now.getTime() + dayInMs) + createdAt: addDays(now, idx) })).one() - await (global.em).persistAndFlush(event) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: [event.getInsertableData()], + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/events`) @@ -205,8 +214,7 @@ describe('Event service - get', () => { .auth(token, { type: 'bearer' }) .expect(200) - expect(res.body.events['Join guild'][0].change).toBe(0) - expect(res.body.events['Join guild'][1].change).toBe(1) + expect(res.body.events['Join guild'][0].change).toBe(1) }) it('should not return a list of events for a non-existent game', async () => { @@ -238,7 +246,12 @@ describe('Event service - get', () => { const player = await new PlayerFactory([game]).one() const events = await new EventFactory([player]).state(() => ({ name: 'Talk to NPC', createdAt: new Date() })).many(3) - await (global.em).persistAndFlush(events) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/events`) @@ -246,7 +259,7 @@ describe('Event service - get', () => { .auth(token, { type: 'bearer' }) .expect(200) - expect(res.body.events['Talk to NPC'][1].count).toBe(events.length) + expect(res.body.events['Talk to NPC'][0].count).toBe(events.length) }) it('should not return events by dev build players if the dev data header is not set', async () => { @@ -255,7 +268,12 @@ describe('Event service - get', () => { const player = await new PlayerFactory([game]).devBuild().one() const events = await new EventFactory([player]).state(() => ({ name: 'Talk to NPC', createdAt: new Date() })).many(3) - await (global.em).persistAndFlush(events) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/events`) @@ -272,7 +290,12 @@ describe('Event service - get', () => { const player = await new PlayerFactory([game]).devBuild().one() const events = await new EventFactory([player]).state(() => ({ name: 'Talk to NPC', createdAt: new Date() })).many(3) - await (global.em).persistAndFlush(events) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/events`) @@ -281,6 +304,6 @@ describe('Event service - get', () => { .set('x-talo-include-dev-data', '1') .expect(200) - expect(res.body.events['Talk to NPC'][1].count).toBe(events.length) + expect(res.body.events['Talk to NPC'][0].count).toBe(events.length) }) }) diff --git a/tests/services/headline/get.test.ts b/tests/services/headline/get.test.ts index 1a644162..a88f8484 100644 --- a/tests/services/headline/get.test.ts +++ b/tests/services/headline/get.test.ts @@ -1,11 +1,13 @@ -import { EntityManager } from '@mikro-orm/mysql' +import { Collection, EntityManager } from '@mikro-orm/mysql' import request from 'supertest' -import Event from '../../../src/entities/event' import EventFactory from '../../fixtures/EventFactory' import PlayerFactory from '../../fixtures/PlayerFactory' import { sub, format } from 'date-fns' import createOrganisationAndGame from '../../utils/createOrganisationAndGame' import createUserAndToken from '../../utils/createUserAndToken' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' +import PlayerAlias from '../../../src/entities/player-alias' +import PlayerAliasFactory from '../../fixtures/PlayerAliasFactory' describe('Headline service - get', () => { const startDate = format(sub(new Date(), { days: 7 }), 'yyyy-MM-dd') @@ -16,8 +18,13 @@ describe('Headline service - get', () => { const [token] = await createUserAndToken({}, organisation) const player = await new PlayerFactory([game]).one() - const events: Event[] = await new EventFactory([player]).thisWeek().many(10) - await (global.em).persistAndFlush(events) + const events = await new EventFactory([player]).thisWeek().many(10) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/headlines/events`) @@ -33,8 +40,13 @@ describe('Headline service - get', () => { const [token] = await createUserAndToken({}, organisation) const player = await new PlayerFactory([game]).devBuild().one() - const events: Event[] = await new EventFactory([player]).thisWeek().many(10) - await (global.em).persistAndFlush(events) + const events = await new EventFactory([player]).thisWeek().many(10) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/headlines/events`) @@ -50,8 +62,13 @@ describe('Headline service - get', () => { const [token] = await createUserAndToken({}, organisation) const player = await new PlayerFactory([game]).devBuild().one() - const events: Event[] = await new EventFactory([player]).thisWeek().many(10) - await (global.em).persistAndFlush(events) + const events = await new EventFactory([player]).thisWeek().many(10) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/headlines/events`) @@ -69,7 +86,6 @@ describe('Headline service - get', () => { const newPlayers = await new PlayerFactory([game]).createdThisWeek().many(10) const oldPlayers = await new PlayerFactory([game]).notCreatedThisWeek().many(10) - await (global.em).persistAndFlush([...newPlayers, ...oldPlayers]) const res = await request(global.app) @@ -86,7 +102,6 @@ describe('Headline service - get', () => { const [token] = await createUserAndToken({}, organisation) const newPlayers = await new PlayerFactory([game]).createdThisWeek().devBuild().many(10) - await (global.em).persistAndFlush(newPlayers) const res = await request(global.app) @@ -103,7 +118,6 @@ describe('Headline service - get', () => { const [token] = await createUserAndToken({}, organisation) const newPlayers = await new PlayerFactory([game]).createdThisWeek().devBuild().many(10) - await (global.em).persistAndFlush(newPlayers) const res = await request(global.app) @@ -128,7 +142,6 @@ describe('Headline service - get', () => { .many(4) const playersSignedupThisWeek = await new PlayerFactory([game]).notSeenThisWeek().many(5) - await (global.em).persistAndFlush([...playersNotSeenThisWeek, ...returningPlayersSeenThisWeek, ...playersSignedupThisWeek]) const res = await request(global.app) @@ -187,18 +200,29 @@ describe('Headline service - get', () => { const [organisation, game] = await createOrganisationAndGame() const [token] = await createUserAndToken({}, organisation) - const players = await new PlayerFactory([game]).many(4) + const players = await new PlayerFactory([game]).state(async (player) => { + const alias = await new PlayerAliasFactory(player).one() + return { + aliases: new Collection(player, [alias]) + } + }).many(4) + const validEvents = await new EventFactory([players[0]]).many(3) const validEventsButNotThisWeek = await new EventFactory([players[1]]).state(() => ({ createdAt: sub(new Date(), { weeks: 2 }) })).many(3) const moreValidEvents = await new EventFactory([players[2]]).many(3) - await (global.em).persistAndFlush([ - ...validEvents, - ...validEventsButNotThisWeek, - ...moreValidEvents - ]) + await (global.em).persistAndFlush(players) + await (global.clickhouse).insert({ + table: 'events', + values: [ + ...validEvents, + ...validEventsButNotThisWeek, + ...moreValidEvents + ].map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/headlines/unique_event_submitters`) @@ -215,8 +239,12 @@ describe('Headline service - get', () => { const player = await new PlayerFactory([game]).devBuild().one() const validEvents = await new EventFactory([player]).many(3) - - await (global.em).persistAndFlush(validEvents) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: validEvents.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/headlines/unique_event_submitters`) @@ -233,8 +261,12 @@ describe('Headline service - get', () => { const player = await new PlayerFactory([game]).devBuild().one() const validEvents = await new EventFactory([player]).many(3) - - await (global.em).persistAndFlush(validEvents) + await (global.em).persistAndFlush(player) + await (global.clickhouse).insert({ + table: 'events', + values: validEvents.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/headlines/unique_event_submitters`) diff --git a/tests/services/player/events.test.ts b/tests/services/player/events.test.ts index 4d55e5c1..943fa50c 100644 --- a/tests/services/player/events.test.ts +++ b/tests/services/player/events.test.ts @@ -4,6 +4,7 @@ import PlayerFactory from '../../fixtures/PlayerFactory' import EventFactory from '../../fixtures/EventFactory' import createOrganisationAndGame from '../../utils/createOrganisationAndGame' import createUserAndToken from '../../utils/createUserAndToken' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' describe('Player service - get events', () => { it('should get a player\'s events', async () => { @@ -11,12 +12,18 @@ describe('Player service - get events', () => { const [token] = await createUserAndToken({}, organisation) const player = await new PlayerFactory([game]).one() - const events = await new EventFactory([player]).many(3) + await (global.em).persistAndFlush(player) - await (global.em).persistAndFlush([player, ...events]) + const events = await new EventFactory([player]).many(3) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/players/${player.id}/events`) + .query({ page: 0 }) .auth(token, { type: 'bearer' }) .expect(200) @@ -28,11 +35,11 @@ describe('Player service - get events', () => { const [token] = await createUserAndToken() const player = await new PlayerFactory([game]).one() - await (global.em).persistAndFlush(player) await request(global.app) .get(`/games/${game.id}/players/${player.id}/events`) + .query({ page: 0 }) .auth(token, { type: 'bearer' }) .expect(403) }) @@ -42,13 +49,19 @@ describe('Player service - get events', () => { const [token] = await createUserAndToken({}, organisation) const player = await new PlayerFactory([game]).one() + await (global.em).persistAndFlush(player) + const events = await new EventFactory([player]).state(() => ({ name: 'Find secret' })).many(3) const otherEvents = await new EventFactory([player]).state(() => ({ name: 'Kill boss' })).many(3) - await (global.em).persistAndFlush([...events, ...otherEvents]) + await (global.clickhouse).insert({ + table: 'events', + values: [...events, ...otherEvents].map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/players/${player.id}/events`) - .query({ search: 'Find secret' }) + .query({ search: 'Find secret', page: 0 }) .auth(token, { type: 'bearer' }) .expect(200) @@ -62,14 +75,18 @@ describe('Player service - get events', () => { const count = 82 const player = await new PlayerFactory([game]).one() - const events = await new EventFactory([player]).many(count) - await (global.em).persistAndFlush(events) + await (global.em).persistAndFlush(player) - const page = Math.floor(count / 50) + const events = await new EventFactory([player]).many(count) + await (global.clickhouse).insert({ + table: 'events', + values: events.map((event) => event.getInsertableData()), + format: 'JSONEachRow' + }) const res = await request(global.app) .get(`/games/${game.id}/players/${player.id}/events`) - .query({ page }) + .query({ page: 1 }) .auth(token, { type: 'bearer' }) .expect(200) @@ -84,6 +101,7 @@ describe('Player service - get events', () => { const res = await request(global.app) .get(`/games/${game.id}/players/21312321321/events`) + .query({ page: 0 }) .auth(token, { type: 'bearer' }) .expect(404) diff --git a/tests/setupTest.ts b/tests/setupTest.ts index e063c0a4..c66a37de 100644 --- a/tests/setupTest.ts +++ b/tests/setupTest.ts @@ -1,6 +1,8 @@ import { EntityManager, MikroORM } from '@mikro-orm/mysql' import init from '../src' import ormConfig from '../src/config/mikro-orm.config' +import createClickhouseClient from '../src/lib/clickhouse/createClient' +import { NodeClickHouseClient } from '@clickhouse/client/dist/client' beforeAll(async () => { vi.mock('@sendgrid/mail') @@ -13,10 +15,20 @@ beforeAll(async () => { const app = await init() global.app = app.callback() global.em = app.context.em + + global.clickhouse = createClickhouseClient() + await (global.clickhouse as NodeClickHouseClient).query({ + query: `TRUNCATE ALL TABLES from ${process.env.CLICKHOUSE_DB}` + }) }) afterAll(async () => { await (global.em as EntityManager).getConnection().close(true) + + const clickhouse = global.clickhouse as NodeClickHouseClient + clickhouse.close() + delete global.em delete global.app + delete global.clickhouse })