migrate events from mysql to clickhouse

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         run: npm ci --prefer-offline
 
       - name: Setup environment
-        run: cp envs/.env.dev .env
+        run: cp envs/.env.test .env
 
       - name: Run tests
         run: npm test -- --coverage

clickhouse/migrations/20240813213400CreateEventsTable.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS gs_ch.events
+(
+    id String,
+    name String,
+    game_id UInt32,
+    player_alias_id UInt32,
+    dev_build Boolean,
+    created_at DateTime,
+    updated_at DateTime,
+    PRIMARY KEY (id),
+    INDEX name_idx (name) TYPE bloom_filter(0.01) GRANULARITY 64,
+    INDEX game_id_idx (game_id) TYPE minmax GRANULARITY 64,
+    INDEX player_alias_id_idx (player_alias_id) TYPE minmax GRANULARITY 64
+)
+ENGINE = MergeTree()
+ORDER BY (id, created_at, game_id, player_alias_id);

clickhouse/migrations/20240815105700CreateEventPropsTable.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS gs_ch.event_props (
+    event_id String,
+    prop_key String,
+    prop_value String
+) ENGINE = MergeTree()
+ORDER BY (event_id, prop_key);

clickhouse/migrations/run-migrations.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+MIGRATIONS_DIR="/migrations"
+
+echo "Starting migrations..."
+
+for migration in $(ls ${MIGRATIONS_DIR}/*.sql | sort); do
+  echo "Applying migration: $migration..."
+
+  if [ -z "$CLICKHOUSE_USER" ] && [ -z "$CLICKHOUSE_PASS" ]; then
+    clickhouse-client --query="$(cat $migration)"
+  else
+    clickhouse-client --user="${CLICKHOUSE_USER}" --password="${CLICKHOUSE_PASS}" --query="$(cat $migration)"
+  fi
+done
+
+echo "Migrations complete!"

docker-compose.ci.yml

@@ -1,45 +1,9 @@
 services:
-  test-db:
-    image: mysql:8.4
-    command: --mysql-native-password=ON
-    environment:
-      - MYSQL_DATABASE=${DB_NAME}
-      - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"]
-      interval: 2s
-      timeout: 2s
-      retries: 10
-    ports:
-      - ${DB_PORT}:3306
-    volumes:
-      - test-data:/var/lib/mysql
-    networks:
-      - test-network
-
   test-redis:
     image: bitnami/redis:7.2
+    command:
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD}
-    ports:
-      - ${REDIS_PORT}:6379
-    depends_on:
-      test-db:
-        condition: service_healthy
-    networks:
-      - test-network
 
   stripe-api:
     image: stripe/stripe-mock:latest
-    ports:
-      - 12111:12111
-      - 12112:12112
-    networks:
-      - test-network
-
-volumes:
-  test-data:
-
-networks:
-  test-network:

docker-compose.dev.yml

@@ -4,10 +4,22 @@ services:
       context: .
       target: dev
     image: backend
+    depends_on:
+      - db
     ports:
       - 3000:80
     volumes:
       - ./src:/usr/backend/src
       - ./tests:/usr/backend/tests
+
+  clickhouse-ui:
+    image: ghcr.io/caioricciuti/ch-ui:latest
     depends_on:
-      - db
+      - clickhouse
+    environment:
+      - VITE_CLICKHOUSE_URL=http://localhost:${CLICKHOUSE_PORT}
+      - VITE_CLICKHOUSE_USER=${CLICKHOUSE_USER}
+      - VITE_CLICKHOUSE_PASS=${CLICKHOUSE_PASS}
+    ports:
+      - 5521:5521
+

docker-compose.test.yml

@@ -29,6 +29,24 @@ services:
     networks:
       - test-network
 
+  test-clickhouse:
+    image: clickhouse/clickhouse-server:24-alpine
+    entrypoint: ["/bin/sh", "-c"]
+    command: |
+      "/entrypoint.sh & \
+      sleep 5 && \
+      /migrations/run-migrations.sh && \
+      wait"
+    environment:
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - ./clickhouse/migrations:/migrations
+    networks:
+      - test-network
+
   stripe-api:
     image: stripe/stripe-mock:latest-arm64
     ports:

docker-compose.yml

@@ -5,14 +5,15 @@ services:
     depends_on:
       - db
       - redis
+      - clickhouse
 
   db:
     image: mysql:8.4
     command: --mysql-native-password=ON
     environment:
       - MYSQL_DATABASE=${DB_NAME}
       - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
+    restart: unless-stopped
     ports:
       - 3306:3306
     volumes:
@@ -24,5 +25,25 @@ services:
     ports:
       - 6379:6379
 
+  clickhouse:
+    image: clickhouse/clickhouse-server:24-alpine
+    entrypoint: ["/bin/sh", "-c"]
+    command: |
+      "/entrypoint.sh & \
+      sleep 5 && \
+      /migrations/run-migrations.sh && \
+      wait"
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASS: ${CLICKHOUSE_PASS}
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+      - ./clickhouse/migrations:/migrations
+
 volumes:
   data:
+  clickhouse-data:

envs/.env.dev

@@ -9,6 +9,12 @@ DB_PASS=password
 
 REDIS_PASSWORD=password
 
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_USER=gs_ch
+CLICKHOUSE_PASS=password
+CLICKHOUSE_DB=gs_ch
+
 DEMO_ORGANISATION_NAME=Talo Demo
 
 SENDGRID_KEY=

envs/.env.test

@@ -3,6 +3,12 @@ DB_PORT=3307
 DB_NAME=gs_test
 DB_PASS=password
 
+CLICKHOUSE_HOST=127.0.0.1
+CLICKHOUSE_PORT=8124
+CLICKHOUSE_USER=
+CLICKHOUSE_PASS=
+CLICKHOUSE_DB=gs_ch
+
 SENDGRID_KEY=
 SENTRY_DSN=
 

package.json

@@ -47,6 +47,7 @@
     "vitest": "^1.5.2"
   },
   "dependencies": {
+    "@clickhouse/client": "^1.4.1",
     "@dinero.js/currencies": "^2.0.0-alpha.14",
     "@koa/cors": "^5.0.0",
     "@mikro-orm/core": "^6.3.2",

src/entities/event.ts

@@ -1,35 +1,41 @@
-import { Entity, Embedded, ManyToOne, PrimaryKey, Property, Cascade } from '@mikro-orm/mysql'
+import { v4 } from 'uuid'
 import sanitiseProps from '../lib/props/sanitiseProps'
 import Game from './game'
 import PlayerAlias from './player-alias'
 import Prop from './prop'
+import { formatDateForClickHouse } from '../lib/clickhouse/formatDateTime'
+import { EntityManager } from '@mikro-orm/mysql'
+import createClickhouseClient from '../lib/clickhouse/createClient'
 
 const eventMetaProps = ['META_OS', 'META_GAME_VERSION', 'META_WINDOW_MODE', 'META_SCREEN_WIDTH', 'META_SCREEN_HEIGHT']
 
-@Entity()
-export default class Event {
-  @PrimaryKey()
-  id: number
-
-  @Property()
+export type ClickhouseEvent = {
+  id: string
   name: string
+  game_id: number
+  player_alias_id: number
+  dev_build: boolean
+  created_at: string
+  updated_at: string
+}
 
-  @Embedded(() => Prop, { array: true })
-  props: Prop[] = []
+export type ClickhouseEventProp = {
+  event_id: string
+  prop_key: string
+  prop_value: string
+}
 
-  @ManyToOne(() => Game)
+export default class Event {
+  id: string
+  name: string
+  props: Prop[] = []
   game: Game
-
-  @ManyToOne(() => PlayerAlias, { cascade: [Cascade.REMOVE] })
   playerAlias: PlayerAlias
-
-  @Property()
-  createdAt: Date = new Date()
-
-  @Property({ onUpdate: () => new Date() })
+  createdAt: Date
   updatedAt: Date = new Date()
 
   constructor(name: string, game: Game) {
+    this.id = v4()
     this.name = name
     this.game = game
   }
@@ -51,6 +57,26 @@ export default class Event {
     })
   }
 
+  getInsertableData(): ClickhouseEvent {
+    return {
+      id: this.id,
+      name: this.name,
+      game_id: this.game.id,
+      player_alias_id: this.playerAlias.id,
+      dev_build: this.playerAlias.player.isDevBuild(),
+      created_at: formatDateForClickHouse(this.createdAt),
+      updated_at: formatDateForClickHouse(this.updatedAt)
+    }
+  }
+
+  getInsertableProps(): ClickhouseEventProp[] {
+    return this.props.map((prop) => ({
+      event_id: this.id,
+      prop_key: prop.key,
+      prop_value: prop.value
+    }))
+  }
+
   toJSON() {
     return {
       id: this.id,
@@ -62,3 +88,29 @@ export default class Event {
     }
   }
 }
+
+export async function createEventFromClickhouse(em: EntityManager, data: ClickhouseEvent, loadProps = false): Promise<Event> {
+  const game = await em.getRepository(Game).findOne(data.game_id)
+  const playerAlias = await em.getRepository(PlayerAlias).findOne(data.player_alias_id, { populate: ['player'] })
+
+  const event = new Event(data.name, game)
+  event.id = data.id
+  event.playerAlias = playerAlias
+  event.createdAt = new Date(data.created_at)
+  event.updatedAt = new Date(data.updated_at)
+
+  if (loadProps) {
+    const clickhouse = createClickhouseClient()
+
+    const props = await clickhouse.query({
+      query: `SELECT * FROM event_props WHERE event_id = '${data.id}'`,
+      format: 'JSONEachRow'
+    }).then((res) => res.json<ClickhouseEventProp>())
+
+    event.props = props.map((prop) => new Prop(prop.prop_key, prop.prop_value))
+
+    clickhouse.close()
+  }
+
+  return event
+}

src/entities/index.ts

@@ -10,7 +10,6 @@ import Leaderboard from './leaderboard'
 import LeaderboardEntry from './leaderboard-entry'
 import APIKey from './api-key'
 import DataExport from './data-export'
-import Event from './event'
 import FailedJob from './failed-job'
 import Game from './game'
 import Organisation from './organisation'
@@ -58,7 +57,6 @@ export default [
   LeaderboardEntry,
   DataExport,
   APIKey,
-  Event,
   Game,
   FailedJob,
   Organisation,

src/entities/prop.ts

@@ -6,7 +6,7 @@ export default class Prop {
   key: string
 
   @Property()
-  value: string|null
+  value: string | null
 
   constructor(key: string, value?: string) {
     this.key = key