migrate events from mysql to clickhouse

TaloDev · Aug 19, 2024 · 976913b · 976913b
1 parent 6e2f5fc
commit 976913b
Show file tree

Hide file tree

Showing 40 changed files with 750 additions and 341 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,6 +30,12 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs
+          path: clickhouse/logs
+
   build-check:
     runs-on: ubuntu-latest
 

diff --git a/clickhouse/Dockerfile b/clickhouse/Dockerfile
@@ -0,0 +1,10 @@
+FROM clickhouse/clickhouse-server:24.6-alpine
+
+RUN apk add --no-cache gettext
+
+COPY /clickhouse/migrations /docker-entrypoint-initdb.d/
+COPY /clickhouse/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/clickhouse/entrypoint.sh b/clickhouse/entrypoint.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+INIT_DIR="/docker-entrypoint-initdb.d"
+FLAG_FILE="$INIT_DIR/.migrated"
+
+if [ ! -f "$FLAG_FILE" ]; then
+  for file in ${INIT_DIR}/*.sql; do
+    if [ -f "$file" ]; then
+      envsubst < "$file" > "$file.processed" && mv "$file.processed" "$file"
+      echo "Processed migration $file"
+    fi
+  done
+
+  echo "Creating migrations lock"
+  touch "$FLAG_FILE"
+else
+  echo "Migrations lock set; Skipping envsubst processing"
+fi
+
+exec /entrypoint.sh "$@"
diff --git a/clickhouse/migrations/001CreateEventsTable.sql b/clickhouse/migrations/001CreateEventsTable.sql
@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.events
+(
+    id String,
+    name String,
+    game_id UInt32,
+    player_alias_id UInt32,
+    dev_build Boolean,
+    created_at DateTime,
+    updated_at DateTime,
+    PRIMARY KEY (id),
+    INDEX name_idx (name) TYPE bloom_filter(0.01) GRANULARITY 64,
+    INDEX game_id_idx (game_id) TYPE minmax GRANULARITY 64,
+    INDEX player_alias_id_idx (player_alias_id) TYPE minmax GRANULARITY 64
+)
+ENGINE = MergeTree()
+ORDER BY (id, created_at, game_id, player_alias_id);
diff --git a/clickhouse/migrations/002CreateEventPropsTable.sql b/clickhouse/migrations/002CreateEventPropsTable.sql
@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.event_props (
+    event_id String,
+    prop_key String,
+    prop_value String
+) ENGINE = MergeTree()
+ORDER BY (event_id, prop_key);
diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
@@ -1,45 +1,9 @@
 services:
-  test-db:
-    image: mysql:8.4
-    command: --mysql-native-password=ON
-    environment:
-      - MYSQL_DATABASE=${DB_NAME}
-      - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"]
-      interval: 2s
-      timeout: 2s
-      retries: 10
-    ports:
-      - ${DB_PORT}:3306
-    volumes:
-      - test-data:/var/lib/mysql
-    networks:
-      - test-network
-
   test-redis:
     image: bitnami/redis:7.2
+    command:
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD}
-    ports:
-      - ${REDIS_PORT}:6379
-    depends_on:
-      test-db:
-        condition: service_healthy
-    networks:
-      - test-network
 
   stripe-api:
     image: stripe/stripe-mock:latest
-    ports:
-      - 12111:12111
-      - 12112:12112
-    networks:
-      - test-network
-
-volumes:
-  test-data:
-
-networks:
-  test-network:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -4,10 +4,11 @@ services:
       context: .
       target: dev
     image: backend
+    depends_on:
+      - db
     ports:
       - 3000:80
     volumes:
       - ./src:/usr/backend/src
       - ./tests:/usr/backend/tests
-    depends_on:
-      - db
+
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -29,6 +29,22 @@ services:
     networks:
       - test-network
 
+  test-clickhouse:
+    build:
+      context: .
+      dockerfile: ./clickhouse/Dockerfile
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - ./clickhouse/logs:/var/log/clickhouse-server
+    networks:
+      - test-network
+
   stripe-api:
     image: stripe/stripe-mock:latest-arm64
     ports:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,14 +5,15 @@ services:
     depends_on:
       - db
       - redis
+      - clickhouse
 
   db:
     image: mysql:8.4
     command: --mysql-native-password=ON
     environment:
       - MYSQL_DATABASE=${DB_NAME}
       - MYSQL_ROOT_PASSWORD=${DB_PASS}
-    restart: always
+    restart: unless-stopped
     ports:
       - 3306:3306
     volumes:
@@ -24,5 +25,20 @@ services:
     ports:
       - 6379:6379
 
+  clickhouse:
+    build:
+      context: .
+      dockerfile: ./clickhouse/Dockerfile
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DB: ${CLICKHOUSE_DB}
+    restart: unless-stopped
+    ports:
+      - ${CLICKHOUSE_PORT}:8123
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+
 volumes:
   data:
+  clickhouse-data:
diff --git a/envs/.dockerignore b/envs/.dockerignore
diff --git a/envs/.env.dev b/envs/.env.dev
@@ -9,6 +9,12 @@ DB_PASS=password
 
 REDIS_PASSWORD=password
 
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_USER=gs_ch
+CLICKHOUSE_PASSWORD=password
+CLICKHOUSE_DB=gs_ch_dev
+
 DEMO_ORGANISATION_NAME=Talo Demo
 
 SENDGRID_KEY=

diff --git a/envs/.env.test b/envs/.env.test
@@ -3,6 +3,12 @@ DB_PORT=3307
 DB_NAME=gs_test
 DB_PASS=password
 
+CLICKHOUSE_HOST=127.0.0.1
+CLICKHOUSE_PORT=8124
+CLICKHOUSE_USER=
+CLICKHOUSE_PASSWORD=
+CLICKHOUSE_DB=gs_ch_test
+
 SENDGRID_KEY=
 SENTRY_DSN=
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -47,6 +47,7 @@
     "vitest": "^1.5.2"
   },
   "dependencies": {
+    "@clickhouse/client": "^1.4.1",
     "@dinero.js/currencies": "^2.0.0-alpha.14",
     "@koa/cors": "^5.0.0",
     "@mikro-orm/core": "^6.3.2",

diff --git a/src/docs/event-api.docs.ts b/src/docs/event-api.docs.ts
@@ -19,19 +19,69 @@ const EventAPIDocs: APIDocs<EventAPIService> = {
           events: [
             { name: 'Levelled up', timestamp: 1657063169020, props: [{ key: 'newLevel', value: '81' }] },
             { name: 'Quest completed', timestamp: 1657063169324, props: [{ key: 'questId', value: '122' }] },
-            { name: 'Quested started', timestamp: 1657063169819, props: [{ key: 'questId', value: '128' }] }
+            { name: 'Quested started', props: [{ key: 'questId', value: '128' }] }
           ]
         }
       },
       {
         title: 'Sample response',
         sample: {
           events: [
-            { name: 'Levelled up', timestamp: 1657063169020, props: [{ key: 'newLevel', value: '81' }] },
-            { name: 'Quest completed', timestamp: 1657063169324, props: [{ key: 'questId', value: '122' }] }
+            {
+              id: '77bdd78a-2292-40c0-8835-7e091adbced5',
+              name: 'Levelled up',
+              props: [{ key: 'newLevel', value: '81' }],
+              playerAlias: {
+                id: 1,
+                service: 'steam',
+                identifier: '11133645',
+                player: {
+                  id: '7a4e70ec-6ee6-418e-923d-b3a45051b7f9',
+                  props: [
+                    { key: 'xPos', value: '13.29' },
+                    { key: 'yPos', value: '26.44' }
+                  ],
+                  aliases: [
+                    '/* [Circular] */'
+                  ],
+                  devBuild: false,
+                  createdAt: '2022-01-15T13:20:32.133Z',
+                  lastSeenAt: '2022-07-05T12:09:43.124Z'
+                }
+              },
+              gameId: 1,
+              createdAt: '2022-07-05T12:26:09.020Z'
+            },
+            {
+              id: 'd58b8a63-525b-4994-b04c-602485976245',
+              name: 'Quest completed',
+              props: [{ key: 'questId', value: '122' }],
+              playerAlias: {
+                id: 1,
+                service: 'steam',
+                identifier: '11133645',
+                player: {
+                  id: '7a4e70ec-6ee6-418e-923d-b3a45051b7f9',
+                  props: [
+                    { key: 'xPos', value: '13.29' },
+                    { key: 'yPos', value: '26.44' }
+                  ],
+                  aliases: [
+                    '/* [Circular] */'
+                  ],
+                  devBuild: false,
+                  createdAt: '2022-01-15T13:20:32.133Z',
+                  lastSeenAt: '2022-07-05T12:09:43.124Z'
+                }
+              },
+              gameId: 1,
+              createdAt: '2022-07-05T12:26:09.324Z'
+            }
           ],
           errors: [
-            'Event is missing the key: timestamp'
+            [],
+            [],
+            ['Event is missing the key: timestamp']
           ]
         }
       }

diff --git a/src/docs/game-feedback-api.docs.ts b/src/docs/game-feedback-api.docs.ts
@@ -14,7 +14,7 @@ const GameFeedbackAPIDocs: APIDocs<GameFeedbackAPIService> = {
               internalName: 'bugs',
               name: 'Bugs',
               description: 'Issues that hinder the player experience',
-              createdAt: '2024-06-01 21:36:02'
+              createdAt: '2024-06-01T21:36:02.008Z'
             }
           ]
         }
@@ -52,12 +52,12 @@ const GameFeedbackAPIDocs: APIDocs<GameFeedbackAPIService> = {
               name: 'Bugs',
               description: 'Issues that hinder the player experience',
               anonymised: false,
-              createdAt: '2024-06-01 21:36:02'
+              createdAt: '2024-06-01T21:36:02.008Z'
             },
             comment: 'The new quest in the new zone cannot be completed!',
             anonymised: true,
             playerAlias: null,
-            createdAt: '2024-06-03 18:04:44'
+            createdAt: '2024-06-03T18:04:44.336Z'
           }
         }
       },
@@ -72,7 +72,7 @@ const GameFeedbackAPIDocs: APIDocs<GameFeedbackAPIService> = {
               name: 'Bugs',
               description: 'Issues that hinder the player experience',
               anonymised: false,
-              createdAt: '2024-06-01 21:36:02'
+              createdAt: '2024-06-01T21:36:02.008Z'
             },
             comment: 'The new quest in the new zone cannot be completed!',
             anonymised: false,
@@ -90,11 +90,11 @@ const GameFeedbackAPIDocs: APIDocs<GameFeedbackAPIService> = {
                   '/* [Circular] */'
                 ],
                 devBuild: false,
-                createdAt: '2022-01-15 13:20:32',
-                lastSeenAt: '2022-04-12 15:09:43'
+                createdAt: '2022-01-15T13:20:32.133Z',
+                lastSeenAt: '2022-04-12T15:09:43.066Z'
               }
             },
-            createdAt: '2024-06-03 18:04:44'
+            createdAt: '2024-06-03T18:04:44.336Z'
           }
         }
       }

diff --git a/src/docs/game-save-api.docs.ts b/src/docs/game-save-api.docs.ts
@@ -45,7 +45,7 @@ const sampleSave = {
       }
     ]
   },
-  updatedAt: '2022-06-07 14:58:13'
+  updatedAt: '2022-06-07T14:58:13.771Z'
 }
 
 const GameSaveAPIDocs: APIDocs<GameSaveAPIService> = {