works

docker-compose.dev.yml

@@ -86,6 +86,8 @@ services:
       - zookeeper
     ports:
       - "9000:9000"
+    volumes:
+      - ./docker/clickhouse-server/config.d:/etc/clickhouse-server/config.d
 
   zookeeper:
     image: zookeeper:3.7.0

docker/clickhouse-server/config.d/config.xml

@@ -0,0 +1,23 @@
+<clickhouse>
+    <!-- Listen wildcard address to allow accepting connections from other containers and host
+    network. -->
+    <listen_host>::</listen_host>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+
+    <!--
+       <logger>
+           <console>1</console>
+       </logger>
+       -->
+    <remote_servers>
+        <housewatch>
+            <shard>
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </housewatch>
+    </remote_servers>
+</clickhouse>

housewatch/clickhouse/backups.py

@@ -1,7 +1,7 @@
 import structlog
 from collections import defaultdict
 from datetime import datetime
-from housewatch.clickhouse.client import run_query
+from housewatch.clickhouse.client import run_query, run_query_on_shards
 from housewatch.models.backup import ScheduledBackup, ScheduledBackupRun
 
 from django.conf import settings
@@ -28,13 +28,26 @@ def get_backup(backup, cluster=None):
         return run_query(QUERY, {"uuid": backup}, use_cache=False)
 
 
-def create_table_backup(database, table, bucket, path, aws_key=None, aws_secret=None):
+def create_table_backup(database, table, bucket, path, cluster=None, aws_key=None, aws_secret=None):
     if aws_key is None or aws_secret is None:
         aws_key = settings.AWS_ACCESS_KEY_ID
         aws_secret = settings.AWS_SECRET_ACCESS_KEY
     QUERY = """BACKUP TABLE %(database)s.%(table)s
     TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
     ASYNC"""
+    if cluster:
+        return run_query_on_shards(
+            QUERY,
+            {
+                "database": database,
+                "table": table,
+                "bucket": bucket,
+                "path": path,
+                "aws_key": aws_key,
+                "aws_secret": aws_secret,
+            },
+            cluster=cluster,
+        )
     return run_query(
         QUERY,
         {
@@ -49,13 +62,26 @@ def create_table_backup(database, table, bucket, path, aws_key=None, aws_secret=
     )
 
 
-def create_database_backup(database, bucket, path, aws_key=None, aws_secret=None):
+def create_database_backup(database, bucket, path, cluster=None, aws_key=None, aws_secret=None):
     if aws_key is None or aws_secret is None:
         aws_key = settings.AWS_ACCESS_KEY_ID
         aws_secret = settings.AWS_SECRET_ACCESS_KEY
     QUERY = """BACKUP DATABASE %(database)s 
                 TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
                 ASYNC"""
+    if cluster:
+        return run_query_on_shards(
+            QUERY,
+            {
+                "database": database,
+                "table": table,
+                "bucket": bucket,
+                "path": path,
+                "aws_key": aws_key,
+                "aws_secret": aws_secret,
+            },
+            cluster=cluster,
+        )
     return run_query(
         QUERY,
         {
@@ -78,6 +104,7 @@ def run_backup(backup_id):
             backup.database,
             backup.bucket,
             path,
+            backup.cluster,
             backup.aws_access_key_id,
             backup.aws_secret_access_key,
         )[0]["id"]
@@ -87,6 +114,7 @@ def run_backup(backup_id):
             backup.table,
             backup.bucket,
             path,
+            backup.cluster,
             backup.aws_access_key_id,
             backup.aws_secret_access_key,
         )[0]["id"]

housewatch/clickhouse/client.py

@@ -31,7 +31,7 @@
 def run_query_on_shards(
     query: str,
     params: Dict[str, str | int] = {},
-    settings: Dict[str, str | int] = {},
+    query_settings: Dict[str, str | int] = {},
     query_id: Optional[str] = None,
     substitute_params: bool = True,
     cluster: Optional[str] = None,
@@ -54,7 +54,7 @@ def run_query_on_shards(
             send_receive_timeout=30,
             password=settings.CLICKHOUSE_PASSWORD,
         )
-        result = client.execute(final_query, settings=settings, with_column_types=True, query_id=query_id)
+        result = client.execute(final_query, settings=query_settings, with_column_types=True, query_id=query_id)
         response = []
         for res in result[0]:
             item = {}

housewatch/clickhouse/clusters.py

@@ -23,7 +23,7 @@ def get_shards(cluster):
     cluster = get_cluster(cluster)
     nodes = defaultdict(list)
     for node in cluster:
-        nodes[node["shard_number"]].append(node)
+        nodes[node["shard_num"]].append(node)
     return nodes