add backup page and api

.env.example

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY
+AWS_DEFAULT_REGION=YOUR_AWS_DEFAULT_REGION

docker-compose.dev.yml

@@ -16,6 +16,9 @@ services:
       CLICKHOUSE_SECURE: false
       CLICKHOUSE_VERIFY: false
       CLICKHOUSE_CA: ""
+      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+      AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
     command:
       - bash
       - -c

docker-compose.yml

@@ -15,6 +15,10 @@ services:
       CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE
       CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY
       CLICKHOUSE_CA: $CLICKHOUSE_CA
+      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+      AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
+
     command:
       - bash
       - -c

frontend/src/pages/Backups/Backups.tsx

@@ -1,44 +1,40 @@
 import React, { useEffect, useState } from 'react'
 import { ColumnType } from 'antd/es/table'
-import { Table, Col, Row, Tooltip, notification } from 'antd'
+import { Table, Button, Col, Row, Tooltip, notification } from 'antd'
 
-interface ClusterNode {
-    cluster: string
-    shard_num: number
-    shard_weight: number
-    replica_num: number
-    host_name: string
-    host_address: string
-    port: number
-    is_local: boolean
-    user: string
-    default_database: string
-    errors_count: number
-    slowdowns_count: number
-    estimated_recovery_time: number
+interface BackupRow {
+    id: string
+    name: string
+    status: string
+    error: string
+    start_time: string
+    end_time: string
+    num_files: number
+    total_size: number
+    num_entries: number
+    uncompressed_size: number
+    compressed_size: number
+    files_read: number
+    bytes_read: number
 }
 
-interface Cluster {
-    cluster: string
-    nodes: ClusterNode[]
-}
-
-interface Clusters {
-    clusters: Cluster[]
+interface Backups {
+    backups: BackupRow[]
 }
 
 export default function Backups() {
-    const [clusters, setClusters] = useState<Clusters>({
-        clusters: [],
+    const [backups, setBackups] = useState<Backups>({
+        backups: [],
     })
-    const [loadingClusters, setLoadingClusters] = useState(false)
+    const [loadingBackups, setLoadingBackups] = useState(false)
 
     const loadData = async () => {
         try {
-            const res = await fetch('/api/clusters')
+            const res = await fetch('/api/backups')
             const resJson = await res.json()
-            const clusters = { clusters: resJson }
-            setClusters(clusters)
+            const backups = { backups: resJson }
+            console.log(backups)
+            setBackups(backups)
         } catch (err) {
             notification.error({ message: 'Failed to load data' })
         }
@@ -48,34 +44,36 @@ export default function Backups() {
         loadData()
     }, [])
 
-    const columns: ColumnType<ClusterNode>[] = [
-        { title: 'Cluster', dataIndex: 'cluster' },
-        { title: 'Shard Number', dataIndex: 'shard_num' },
-        { title: 'Shard Weight', dataIndex: 'shard_weight' },
-        { title: 'Replica Number', dataIndex: 'replica_num' },
-        { title: 'Host Name', dataIndex: 'host_name' },
-        { title: 'Host Address', dataIndex: 'host_address' },
-        { title: 'Port', dataIndex: 'port' },
-        { title: 'Is Local', dataIndex: 'is_local' },
-        { title: 'User', dataIndex: 'user' },
-        { title: 'Default Database', dataIndex: 'default_database' },
-        { title: 'Errors Count', dataIndex: 'errors_count' },
-        { title: 'Slowdowns Count', dataIndex: 'slowdowns_count' },
-        { title: 'Recovery Time', dataIndex: 'estimated_recovery_time' },
+    const columns: ColumnType<BackupRow>[] = [
+        { title: 'UUID', dataIndex: 'id' },
+        { title: 'Name', dataIndex: 'name' },
+        { title: 'Status', dataIndex: 'status' },
+        { title: 'Error', dataIndex: 'error' },
+        { title: 'Start', dataIndex: 'start_time' },
+        { title: 'End', dataIndex: 'end_time' },
+        { title: 'Size', dataIndex: 'total_size' },
+        { title: 'Entries', dataIndex: 'num_entries' },
+        { title: 'Uncompressed Size', dataIndex: 'uncompressed_size' },
+        { title: 'Compressed Size', dataIndex: 'compressed_size' },
+        { title: 'Files Read', dataIndex: 'files_read' },
+        { title: 'Bytes Read', dataIndex: 'bytes_read' },
     ]
 
     return (
         <div>
-            <h1 style={{ textAlign: 'left' }}>Clusters</h1>
+            <h1 style={{ textAlign: 'left' }}>Backups</h1>
+            <br />
+            <Button
+                onClick={() => {
+                    console.log('hi')
+                }}
+            >
+                Create Backup
+            </Button>
             <br />
             <Row gutter={8} style={{ paddingBottom: 8 }}>
                 <ul>
-                    {clusters.clusters.map((cluster) => (
-                        <>
-                            <h1 key={cluster.cluster}>{cluster.cluster}</h1>
-                            <Table columns={columns} dataSource={cluster.nodes} loading={loadingClusters} />
-                        </>
-                    ))}
+                    <Table columns={columns} dataSource={backups.backups} loading={loadingBackups} />
                 </ul>
             </Row>
             <br />

housewatch/api/backups.py

@@ -0,0 +1,35 @@
+import structlog
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.viewsets import GenericViewSet
+from housewatch.clickhouse import backups
+
+
+logger = structlog.get_logger(__name__)
+
+
+class BackupViewset(GenericViewSet):
+    def list(self, request: Request) -> Response:
+        cluster = request.query_params.get("cluster")
+        return Response(backups.get_backups(cluster=cluster))
+
+    def retrieve(self, request: Request, pk: str) -> Response:
+        cluster = request.query_params.get("cluster")
+        return Response(backups.get_backup(pk, cluster=cluster))
+
+    @action(detail=True, methods=["post"])
+    def restore(self, request: Request, pk: str) -> Response:
+        backups.restore_backup(pk)
+        return Response()
+
+    def create(self, request: Request) -> Response:
+        database = request.data.get("database")
+        table = request.data.get("table")
+        bucket = request.data.get("bucket")
+        path = request.data.get("path")
+        if table:
+            res = backups.create_table_backup(database, table, bucket, path)
+        else:
+            res = backups.create_database_backup(database, bucket, path)
+        return Response(res)

housewatch/clickhouse/backups.py

@@ -0,0 +1,65 @@
+from collections import defaultdict
+from housewatch.clickhouse.client import run_query
+
+from django.conf import settings
+
+
+def get_backups(cluster=None):
+    if cluster:
+        QUERY = """SELECT * FROM clusterAllReplicas(%(cluster)s, system.backups)"""
+    else:
+        QUERY = """SELECT * FROM system.backups"""
+    res = run_query(QUERY, {"cluster": cluster})
+    return res
+
+
+def get_backup(backup, cluster=None):
+    if cluster:
+        QUERY = """Select * FROM clusterAllReplicas(%(cluster)s, system.backups) WHERE id = '%(uuid)s' """
+        return run_query(QUERY, {"cluster": cluster, "uuid": backup})
+    else:
+        QUERY = """Select * FROM system.backups WHERE id = '%(uuid)s' """
+        return run_query(QUERY, {"uuid": backup})
+
+
+def create_table_backup(database, table, bucket, path, aws_key=None, aws_secret=None):
+    if aws_key is None or aws_secret is None:
+        aws_key = settings.AWS_ACCESS_KEY_ID
+        aws_secret = settings.AWS_SECRET_ACCESS_KEY
+    QUERY = """BACKUP TABLE %(database)s.%(table)s
+    TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
+    ASYNC"""
+    return run_query(
+        QUERY,
+        {
+            "database": database,
+            "table": table,
+            "bucket": bucket,
+            "path": path,
+            "aws_key": aws_key,
+            "aws_secret": aws_secret,
+        },
+    )
+
+
+def create_database_backup(database, bucket, path, aws_key=None, aws_secret=None):
+    if aws_key is None or aws_secret is None:
+        aws_key = settings.AWS_ACCESS_KEY_ID
+        aws_secret = settings.AWS_SECRET_ACCESS_KEY
+    QUERY = """BACKUP DATABASE %(database)s 
+                TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
+                ASYNC"""
+    return run_query(
+        QUERY,
+        {
+            "database": database,
+            "bucket": bucket,
+            "path": path,
+            "aws_key": aws_key,
+            "aws_secret": aws_secret,
+        },
+    )
+
+
+def restore_backup(backup):
+    pass

housewatch/settings/__init__.py

@@ -253,3 +253,9 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_
 CLICKHOUSE_DATABASE = get_from_env("CLICKHOUSE_DATABASE", "defaul")
 CLICKHOUSE_USER = get_from_env("CLICKHOUSE_USER", "default")
 CLICKHOUSE_PASSWORD = get_from_env("CLICKHOUSE_PASSWORD", "")
+
+
+# AWS settings for Backups
+AWS_ACCESS_KEY_ID = get_from_env("AWS_ACCESS_KEY_ID", "")
+AWS_SECRET_ACCESS_KEY = get_from_env("AWS_SECRET_ACCESS_KEY", "")
+AWS_DEFAULT_REGION = get_from_env("AWS_DEFAULT_REGION", "us-east-1")

housewatch/tests/test_backup_table_fixture.sql

@@ -0,0 +1,9 @@
+CREATE TABLE test_backup (
+    id UUID DEFAULT generateUUIDv4(),
+    name String,
+    timestamp DateTime DEFAULT now()
+) ENGINE = MergeTree()
+ORDER BY id;
+INSERT INTO test_backup (name)
+SELECT substring(toString(rand() * 1000000000), 1, 5) AS random_string
+FROM numbers(100);

housewatch/urls.py

@@ -4,6 +4,7 @@
 from rest_framework_extensions.routers import ExtendedDefaultRouter
 from housewatch.api.instance import InstanceViewset
 from housewatch.api.cluster import ClusterViewset
+from housewatch.api.backups import BackupViewset
 from housewatch.api.analyze import AnalyzeViewset
 from housewatch.api.async_migration import AsyncMigrationsViewset
 from housewatch.views import healthz
@@ -21,6 +22,7 @@ def __init__(self, *args, **kwargs):
 router = DefaultRouterPlusPlus()
 router.register(r"api/instance", InstanceViewset, basename="instance")
 router.register(r"api/clusters", ClusterViewset, basename="cluster")
+router.register(r"api/backups", BackupViewset, basename="backup")
 router.register(r"api/analyze", AnalyzeViewset, basename="analyze")
 router.register(r"api/async_migrations", AsyncMigrationsViewset, basename="async_migrations")
 router.register(r"api/saved_queries", SavedQueryViewset, basename="saved_queries")