backup views (#23)

* Add Backup view

* add backup page and api

* prettier

* form

* backups working quite well

* touch up

.env.example

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY
+AWS_DEFAULT_REGION=YOUR_AWS_DEFAULT_REGION

docker-compose.dev.yml

@@ -16,6 +16,9 @@ services:
       CLICKHOUSE_SECURE: false
       CLICKHOUSE_VERIFY: false
       CLICKHOUSE_CA: ""
+      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+      AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
     command:
       - bash
       - -c

docker-compose.yml

@@ -15,6 +15,10 @@ services:
       CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE
       CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY
       CLICKHOUSE_CA: $CLICKHOUSE_CA
+      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+      AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
+
     command:
       - bash
       - -c

frontend/src/Layout.tsx

@@ -7,6 +7,7 @@ import QueryDetail from './pages/SlowQueries/QueryDetail'
 import SchemaTable from './pages/SchemaStats/SchemaTable'
 import Overview from './pages/Overview/Overview'
 import Clusters from './pages/Clusters/Clusters'
+import Backups from './pages/Backups/Backups'
 import Errors from './pages/Errors/Errors'
 import { Switch, Route, useHistory } from 'react-router-dom'
 
@@ -26,6 +27,7 @@ import {
     BarsOutlined,
     FormOutlined,
     ToolOutlined,
+    SaveOutlined,
 } from '@ant-design/icons'
 import { ConfigProvider, MenuProps } from 'antd'
 import { Layout, Menu } from 'antd'
@@ -39,6 +41,7 @@ type MenuItem = Required<MenuProps>['items'][number]
 const items: MenuItem[] = [
     { key: '', icon: <HomeOutlined />, label: 'Overview' },
     { key: 'clusters', label: 'Clusters', icon: <CloudServerOutlined /> },
+    { key: 'backups', label: 'Backups', icon: <SaveOutlined /> },
     { key: 'query_performance', label: 'Query performance', icon: <ClockCircleOutlined /> },
     { key: 'running_queries', label: 'Running queries', icon: <DashboardOutlined /> },
     { key: 'schema', label: 'Schema stats', icon: <HddOutlined /> },
@@ -102,6 +105,7 @@ export default function AppLayout(): JSX.Element {
                         <Switch>
                             <Route exact path="/" component={Overview}></Route>
                             <Route exact path="/clusters" component={Clusters}></Route>
+                            <Route exact path="/backups" component={Backups}></Route>
                             <Route exact path="/disk_usage">
                                 <DiskUsage />
                             </Route>

frontend/src/pages/Backups/Backups.tsx

@@ -0,0 +1,196 @@
+import React, { useEffect, useState } from 'react'
+import { usePollingEffect } from '../../utils/usePollingEffect'
+import { ColumnType } from 'antd/es/table'
+import { Table, Button, Form, Input, Modal, Tag, Col, Progress, Row, Tooltip, notification } from 'antd'
+
+interface BackupRow {
+    id: string
+    name: string
+    status: string
+    error: string
+    start_time: string
+    end_time: string
+    num_files: number
+    total_size: number
+    num_entries: number
+    uncompressed_size: number
+    compressed_size: number
+    files_read: number
+    bytes_read: number
+}
+
+interface Backups {
+    backups: BackupRow[]
+}
+
+type FieldType = {
+    database?: string
+    table?: string
+    bucket?: string
+    path?: string
+}
+
+export default function Backups() {
+    const [backups, setBackups] = useState<Backups>({
+        backups: [],
+    })
+    const [loadingBackups, setLoadingBackups] = useState(false)
+    const [open, setOpen] = useState(false)
+    const [confirmLoading, setConfirmLoading] = useState(false)
+
+    const [form] = Form.useForm() // Hook to get form API
+
+    const handleSubmit = async () => {
+        try {
+            // Validate and get form values
+            const values = await form.validateFields()
+            setConfirmLoading(true)
+            const res = await fetch(`/api/backups`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(values),
+            })
+            setOpen(false)
+            setConfirmLoading(false)
+            loadData()
+            return await res.json()
+        } catch (error) {
+            notification.error({
+                message: 'Creating backup failed',
+            })
+        }
+    }
+
+    const showModal = () => {
+        setOpen(true)
+    }
+    const handleCancel = () => {
+        console.log('Clicked cancel button')
+        setOpen(false)
+    }
+
+    const loadData = async () => {
+        try {
+            const res = await fetch('/api/backups')
+            const resJson = await res.json()
+            const backups = { backups: resJson }
+            console.log(backups)
+            setBackups(backups)
+        } catch (err) {
+            notification.error({ message: 'Failed to load data' })
+        }
+    }
+
+    useEffect(() => {
+        loadData()
+    }, [])
+
+    const columns: ColumnType<BackupRow>[] = [
+        { title: 'UUID', dataIndex: 'id' },
+        { title: 'Name', dataIndex: 'name' },
+        {
+            title: 'Status',
+            dataIndex: 'status',
+            render: (_, { status }) => {
+                var color = 'volcano'
+                switch (status) {
+                    case 'CREATING_BACKUP' || 'RESTORING':
+                        color = 'black'
+                        break
+                    case 'BACKUP_CREATED' || 'RESTORED':
+                        color = 'green'
+                        break
+                    case 'BACKUP_FAILED' || 'RESTORE_FAILED':
+                        color = 'volcano'
+                        break
+                }
+                return (
+                    <Tag color={color} key={status}>
+                        {status.toUpperCase()}
+                    </Tag>
+                )
+            },
+        },
+        { title: 'Error', dataIndex: 'error' },
+        { title: 'Start', dataIndex: 'start_time' },
+        { title: 'End', dataIndex: 'end_time' },
+        { title: 'Size', dataIndex: 'total_size' },
+    ]
+
+    usePollingEffect(
+        async () => {
+            loadData()
+        },
+        [],
+        { interval: 5000 }
+    )
+
+    return (
+        <div>
+            <h1 style={{ textAlign: 'left' }}>Backups</h1>
+            <Button onClick={showModal}>Create Backup</Button>
+            <br />
+            <Modal
+                title="Create Backup"
+                open={open}
+                onOk={handleSubmit}
+                confirmLoading={confirmLoading}
+                onCancel={handleCancel}
+            >
+                <Form
+                    name="basic"
+                    form={form}
+                    labelCol={{ span: 8 }}
+                    wrapperCol={{ span: 16 }}
+                    style={{ maxWidth: 600 }}
+                    initialValues={{ remember: true }}
+                    autoComplete="on"
+                >
+                    <Form.Item<FieldType>
+                        label="Database"
+                        name="database"
+                        initialValue="default"
+                        rules={[{ required: true, message: 'Please select a database to back up from' }]}
+                    >
+                        <Input />
+                    </Form.Item>
+
+                    <Form.Item<FieldType>
+                        label="Table"
+                        name="table"
+                        initialValue="test_backup"
+                        rules={[{ required: true, message: 'Please select a table to back up' }]}
+                    >
+                        <Input />
+                    </Form.Item>
+
+                    <Form.Item<FieldType>
+                        label="S3 Bucket"
+                        name="bucket"
+                        initialValue="posthog-clickhouse"
+                        rules={[{ required: true, message: 'What S3 bucket to backup into' }]}
+                    >
+                        <Input />
+                    </Form.Item>
+
+                    <Form.Item<FieldType>
+                        label="S3 Path"
+                        name="path"
+                        initialValue="testing/test_backup/7"
+                        rules={[{ required: true, message: 'What is the path in the bucket to backup to' }]}
+                    >
+                        <Input />
+                    </Form.Item>
+                </Form>
+            </Modal>
+            <Row gutter={8} style={{ paddingBottom: 8 }}>
+                <ul>
+                    <Table columns={columns} dataSource={backups.backups} loading={loadingBackups} />
+                </ul>
+            </Row>
+            <br />
+        </div>
+    )
+}

housewatch/api/backups.py

@@ -0,0 +1,35 @@
+import structlog
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.viewsets import GenericViewSet
+from housewatch.clickhouse import backups
+
+
+logger = structlog.get_logger(__name__)
+
+
+class BackupViewset(GenericViewSet):
+    def list(self, request: Request) -> Response:
+        cluster = request.query_params.get("cluster")
+        return Response(backups.get_backups(cluster=cluster))
+
+    def retrieve(self, request: Request, pk: str) -> Response:
+        cluster = request.query_params.get("cluster")
+        return Response(backups.get_backup(pk, cluster=cluster))
+
+    @action(detail=True, methods=["post"])
+    def restore(self, request: Request, pk: str) -> Response:
+        backups.restore_backup(pk)
+        return Response()
+
+    def create(self, request: Request) -> Response:
+        database = request.data.get("database")
+        table = request.data.get("table")
+        bucket = request.data.get("bucket")
+        path = request.data.get("path")
+        if table:
+            res = backups.create_table_backup(database, table, bucket, path)
+        else:
+            res = backups.create_database_backup(database, bucket, path)
+        return Response(res)

housewatch/clickhouse/backups.py

@@ -0,0 +1,67 @@
+from collections import defaultdict
+from housewatch.clickhouse.client import run_query
+
+from django.conf import settings
+
+
+def get_backups(cluster=None):
+    if cluster:
+        QUERY = """SELECT * FROM clusterAllReplicas(%(cluster)s, system.backups) ORDER BY start_time DESC"""
+    else:
+        QUERY = """SELECT * FROM system.backups ORDER BY start_time DESC"""
+    res = run_query(QUERY, {"cluster": cluster}, use_cache=False)
+    return res
+
+
+def get_backup(backup, cluster=None):
+    if cluster:
+        QUERY = """Select * FROM clusterAllReplicas(%(cluster)s, system.backups) WHERE id = '%(uuid)s' """
+        return run_query(QUERY, {"cluster": cluster, "uuid": backup}, use_cache=False)
+    else:
+        QUERY = """Select * FROM system.backups WHERE id = '%(uuid)s' """
+        return run_query(QUERY, {"uuid": backup}, use_cache=False)
+
+
+def create_table_backup(database, table, bucket, path, aws_key=None, aws_secret=None):
+    if aws_key is None or aws_secret is None:
+        aws_key = settings.AWS_ACCESS_KEY_ID
+        aws_secret = settings.AWS_SECRET_ACCESS_KEY
+    QUERY = """BACKUP TABLE %(database)s.%(table)s
+    TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
+    ASYNC"""
+    return run_query(
+        QUERY,
+        {
+            "database": database,
+            "table": table,
+            "bucket": bucket,
+            "path": path,
+            "aws_key": aws_key,
+            "aws_secret": aws_secret,
+        },
+        use_cache=False,
+    )
+
+
+def create_database_backup(database, bucket, path, aws_key=None, aws_secret=None):
+    if aws_key is None or aws_secret is None:
+        aws_key = settings.AWS_ACCESS_KEY_ID
+        aws_secret = settings.AWS_SECRET_ACCESS_KEY
+    QUERY = """BACKUP DATABASE %(database)s 
+                TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
+                ASYNC"""
+    return run_query(
+        QUERY,
+        {
+            "database": database,
+            "bucket": bucket,
+            "path": path,
+            "aws_key": aws_key,
+            "aws_secret": aws_secret,
+        },
+        use_cache=False,
+    )
+
+
+def restore_backup(backup):
+    pass

housewatch/settings/__init__.py

@@ -253,3 +253,9 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_
 CLICKHOUSE_DATABASE = get_from_env("CLICKHOUSE_DATABASE", "defaul")
 CLICKHOUSE_USER = get_from_env("CLICKHOUSE_USER", "default")
 CLICKHOUSE_PASSWORD = get_from_env("CLICKHOUSE_PASSWORD", "")
+
+
+# AWS settings for Backups
+AWS_ACCESS_KEY_ID = get_from_env("AWS_ACCESS_KEY_ID", "")
+AWS_SECRET_ACCESS_KEY = get_from_env("AWS_SECRET_ACCESS_KEY", "")
+AWS_DEFAULT_REGION = get_from_env("AWS_DEFAULT_REGION", "us-east-1")

housewatch/tests/test_backup_table_fixture.sql

@@ -0,0 +1,9 @@
+CREATE TABLE test_backup (
+    id UUID DEFAULT generateUUIDv4(),
+    name String,
+    timestamp DateTime DEFAULT now()
+) ENGINE = MergeTree()
+ORDER BY id;
+INSERT INTO test_backup (name)
+SELECT substring(toString(rand() * 1000000000), 1, 5) AS random_string
+FROM numbers(100);

housewatch/urls.py

@@ -4,6 +4,7 @@
 from rest_framework_extensions.routers import ExtendedDefaultRouter
 from housewatch.api.instance import InstanceViewset
 from housewatch.api.cluster import ClusterViewset
+from housewatch.api.backups import BackupViewset
 from housewatch.api.analyze import AnalyzeViewset
 from housewatch.api.async_migration import AsyncMigrationsViewset
 from housewatch.views import healthz
@@ -21,6 +22,7 @@ def __init__(self, *args, **kwargs):
 router = DefaultRouterPlusPlus()
 router.register(r"api/instance", InstanceViewset, basename="instance")
 router.register(r"api/clusters", ClusterViewset, basename="cluster")
+router.register(r"api/backups", BackupViewset, basename="backup")
 router.register(r"api/analyze", AnalyzeViewset, basename="analyze")
 router.register(r"api/async_migrations", AsyncMigrationsViewset, basename="async_migrations")
 router.register(r"api/saved_queries", SavedQueryViewset, basename="saved_queries")