Add a clusters page and clusters endpoints (for backups later)

frontend/src/Layout.tsx

@@ -6,6 +6,7 @@ import Schema from './pages/SchemaStats/SchemaStats'
 import QueryDetail from './pages/SlowQueries/QueryDetail'
 import SchemaTable from './pages/SchemaStats/SchemaTable'
 import Overview from './pages/Overview/Overview'
+import Clusters from './pages/Clusters/Clusters'
 import Errors from './pages/Errors/Errors'
 import { Switch, Route, useHistory } from 'react-router-dom'
 
@@ -14,6 +15,7 @@ import RunningQueries from './pages/RunningQueries/RunningQueries'
 import Logs from './pages/Logs/Logs'
 import {
     ApartmentOutlined,
+    CloudServerOutlined,
     CodeOutlined,
     DashboardOutlined,
     HddOutlined,
@@ -36,6 +38,7 @@ type MenuItem = Required<MenuProps>['items'][number]
 
 const items: MenuItem[] = [
     { key: '', icon: <HomeOutlined />, label: 'Overview' },
+    { key: 'clusters', label: 'Clusters', icon: <CloudServerOutlined /> },
     { key: 'query_performance', label: 'Query performance', icon: <ClockCircleOutlined /> },
     { key: 'running_queries', label: 'Running queries', icon: <DashboardOutlined /> },
     { key: 'schema', label: 'Schema stats', icon: <HddOutlined /> },
@@ -98,6 +101,7 @@ export default function AppLayout(): JSX.Element {
                     <Content style={{ margin: 'auto', display: 'block', width: '85%', marginTop: 20 }}>
                         <Switch>
                             <Route exact path="/" component={Overview}></Route>
+                            <Route exact path="/clusters" component={Clusters}></Route>
                             <Route exact path="/disk_usage">
                                 <DiskUsage />
                             </Route>

frontend/src/pages/Clusters/Clusters.tsx

@@ -0,0 +1,53 @@
+import React, { useEffect, useState } from 'react'
+import { Line } from '@ant-design/charts'
+import { Card, Col, Row, Tooltip, notification } from 'antd'
+import { InfoCircleOutlined } from '@ant-design/icons'
+
+interface Cluster {
+    cluster: string
+}
+
+interface Clusters {
+    clusters: Cluster[]
+}
+
+export default function Clusters() {
+    const [clusters, setClusters] = useState<Clusters>({
+        clusters: [],
+    })
+
+    const loadData = async () => {
+        try {
+            const res = await fetch('/api/clusters')
+            const resJson = await res.json()
+            const clusters = { clusters: resJson }
+            setClusters(clusters)
+        } catch (err) {
+            notification.error({ message: 'Failed to load data' })
+        }
+    }
+
+    useEffect(() => {
+        loadData()
+    }, [])
+
+    const now = new Date()
+    const dayOfTheYear = Math.floor(
+        (now.getTime() - new Date(now.getFullYear(), 0, 0).getTime()) / (1000 * 60 * 60 * 24)
+    )
+
+    return (
+        <div>
+            <h1 style={{ textAlign: 'left' }}>Clusters</h1>
+            <br />
+            <Row gutter={8} style={{ paddingBottom: 8 }}>
+                <ul>
+                    {clusters.clusters.map((cluster) => (
+                        <li key={cluster.cluster}>{cluster.cluster}</li>
+                    ))}
+                </ul>
+            </Row>
+            <br />
+        </div>
+    )
+}

housewatch/api/analyze.py

@@ -268,7 +268,6 @@ def tables(self, request: Request):
 
     @action(detail=False, methods=["POST"])
     def natural_language_query(self, request: Request):
-
         table_schema_sql_conditions = []
         for full_table_name in request.data["tables_to_query"]:
             database, table = full_table_name.split(">>>>>")

housewatch/api/cluster.py

@@ -0,0 +1,17 @@
+import structlog
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.viewsets import GenericViewSet
+from housewatch.clickhouse import clusters
+
+
+logger = structlog.get_logger(__name__)
+
+
+class ClusterViewset(GenericViewSet):
+    def list(self, request: Request) -> Response:
+        return Response(clusters.get_clusters())
+
+    def retrieve(self, request: Request, pk: str) -> Response:
+        return Response(clusters.get_cluster(pk))

housewatch/api/instance.py

@@ -1,7 +1,11 @@
 import structlog
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
 from rest_framework.viewsets import ModelViewSet
 from rest_framework.serializers import ModelSerializer
 from housewatch.models import Instance
+from housewatch.clickhouse import clusters
 
 
 logger = structlog.get_logger(__name__)

housewatch/celery.py

@@ -17,10 +17,6 @@
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
 
-# Make sure Redis doesn't add too many connections
-# https://stackoverflow.com/questions/47106592/redis-connections-not-being-released-after-celery-task-is-complete
-app.conf.broker_pool_limit = 0
-
 app.steps["worker"].add(DjangoStructLogInitStep)
 
 

housewatch/clickhouse/clusters.py

@@ -0,0 +1,11 @@
+from housewatch.clickhouse.client import run_query
+
+
+def get_clusters():
+    QUERY = """Select cluster FROM system.clusters GROUP BY cluster"""
+    return run_query(QUERY)
+
+
+def get_cluster(cluster):
+    QUERY = """Select * FROM system.clusters WHERE cluster = '%(cluster_name)s' """
+    return run_query(QUERY, {"cluster_name": cluster})

housewatch/urls.py

@@ -3,6 +3,7 @@
 from django.urls import path
 from rest_framework_extensions.routers import ExtendedDefaultRouter
 from housewatch.api.instance import InstanceViewset
+from housewatch.api.cluster import ClusterViewset
 from housewatch.api.analyze import AnalyzeViewset
 from housewatch.api.async_migration import AsyncMigrationsViewset
 from housewatch.views import healthz
@@ -19,6 +20,7 @@ def __init__(self, *args, **kwargs):
 
 router = DefaultRouterPlusPlus()
 router.register(r"api/instance", InstanceViewset, basename="instance")
+router.register(r"api/clusters", ClusterViewset, basename="cluster")
 router.register(r"api/analyze", AnalyzeViewset, basename="analyze")
 router.register(r"api/async_migrations", AsyncMigrationsViewset, basename="async_migrations")
 router.register(r"api/saved_queries", SavedQueryViewset, basename="saved_queries")