feat: docker-compose quick start

docker-compose/README.md

@@ -0,0 +1,33 @@
+# docker-compose for ckibana
+
+## 简介
+
+使用docker-compose部署ckibana及关联服务，便于快速体验。
+
+内置了mock数据的流程：nginx生产日志，filebeat抓取日志并写入clickhouse中。
+
+
+## 使用
+```shell
+# 进入docker-compose目录
+cd ckibana/docker-compose
+
+# 部署
+docker-compose up -d
+
+# 卸载
+docker-compose down
+```
+部署完成后，可在浏览器访问kibana：http://127.0.0.1:5601/
+
+
+## 导入kibana配置文件
+### 导入配置：
+已内置了mock数据的流程，导入kibana配置文件（大盘、index-pattern等），就可以开始体验了！
+
+配置文件路径：docker-compose/quickstart-export.json
+![](image/dashboard-import.jpg)
+
+### 效果展示：
+![](image/dashboard.jpg)
+

docker-compose/config/ckibana-application.yaml

@@ -0,0 +1,14 @@
+spring:
+  application:
+    name: ckibana
+server:
+  port: 8080
+logging:
+  config: classpath:logback-spring.xml
+  file:
+    path: logs
+
+metadata-config:
+  hosts: elasticsearch:9200
+  headers:
+    mock: 'mock'

docker-compose/config/clickhouse/config/00_default_overrides.xml

@@ -0,0 +1,43 @@
+<clickhouse>
+    <!-- Macros -->
+    <macros>
+        <shard from_env="CLICKHOUSE_SHARD_ID"></shard>
+        <layer>ckibana-clickhouse</layer>
+    </macros>
+    <!-- Log Level -->
+    <logger>
+        <level>information</level>
+    </logger>
+    <!-- Cluster configuration - Any update of the shards and replicas requires helm upgrade -->
+    <remote_servers>
+        <default>
+            <shard>
+                <internal_replication>false</internal_replication>
+                <replica>
+                    <host>clickhouse-1</host>
+                    <port>9000</port>
+                    <user from_env="CLICKHOUSE_ADMIN_USER"></user>
+                    <password from_env="CLICKHOUSE_ADMIN_PASSWORD"></password>
+                </replica>
+            </shard>
+            <shard>
+                <internal_replication>false</internal_replication>
+                <replica>
+                    <host>clickhouse-2</host>
+                    <port>9000</port>
+                    <user from_env="CLICKHOUSE_ADMIN_USER"></user>
+                    <password from_env="CLICKHOUSE_ADMIN_PASSWORD"></password>
+                </replica>
+            </shard>
+            <shard>
+                <internal_replication>false</internal_replication>
+                <replica>
+                    <host>clickhouse-3</host>
+                    <port>9000</port>
+                    <user from_env="CLICKHOUSE_ADMIN_USER"></user>
+                    <password from_env="CLICKHOUSE_ADMIN_PASSWORD"></password>
+                </replica>
+            </shard>
+        </default>
+    </remote_servers>
+</clickhouse>

docker-compose/config/clickhouse/custom-init-scripts/init-db.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+clickhouse-client --user=default --password=default --query="create database ops"
+
+clickhouse-client --user=default --password=default << 'EOF'
+CREATE TABLE ops.ops_nginx_local
+(
+`request_method` LowCardinality(String),
+`request_uri` String,
+`status` Int64,
+`request_time` Float64,
+`request_length` Int64,
+`http_referer` String,
+`http_user_agent` String,
+`http_host` String,
+`request_id` String,
+`remote_addr` String,
+`bytes_sent` Int64,
+`body_bytes_sent` Int64,
+`@timestamp` UInt64,
+`ck_assembly_extension` String,
+INDEX timestamp_index `@timestamp` TYPE minmax GRANULARITY 8192
+)
+ENGINE = MergeTree
+PARTITION BY (toYYYYMMDD(toDateTime(`@timestamp` / 1000, 'Asia/Shanghai')), toHour(toDateTime(`@timestamp` / 1000, 'Asia/Shanghai')))
+ORDER BY (`http_host`, intHash64(`@timestamp`))
+SAMPLE BY intHash64(`@timestamp`)
+SETTINGS in_memory_parts_enable_wal = 0, index_granularity = 8192
+EOF
+
+clickhouse-client --user=default --password=default << 'EOF'
+CREATE TABLE IF NOT EXISTS ops.ops_nginx_all AS ops.ops_nginx_local ENGINE Distributed('default', 'ops', 'ops_nginx_local', rand());
+EOF

docker-compose/config/clickhouse/setup.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Execute entrypoint as usual after obtaining KEEPER_SERVER_ID
+# check KEEPER_SERVER_ID in persistent volume via myid
+# if not present, set based on POD hostname
+if [[ -f "/bitnami/clickhouse/keeper/data/myid" ]]; then
+    export KEEPER_SERVER_ID="$(cat /bitnami/clickhouse/keeper/data/myid)"
+else
+    HOSTNAME="$(hostname -s)"
+    if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
+        export KEEPER_SERVER_ID=${BASH_REMATCH[2]}
+    else
+        echo "Failed to get index from hostname $HOST"
+        exit 1
+    fi
+fi
+exec /opt/bitnami/scripts/clickhouse/entrypoint.sh /opt/bitnami/scripts/clickhouse/run.sh -- --listen_host=0.0.0.0

docker-compose/config/mock-curl2nginx.sh

@@ -0,0 +1,17 @@
+HOST=mock-nginx
+DELAY=5
+while true; do
+  random_num=$((RANDOM))
+  curl  "http://${HOST}"
+  curl -X POST  "http://${HOST}/mock-post"
+  if [ $((random_num % 10)) -eq 0 ]; then
+      curl -X DELETE  "http://${HOST}/mock-delete"
+  fi
+  if [ $((random_num % 10)) -eq 0 ]; then
+      curl  "http://${HOST}/mock-500"
+  fi
+  if [ $((random_num % 10)) -eq 0 ]; then
+      curl  "http://${HOST}/400"
+  fi
+  # sleep $DELAY
+done

docker-compose/config/mock-filebeat.yml

@@ -0,0 +1,25 @@
+filebeat.inputs:
+  - type: log
+    paths:
+      - /var/log/nginx/access.log
+    encoding: utf-8
+    json.keys_under_root: true
+    json.add_error_key: true
+output.clickHouse:
+  # clickhouse数据库配置
+  # https://github.com/ClickHouse/clickhouse-go
+  # 示例 tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000
+  url: "tcp://clickhouse-1:9000?username=default&password=default&debug=true"
+  # 接收数据的表名
+  table: ops.ops_nginx_local
+  # 数据过滤器的表列，匹配日志文件中对应的键
+  columns: [ "request_method", "request_uri", "status", "request_time", "request_length", "http_referer", "http_user_agent", "http_host", "request_id", "remote_addr", "bytes_sent", "body_bytes_sent", "@timestamp"]
+  # 异常重试休眠时间 单位：秒
+  retry_interval: 60
+  # 是否跳过异常事件推送 true-表示跳过执行异常实践 false-会一直重试，重试间隔为retry_interval
+  skip_unexpected_type_row: false
+  # 批处理数据量，影响filebeat每次抓取数据行
+  bulk_max_size: 1000
+
+  #output.console:
+  #  pretty: true

docker-compose/config/mock-init.sh

@@ -0,0 +1,20 @@
+URL="http://elasticsearch:9200/_bulk"
+DELAY=5
+while true; do
+    status_code=$( \
+    curl -sL -m 5 -w "%{http_code}"  -o /dev/null -XPOST "$URL"  -H 'Content-Type: application/json' -d'
+{ "index": { "_index" : "proxy-settings", "_type" : "info", "_id": "kibana"}}
+{ "key":"kibana", "value": "  query:\n    sampleIndexPatterns:\n      - ops_nginx_all\n    sampleCountMaxThreshold: 1500000\n    useCache: true\n  proxy:\n    roundAbleMinPeriod: 120000\n    maxTimeRange: 86400000\n    blackIndexList:\n    whiteIndexList:\n      - ops_nginx_all\n    ck:\n      url: clickhouse-1:8123\n      user: default\n      pass: default\n      defaultCkDatabase: ops"}
+' )
+    echo $status_code
+    if [[ "$status_code" == "200" ]]; then
+        echo "Success! Got 200 OK response from $URL"
+        break
+    else
+        echo "Still waiting for 200 OK response from $URL (got $status_code)"
+    fi
+    sleep $DELAY
+done
+while true; do
+    sleep 60
+done

docker-compose/config/mock-nginx.conf

@@ -0,0 +1,66 @@
+user nginx;
+worker_processes auto;
+
+error_log /var/log/nginx/error.log warn;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    map $msec $timestamp {
+        default "0";
+        ~^(?<seconds>\d+)\.(?<milliseconds>\d+)$ $seconds$milliseconds;
+    }
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format json '{"timestamp": $timestamp, "request_method": "$request_method", "request_uri": "$request_uri", "status": $status, "request_time": $request_time,  "request_length": $request_length, "http_referer": "$http_referer", "http_user_agent": "$http_user_agent", "http_host": "$http_host", "request_id": "$request_id", "remote_addr":"$remote_addr", "bytes_sent": $bytes_sent, "body_bytes_sent": $body_bytes_sent}';
+
+    access_log /var/log/nginx/access.log json;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+
+    server_names_hash_bucket_size 64;
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  _;
+        root         /usr/share/nginx/html;
+
+        location / {
+            index   index.html index.htm;
+        }
+
+        location /mock-post {
+            if ($request_method = POST) {
+                return 200 "This is a mock-post response";
+            }
+        }
+
+        location /mock-delete {
+            if ($request_method = DELETE) {
+                return 200 "This is a mock-delete response";
+            }
+        }
+
+        location /mock-500 {
+            return 500 "This is a mock response";
+        }
+
+        error_page  404              /404.html;
+        location = /404.html {
+            internal;
+        }
+
+        error_page   500 502 503 504  /50x.html;
+        location = /50x.html {
+            internal;
+        }
+    }
+}