observability support

.env

@@ -1,5 +1,5 @@
 DOCKER_REGISTRY=ghcr.io/trifork/
 COMPOSE_PROJECT_NAME=cheetah-infrastructure
 COMPOSE_PATH_SEPARATOR=:
-COMPOSE_FILE=docker-compose/kafka.yaml:docker-compose/os.yaml:docker-compose/oauth.yaml
+COMPOSE_FILE=docker-compose/kafka.yaml:docker-compose/os.yaml:docker-compose/oauth.yaml:docker-compose/observability.yaml
 COMPOSE_PROFILES=opensearch,kafka,oauth,redpanda,schemaregistry,opensearch_dashboard

config/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: "Prometheus"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    editable: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

config/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,50 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+  - name: Prometheus
+    # <string, required> datasource type. Required
+    type: prometheus
+    # <string, required> access mode. direct or proxy. Required
+    access: proxy
+    # <int> org id. will default to orgId 1 if not specified
+    orgId: 1
+    # <string> url
+    url: http://prometheus:9090
+    # <string> database password, if used
+    password:
+    # <string> database user, if used
+    user:
+    # <string> database name, if used
+    database:
+    # <bool> enable/disable basic auth
+    basicAuth: false
+    # <string> basic auth username, if used
+    basicAuthUser:
+    # <string> basic auth password, if used
+    basicAuthPassword:
+    # <bool> enable/disable with credentials headers
+    withCredentials:
+    # <bool> mark as default datasource. Max one per org
+    isDefault: true
+    # <map> fields that will be converted to json and stored in json_data
+    jsonData:
+      graphiteVersion: "1.1"
+      tlsAuth: false
+      tlsAuthWithCACert: false
+    # <string> json object of data that will be encrypted.
+    secureJsonData:
+      tlsCACert: "..."
+      tlsClientCert: "..."
+      tlsClientKey: "..."
+    version: 1
+    # <bool> allow users to edit datasources from the UI.
+    editable: true

config/prometheus/prometheus.yml

@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  #- job_name: "prometheus"
+  #  static_configs:
+  #    - targets: ["localhost:9090"]
+  - job_name: app
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["host.docker.internal:1854"]

docker-compose/kafka.yaml

@@ -14,6 +14,8 @@ services:
     mem_limit: 512m
     profiles:
       - kafka
+      - basic
+      - full
 
   kafka:
     image: ${DOCKER_REGISTRY-}cheetah-infrastructure-utils-kafka:strimzi-0.33.0-kafka-3.3.2-trifork-1.4.0
@@ -47,6 +49,8 @@ services:
       retries: 10
     profiles:
       - kafka
+      - basic
+      - full
 
   kafka-setup:
     image: quay.io/strimzi/kafka:0.33.2-kafka-3.3.2
@@ -68,6 +72,7 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: ignored
     profiles:
       - kafka
+      - full
 
   redpanda:
     image: redpandadata/console-unstable:master-f155c44
@@ -95,6 +100,7 @@ services:
         condition: service_completed_successfully
     profiles:
       - redpanda
+      - full
 
   # Kafka Prometheus exporter https://github.com/cloudhut/kminion
   kafka-minion:
@@ -110,6 +116,8 @@ services:
       KAFKA_BROKERS: kafka:19093
     profiles:
       - prometheus
+      - observability
+      - full
 
   schema-registry:
     mem_limit: 256m
@@ -160,6 +168,7 @@ services:
       retries: 10
     profiles:
       - schemaregistry
+      - full
 
 networks:
   default:

docker-compose/oauth.yaml

@@ -15,6 +15,8 @@ services:
       - Simulator__DefaultKafkaRole=*_all
     profiles:
       - oauth
+      - basic
+      - full
 
   # Not enabled by default
   cheetah.oauth.simulator.https:
@@ -40,4 +42,3 @@ services:
 networks:
   default:
     name: "cheetah-infrastructure"
-

docker-compose/observability.yaml

@@ -0,0 +1,42 @@
+version: "3"
+services:
+  prometheus:
+    image: quay.io/prometheus/prometheus:v2.36.2
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+    ports:
+      - 9090:9090
+    #restart: always
+    volumes:
+      - ../config/prometheus:/etc/prometheus/
+    #  - prometheus_data:/prometheus
+    profiles:
+      - prometheus
+      - observability
+      - full
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  grafana:
+    image: docker.io/grafana/grafana:10.0.1
+    #restart: always
+    environment:
+      GF_INSTALL_PLUGINS: "grafana-clock-panel,grafana-simple-json-datasource"
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_USERS_ALLOW_SIGN_UP: false
+      GF_AUTH_ANONYMOUS_ENABLED: true
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+    volumes:
+      #- grafana_data:/var/lib/grafana
+      - ../config/grafana/provisioning/:/etc/grafana/provisioning/
+    ports:
+      - 32769:3000
+    profiles:
+      - grafana
+      - observability
+      - full
+
+networks:
+  default:
+    name: "cheetah-infrastructure"

docker-compose/os.yaml

@@ -45,6 +45,8 @@ services:
       - 9600:9600
     profiles:
       - opensearch
+      - basic
+      - full
     healthcheck:
       test: curl -u admin:admin -s -f opensearch:9200/_cat/health > /dev/null || exit 1
       interval: 30s
@@ -70,6 +72,7 @@ services:
       - ../config/opensearch-configurer/cluster/:/app/cluster/
     profiles:
       - opensearch
+      - full
 
   # login kibanaserver:kibanaserver
   opensearch-dashboards:
@@ -89,6 +92,7 @@ services:
         condition: service_healthy
     profiles:
       - opensearch_dashboard
+      - full
 
 volumes:
   opensearch: