Show example of using loki for logging

Store logs for the RQD, Cuebot and DB pod's in loki and make
the datasource available in grafana.

sandbox/README.md

@@ -7,3 +7,18 @@ machine.
 
 To learn how to run the sandbox environment, see
 https://www.opencue.io/docs/quick-starts/.
+
+## Monitoring
+
+To get started with monitoring there is also a additional docker-compose which sets up 
+monitoring for key services.
+
+This can be started from the OpenCue root directory with:
+```bash
+docker-compose --project-directory . -f sandbox/docker-compose.yml -f sandbox/docker-compose.monitoring.yml up
+```
+
+## Loki logging
+
+Too use loki to store logs requires installing the docker drivers. see:
+https://grafana.com/docs/loki/latest/clients/docker-driver/

sandbox/config/datasource.yml

@@ -0,0 +1,16 @@
+# config file version
+apiVersion: 1
+
+deleteDatasources:
+  - name: loki
+
+datasources:
+- name: loki
+  type: loki
+  access: proxy
+  orgId: 1
+  url: http://loki:3100
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: false

sandbox/config/loki.yaml

@@ -0,0 +1,54 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+schema_config:
+  configs:
+  - from: 2020-01-20
+    store: boltdb
+    object_store: filesystem
+    schema: v9
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: /tmp/loki/index
+
+  filesystem:
+    directory: /tmp/loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0
+
+table_manager:
+  chunk_tables_provisioning:
+    inactive_read_throughput: 0
+    inactive_write_throughput: 0
+    provisioned_read_throughput: 0
+    provisioned_write_throughput: 0
+  index_tables_provisioning:
+    inactive_read_throughput: 0
+    inactive_write_throughput: 0
+    provisioned_read_throughput: 0
+    provisioned_write_throughput: 0
+  retention_deletes_enabled: false
+  retention_period: 0

sandbox/docker-compose.monitoring.yml

@@ -0,0 +1,53 @@
+version: '3'
+
+services:
+  db:
+    logging:
+      driver: loki
+      options:
+        loki-url: http://localhost:3100/loki/api/v1/push
+        loki-external-labels: job=db
+
+  cuebot:
+    logging:
+      driver: loki
+      options:
+        loki-url: http://localhost:3100/loki/api/v1/push
+        loki-external-labels: job=cuebot
+
+  rqd:
+    logging:
+      driver: loki
+      options:
+        loki-url: http://localhost:3100/loki/api/v1/push
+        loki-external-labels: job=rqd
+
+  metrics:
+    build:
+      context: ./
+      dockerfile: ./connectors/prometheus_metrics/Dockerfile
+    environment:
+      - CUEBOT_HOSTS=cuebot
+    depends_on:
+      - cuebot
+    links:
+      - cuebot
+    ports:
+      - "8302:8302"
+
+  grafana:
+    image: grafana/grafana:7.1.1
+    volumes:
+    - ./sandbox/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+    ports:
+    - "3000:3000"
+
+  loki:
+   image: grafana/loki:v1.3.0
+   volumes:
+     - ./sandbox/config/loki.yaml:/etc/config/loki.yaml
+   entrypoint:
+     - /usr/bin/loki
+     - -config.file=/etc/config/loki.yaml
+   ports:
+     - "3100:3100"

sandbox/docker-compose.yml

@@ -11,6 +11,11 @@ services:
       - "5432:5432"
     volumes:
       - ./sandbox/db-data:/var/lib/postgresql/data
+    logging:
+      driver: loki
+      options:
+        loki-url: http://localhost:3100/loki/api/v1/push
+        loki-external-labels: job=db
 
   flyway:
     build:
@@ -42,6 +47,11 @@ services:
     environment:
       - CUE_FRAME_LOG_DIR=$CUE_FRAME_LOG_DIR
     command: --datasource.cue-data-source.jdbc-url=jdbc:postgresql://db/cuebot --datasource.cue-data-source.username=cuebot --datasource.cue-data-source.password=$POSTGRES_PASSWORD
+    logging:
+      driver: loki
+      options:
+        loki-url: http://localhost:3100/loki/api/v1/push
+        loki-external-labels: job=cuebot
 
   rqd:
     image: opencue/rqd
@@ -56,16 +66,8 @@ services:
     volumes:
       - /tmp/rqd/logs:/tmp/rqd/logs
       - /tmp/rqd/shots:/tmp/rqd/shots
-
-  metrics:
-    build:
-      context: ./
-      dockerfile: ./connectors/prometheus_metrics/Dockerfile
-    environment:
-      - CUEBOT_HOSTS=cuebot
-    depends_on:
-      - cuebot
-    links:
-      - cuebot
-    ports:
-      - "8302:8302"
+    logging:
+      driver: loki
+      options:
+        loki-url: http://localhost:3100/loki/api/v1/push
+        loki-external-labels: job=rqd