From 51f00cfc3affbfbd3fa4685760864fb817771d54 Mon Sep 17 00:00:00 2001
From: Wyatt Alt <wyatt.alt@gmail.com>
Date: Wed, 11 Jan 2023 06:31:00 -0800
Subject: [PATCH] Add autoscalers for inbox-listener and stream service

Adds autoscalers for the inbox-listener and stream service. The inbox
listener autoscaler relies on KEDA. The stream service autoscaler is a
standard HPA.

KEDA has some unfortunate friction with helm. It cannot be added as a
dependency to the chart and installed in a reasonable way, because it
relies on CRDs, which helm will not be responsible for updating or
installing in order.

Based on conversation in https://github.com/kedacore/charts/issues/226,
there are two alternative approaches open to us:

* We can instruct users to install KEDA on their cluster prior to
  installing our deployment. The command for this is,
    helm install keda --version 2.9.1 --namespace keda kedacore/keda --create-namespace

* We can vendor KEDA's CRDs and put them into our chart, under a crd
  directory. This should enable a single-command install, but comes with
  the downsides that,
  1. Helm will not update or manage the keda CRDs for us
  2. If users already have KEDA installed (certainly possible) we will
     encounter a conflict.

  Mainly due to the second item, this PR takes the first approach.
---
 .../inbox-listener-scaledobject.yaml          | 34 +++++++++++++++++++
 .../templates/stream-service-hpa.yaml         | 19 +++++++++++
 charts/primary-site/values.yaml               | 14 ++++++--
 3 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100644 charts/primary-site/templates/inbox-listener-scaledobject.yaml
 create mode 100644 charts/primary-site/templates/stream-service-hpa.yaml

diff --git a/charts/primary-site/templates/inbox-listener-scaledobject.yaml b/charts/primary-site/templates/inbox-listener-scaledobject.yaml
new file mode 100644
index 0000000..0bdace3
--- /dev/null
+++ b/charts/primary-site/templates/inbox-listener-scaledobject.yaml
@@ -0,0 +1,34 @@
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: keda-inbox-listener-auth
+spec:
+  secretTargetRef:
+    - parameter: token
+      name: foxglove-site
+      key: token
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: inbox-listener-scaledobject
+  labels:
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: deployment
+    name: inbox-listener
+    envSourceContainerName: inbox-listener
+  pollingInterval: {{ .Values.inboxListener.deployment.autoscalePollingInterval }}
+  cooldownPeriod: {{ .Values.inboxListener.deployment.autoscaleCooldownPeriod }}
+  minReplicaCount: {{ .Values.inboxListener.deployment.minReplicas }}
+  maxReplicaCount: {{ .Values.inboxListener.deployment.maxReplicas }}
+  triggers:
+    - type: metrics-api
+      metadata:
+        targetValue: "{{ .Values.inboxListener.deployment.targetQueueDepth }}"
+        url: "{{ .Values.globals.foxgloveApiUrl }}/internal/platform/v1/pending-imports-stats"
+        valueLocation: 'unleased'
+        authMode: "apiKey"
+      authenticationRef:
+        name: keda-inbox-listener-auth
diff --git a/charts/primary-site/templates/stream-service-hpa.yaml b/charts/primary-site/templates/stream-service-hpa.yaml
new file mode 100644
index 0000000..86542fc
--- /dev/null
+++ b/charts/primary-site/templates/stream-service-hpa.yaml
@@ -0,0 +1,19 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ .Values.streamService.name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Values.streamService.name }}
+  minReplicas: {{ .Values.streamService.deployment.scaling.minReplicas }}
+  maxReplicas: {{ .Values.streamService.deployment.scaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.streamService.deployment.scaling.targetCPUUtilization }}
+
diff --git a/charts/primary-site/values.yaml b/charts/primary-site/values.yaml
index 602d3d9..2f25fdd 100644
--- a/charts/primary-site/values.yaml
+++ b/charts/primary-site/values.yaml
@@ -1,7 +1,6 @@
 globals:
   siteToken:
   foxgloveApiUrl: https://api.foxglove.dev
-
   ## Supported storageProvider values are: `google_cloud` or `azure`
   ## If `azure` is used, then the `@azure.storageAccountName` and `@azure.serviceUrl` values
   ## are required.
@@ -11,14 +10,20 @@ globals:
   inbox:
     storageProvider: google_cloud
     bucketName: foxglove-inbox
-
   azure:
     storageAccountName: ""
     ## For example: https://<resourcegroup>.blob.core.windows.net
     serviceUrl: ""
 
 inboxListener:
+  name: inbox-listener
   deployment:
+    scaling:
+      autoscalePollingInterval: 10
+      autoscaleCooldownPeriod: 10
+      minReplicas: 1
+      maxReplicas: 10
+      targetQueueDepth: 2
     resources:
       requests:
         cpu: 1000m
@@ -28,7 +33,12 @@ inboxListener:
         memory: 1Gi
 
 streamService:
+  name: stream-service
   deployment:
+    scaling:
+      minReplicas: 1
+      maxReplicas: 3
+      targetCPUUtilization: 80
     resources:
       requests:
         cpu: 1000m