Add autoscale config

SAFEHR-data · Aug 8, 2023 · 242415e · 242415e
1 parent a7f58d1
commit 242415e
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 0 deletions.
diff --git a/config.sample.yaml b/config.sample.yaml
@@ -62,6 +62,9 @@ transform:  # Optional
       local_disk_min_size: 600
       category: "Memory Optimised"
     autotermination_minutes: 120
+    autoscale:
+      min_workers: 1
+      max_workers: 3
     init_scripts:
       - /workspaces/FlowEHR/transform/sample_init_script.sh
 

diff --git a/infrastructure/transform/databricks.tf b/infrastructure/transform/databricks.tf
@@ -70,6 +70,10 @@ resource "databricks_cluster" "cluster" {
   spark_version           = data.databricks_spark_version.latest.id
   node_type_id            = data.databricks_node_type.node_type.id
   autotermination_minutes = var.transform.databricks_cluster.autotermination_minutes
+  autoscale {
+    min_workers = var.transform.databricks_cluster.autoscale.min_workers
+    max_workers = var.transform.databricks_cluster.autoscale.max_workers
+  }
 
   spark_conf = merge(
     # Secrets for SQL Feature store

diff --git a/infrastructure/transform/variables.tf b/infrastructure/transform/variables.tf
@@ -146,6 +146,10 @@ variable "transform" {
       }), {}),
       autotermination_minutes = optional(number, 0),
       init_scripts            = optional(list(string), [])
+      autoscale = optional(object({
+        min_workers = optional(number, 0)
+        max_workers = optional(number, 0)
+      }), {})
     }), {})
   })
   default = {