Custom Metrics Stackdriver Adapter Module

modules/custom-metrics-stackdriver-adapter/README.md

@@ -0,0 +1,65 @@
+# Custom Metrics Stackdriver Adapter
+
+Adapted from https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/production/adapter_new_resource_model.yaml
+
+## Usage
+
+To use this module, include it from your terraform main:
+
+```
+module "custom_metrics_stackdriver_adapter" {
+  source = "./path/to/custom-metrics-stackdriver-adapter"
+}
+```
+
+For a workload identity enabled cluster, some additional configuration is
+needed:
+
+```
+module "custom_metrics_stackdriver_adapter" {
+  source = "./path/to/custom-metrics-stackdriver-adapter"
+  workload_identity = {
+    enabled = true
+    project_id = "<PROJECT_ID>"
+  }
+}
+```
+
+## Bash equivalent of this module
+
+Assure the following are set before running:
+   - PROJECT_ID: Your GKE project ID
+   - WORKLOAD_IDENTITY: Is workload identity federation enabled in the target cluster?
+
+```
+if [ -z "$WORKLOAD_IDENTITY" ]; then
+    WORKLOAD_IDENTITY=false
+fi
+
+kubectl create namespace custom-metrics
+kubectl create serviceaccount custom-metrics-stackdriver-adapter -n custom-metrics
+
+# If workload identity is enabled, extra steps are required. We need to:
+# - create a service account
+# - grant it the monitoring.viewer IAM role
+# - bind it to the workload identity user for the CMSA
+# - annotate the CMSA service account (done above)
+if [ "$WORKLOAD_IDENTITY" == "true" ]; then
+    gcloud iam service-accounts create cmsa-sa
+    gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:cmsa-sa@$PROJECT_ID.iam.gserviceaccount.com" --role=roles/monitoring.viewer
+    gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:cmsa-sa@$PROJECT_ID.iam.gserviceaccount.com" --role=roles/iam.serviceAccountTokenCreator
+    gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:$PROJECT_ID.svc.id.goog[custom-metrics/custom-metrics-stackdriver-adapter]" "cmsa-sa@$PROJECT_ID.iam.gserviceaccount.com"
+    kubectl annotate serviceaccount custom-metrics-stackdriver-adapter -n custom-metrics "iam.gke.io/gcp-service-account"="cmsa-sa@tpu-vm-gke-testing.iam.gserviceaccount.com"
+fi
+
+kubectl apply -f clusterrolebinding_custom-metrics:system:auth-delegator.yaml.tftpl
+kubectl apply -f rolebinding_custom-metrics-auth-reader.yaml.tftpl
+kubectl apply -f clusterrole_custom-metrics-resource-reader.yaml.tftpl
+kubectl apply -f clusterrolebinding_custom-metrics-resource-reader.yaml.tftpl
+kubectl apply -f deployment_custom-metrics-stackdriver-adapter.yaml.tftpl
+kubectl apply -f service_custom-metrics-stackdriver-adapter.yaml.tftpl
+kubectl apply -f apiservice_v1beta1.custom.metrics.k8s.io.yaml.tftpl
+kubectl apply -f apiservice_v1beta2.custom.metrics.k8s.io.yaml.tftpl
+kubectl apply -f apiservice_v1beta1.external.metrics.k8s.io.yaml.tftpl
+kubectl apply -f clusterrolebinding_external-metrics-reader.yaml.tftpl
+```

modules/custom-metrics-stackdriver-adapter/main.tf

@@ -0,0 +1,130 @@
+locals {
+  v1beta1-custom-metrics-k8s-io                       = "${path.module}/templates/apiservice_v1beta1.custom.metrics.k8s.io.yaml.tftpl"
+  v1beta1-external-metrics-k8s-io                     = "${path.module}/templates/apiservice_v1beta1.external.metrics.k8s.io.yaml.tftpl"
+  v1beta2-custom-metrics-k8s-io                       = "${path.module}/templates/apiservice_v1beta2.custom.metrics.k8s.io.yaml.tftpl"
+  cluster-role-custom-metrics-resource-reader         = "${path.module}/templates/clusterrole_custom-metrics-resource-reader.yaml.tftpl"
+  cluster-role-binding-custom-metrics-resource-reader = "${path.module}/templates/clusterrolebinding_custom-metrics-resource-reader.yaml.tftpl"
+  custom-metrics-system-auth-delegator                = "${path.module}/templates/clusterrolebinding_custom-metrics:system:auth-delegator.yaml.tftpl"
+  external-metrics-reader                             = "${path.module}/templates/clusterrolebinding_external-metrics-reader.yaml.tftpl"
+  deployment-custom-metrics-stackdriver-adapter       = "${path.module}/templates/deployment_custom-metrics-stackdriver-adapter.yaml.tftpl"
+  service-custom-metrics-stackdriver-adapter          = "${path.module}/templates/service_custom-metrics-stackdriver-adapter.yaml.tftpl"
+  service-account-custom-metrics-stackdriver-adapter  = "${path.module}/templates/serviceaccount_custom-metrics-stackdriver-adapter.yaml.tftpl"
+  custom-metrics-auth-reader                          = "${path.module}/templates/rolebinding_custom-metrics-auth-reader.yaml.tftpl"
+}
+
+resource "kubernetes_namespace_v1" "custom-metrics" {
+  metadata {
+    name = "custom-metrics"
+  }
+}
+
+resource "kubernetes_service_account_v1" "custom-metrics-stackdriver-adapter" {
+  count = 1
+  metadata {
+    name      = "custom-metrics-stackdriver-adapter"
+    namespace = kubernetes_namespace_v1.custom-metrics.metadata[0].name
+    annotations = var.workload_identity.enabled ? {
+      "iam.gke.io/gcp-service-account" = google_service_account.cmsa-sa[0].email
+    } : {}
+  }
+}
+
+resource "kubernetes_manifest" "custom-metrics-system-auth-delegator" {
+  count    = 1
+  manifest = yamldecode(file(local.custom-metrics-system-auth-delegator))
+}
+
+resource "kubernetes_manifest" "custom-metrics-auth-reader" {
+  count    = 1
+  manifest = yamldecode(file(local.custom-metrics-auth-reader))
+}
+
+resource "kubernetes_manifest" "cluster-role-custom-metrics-resource-reader" {
+  count    = 1
+  manifest = yamldecode(file(local.cluster-role-custom-metrics-resource-reader))
+}
+
+resource "kubernetes_manifest" "cluster-role-binding-custom-metrics-resource-reader" {
+  count    = 1
+  manifest = yamldecode(file(local.cluster-role-binding-custom-metrics-resource-reader))
+}
+
+resource "kubernetes_manifest" "deployment-custom-metrics-stackdriver-adapter" {
+  count    = 1
+  manifest = yamldecode(file(local.deployment-custom-metrics-stackdriver-adapter))
+}
+
+resource "kubernetes_manifest" "service-custom-metrics-stackdriver-adapter" {
+  count    = 1
+  manifest = yamldecode(file(local.service-custom-metrics-stackdriver-adapter))
+}
+
+resource "kubernetes_manifest" "v1beta1-custom-metrics-k8s-io" {
+  count    = 1
+  manifest = yamldecode(file(local.v1beta1-custom-metrics-k8s-io))
+}
+
+resource "kubernetes_manifest" "v1beta2-custom-metrics-k8s-io" {
+  count    = 1
+  manifest = yamldecode(file(local.v1beta2-custom-metrics-k8s-io))
+}
+
+resource "kubernetes_manifest" "v1beta1-external-metrics-k8s-io" {
+  count    = 1
+  manifest = yamldecode(file(local.v1beta1-external-metrics-k8s-io))
+}
+
+resource "kubernetes_manifest" "external-metrics-reader" {
+  count    = 1
+  manifest = yamldecode(file(local.external-metrics-reader))
+}
+
+# If workload identity is enabled, extra steps are required. We need to:
+# - create a service account
+# - grant it the monitoring.viewer IAM role
+# - bind it to the workload identity user for the cmsa
+# - annotate the cmsa service account (done above)
+
+resource "google_service_account" "cmsa-sa" {
+  count      = var.workload_identity.enabled ? 1 : 0
+  account_id = "cmsa-sa"
+  project    = var.workload_identity.project_id
+}
+
+# Equivalent to:
+#   gcloud projects add-iam-policy-binding PROJECT_ID \
+#       --member=serviceAccount:cmsa-sa@PROJECT_ID.iam.gserviceaccount.com \
+#       --role=roles/monitoring.viewer
+resource "google_project_iam_binding" "cmsa-project-binding-sa-monitoring-viewer" {
+  count   = var.workload_identity.enabled ? 1 : 0
+  project = var.workload_identity.project_id
+  role    = "roles/monitoring.viewer"
+  members = [
+    "serviceAccount:${google_service_account.cmsa-sa[0].account_id}@${var.workload_identity.project_id}.iam.gserviceaccount.com"
+  ]
+}
+
+# Equivalent to:
+#   gcloud projects add-iam-policy-binding PROJECT_ID \
+#       --member=serviceAccount:cmsa-sa@PROJECT_ID.iam.gserviceaccount.com \
+#       --role=roles/iam.serviceAccountTokenCreator
+resource "google_project_iam_binding" "cmsa-project-binding-sa-token-creator" {
+  count   = var.workload_identity.enabled ? 1 : 0
+  project = var.workload_identity.project_id
+  role    = "roles/iam.serviceAccountTokenCreator"
+  members = [
+    "serviceAccount:${google_service_account.cmsa-sa[0].account_id}@${var.workload_identity.project_id}.iam.gserviceaccount.com"
+  ]
+}
+
+# Equivalent to:
+#   gcloud iam service-accounts add-iam-policy-binding \
+#       --role roles/iam.workloadIdentityUser \
+#       --member "serviceAccount:PROJECT_ID.svc.id.goog[custom-metrics/custom-metrics-stackdriver-adapter]" \
+#       cmsa-sa@PROJECT_ID.iam.gserviceaccount.com
+resource "google_service_account_iam_member" "cmsa-bind-to-gsa" {
+  count              = var.workload_identity.enabled ? 1 : 0
+  service_account_id = google_service_account.cmsa-sa[0].name
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "serviceAccount:${var.workload_identity.project_id}.svc.id.goog[custom-metrics/custom-metrics-stackdriver-adapter]"
+}

modules/custom-metrics-stackdriver-adapter/templates/apiservice_v1beta1.custom.metrics.k8s.io.yaml.tftpl

@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  insecureSkipTLSVerify: true
+  group: custom.metrics.k8s.io
+  groupPriorityMinimum: 100
+  versionPriority: 100
+  service:
+    name: custom-metrics-stackdriver-adapter
+    namespace: custom-metrics
+  version: v1beta1

modules/custom-metrics-stackdriver-adapter/templates/apiservice_v1beta1.external.metrics.k8s.io.yaml.tftpl

@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.external.metrics.k8s.io
+spec:
+  insecureSkipTLSVerify: true
+  group: external.metrics.k8s.io
+  groupPriorityMinimum: 100
+  versionPriority: 100
+  service:
+    name: custom-metrics-stackdriver-adapter
+    namespace: custom-metrics
+  version: v1beta1

modules/custom-metrics-stackdriver-adapter/templates/apiservice_v1beta2.custom.metrics.k8s.io.yaml.tftpl

@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta2.custom.metrics.k8s.io
+spec:
+  insecureSkipTLSVerify: true
+  group: custom.metrics.k8s.io
+  groupPriorityMinimum: 100
+  versionPriority: 200
+  service:
+    name: custom-metrics-stackdriver-adapter
+    namespace: custom-metrics
+  version: v1beta2

modules/custom-metrics-stackdriver-adapter/templates/clusterrole_custom-metrics-resource-reader.yaml.tftpl

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: custom-metrics-resource-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "pods"
+  - "nodes"
+  - "nodes/stats"
+  verbs:
+  - list
+  - get
+  - watch

modules/custom-metrics-stackdriver-adapter/templates/clusterrolebinding_custom-metrics-resource-reader.yaml.tftpl

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics-resource-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-stackdriver-adapter
+  namespace: custom-metrics

modules/custom-metrics-stackdriver-adapter/templates/clusterrolebinding_custom-metrics:system:auth-delegator.yaml.tftpl

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-stackdriver-adapter
+  namespace: custom-metrics

modules/custom-metrics-stackdriver-adapter/templates/clusterrolebinding_external-metrics-reader.yaml.tftpl

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: external-metrics-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: external-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: horizontal-pod-autoscaler
+  namespace: kube-system

modules/custom-metrics-stackdriver-adapter/templates/deployment_custom-metrics-stackdriver-adapter.yaml.tftpl

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: custom-metrics-stackdriver-adapter
+  namespace: custom-metrics
+  labels:
+    run: custom-metrics-stackdriver-adapter
+    k8s-app: custom-metrics-stackdriver-adapter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: custom-metrics-stackdriver-adapter
+      k8s-app: custom-metrics-stackdriver-adapter
+  template:
+    metadata:
+      labels:
+        run: custom-metrics-stackdriver-adapter
+        k8s-app: custom-metrics-stackdriver-adapter
+        kubernetes.io/cluster-service: "true"
+    spec:
+      serviceAccountName: custom-metrics-stackdriver-adapter
+      containers:
+      - image: gcr.io/gke-release/custom-metrics-stackdriver-adapter:v0.14.2-gke.0
+        imagePullPolicy: Always
+        name: pod-custom-metrics-stackdriver-adapter
+        command:
+        - /adapter
+        - --use-new-resource-model=true
+        - --fallback-for-container-metrics=true
+        resources:
+          limits:
+            cpu: 250m
+            memory: 200Mi
+          requests:
+            cpu: 250m
+            memory: 200Mi

modules/custom-metrics-stackdriver-adapter/templates/rolebinding_custom-metrics-auth-reader.yaml.tftpl

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: custom-metrics-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-stackdriver-adapter
+  namespace: custom-metrics

modules/custom-metrics-stackdriver-adapter/templates/service_custom-metrics-stackdriver-adapter.yaml.tftpl

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    run: custom-metrics-stackdriver-adapter
+    k8s-app: custom-metrics-stackdriver-adapter
+    kubernetes.io/cluster-service: 'true'
+    kubernetes.io/name: Adapter
+  name: custom-metrics-stackdriver-adapter
+  namespace: custom-metrics
+spec:
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: 443
+  selector:
+    run: custom-metrics-stackdriver-adapter
+    k8s-app: custom-metrics-stackdriver-adapter
+  type: ClusterIP

modules/custom-metrics-stackdriver-adapter/variables.tf

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "workload_identity" {
+  type = object({
+    enabled    = bool
+    project_id = optional(string)
+  })
+  default = {
+    enabled = false
+  }
+  validation {
+    condition = (
+      (var.workload_identity.enabled && var.workload_identity.project_id != null)
+      || (!var.workload_identity.enabled)
+    )
+    error_message = "A project_id must be specified if workload_identity_enabled is set."
+  }
+}