add terraform for utility cluster. Add name override to gke

.test-infra/kafka/strimzi/02-kafka-persistent/base/v0.33.2/kafka-persistent.yaml

@@ -26,7 +26,7 @@ metadata:
   name: beam-testing-cluster
 spec:
   kafka:
-    version: 3.4.0
+    version: 3.6.0
     replicas: 3
     config:
       offsets.topic.replication.factor: 3

.test-infra/kafka/strimzi/02-kafka-persistent/overlays/gke-internal-load-balanced/kustomization.yaml

@@ -18,8 +18,6 @@
 # This overlay patches the base strimzi kafka cluster with a GKE TCP
 # internal load balancer ingress
 
-namespace: strimzi
-
 resources:
 - ../../base/v0.33.2
 patchesStrategicMerge:

.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/README.md

@@ -49,6 +49,9 @@ in [.test-infra/terraform/google-cloud-platform/google-kubernetes-engine](.).
 Edit with your IDE terraform plugin installed and it will autocomplete the
 variable names.
 
+## Name override
+If you need a predictable name (omit the suffix) you can set the `cluster_name_override` in order to set the name. Keep in mind that you can cause conflicts so use with care. Using this option still needs `cluster_name_prefix` but its not use so set it to the same value.
+
 # Step 2. Initialize and apply the terraform module.
 
 ## If you are provisioning in `apache-beam-testing`:

.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/cluster.tf

@@ -25,7 +25,7 @@ resource "random_string" "postfix" {
 resource "google_container_cluster" "default" {
   depends_on          = [google_project_service.required]
   deletion_protection = false
-  name                = "${var.cluster_name_prefix}-${random_string.postfix.result}"
+  name                = coalesce(var.cluster_name_override,"${var.cluster_name_prefix}-${random_string.postfix.result}")
   location            = var.region
   enable_autopilot    = true
   network             = data.google_compute_network.default.id
@@ -34,7 +34,14 @@ resource "google_container_cluster" "default" {
     enable_private_nodes    = true
     enable_private_endpoint = false
   }
-  node_config {
-    service_account = data.google_service_account.default.email
+
+  cluster_autoscaling {
+    auto_provisioning_defaults {
+      service_account = var.service_account_id
+      oauth_scopes = [ 
+      "https://www.googleapis.com/auth/cloud-platform"
+      ]
+
+    }
   }
 }

.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/outputs.tf

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output kubernetes_api_endpoint {
+  value = google_container_cluster.default.endpoint
+}
+
+output cluster_ca_certificate {
+  value = google_container_cluster.default.master_auth[0].cluster_ca_certificate
+  sensitive = true
+}

.test-infra/terraform/google-cloud-platform/google-kubernetes-engine/variables.tf

@@ -31,6 +31,12 @@ variable "cluster_name_prefix" {
   description = "The prefix to assign the provisioned Google Kubernetes Engine (GKE) cluster; a random string is appended to this value"
 }
 
+variable "cluster_name_override" {
+  type = string
+  description = "Use this to override naming and omit the postfix. Leave empty to use prefix-suffix format"
+  default = ""
+}
+
 variable "network" {
   type        = string
   description = "The Google Cloud Virtual Private Cloud (VPC) network name"

.test-infra/terraform/google-cloud-platform/utility-cluster/README.md

@@ -0,0 +1,43 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+# Overview
+
+This module will provision a utility cluster which is used to run infrastructure needed for tests running in beam CI.
+
+The purpose of this cluster is to run the necessary tooling that the tests in CI require. Right now it will run Kafka clusters but in the future plan is to have hadoop, flink and any other utility that the test might need which was historically the purpose of `datastores-io` cluster. Unlike `datastores-io` this cluster will be managed by code with Terraform.
+
+# Deployment
+
+# Step 1. Initialize and apply the terraform module.
+
+Make sure you have set the prefix in terraform block in the provider.tf file. Change it if trying to deploy another cluster.
+
+For initialization make sure you use the `beam-infra-terraform-state`
+```
+terraform init -backend-config="bucket=beam-infra-terraform-state"
+```
+## Step 2. Plan the module. Make sure that terraform is not replacing or deleting resrouces unless expected:
+```
+terraform plan
+```
+## Step 3. Apply the module.
+```
+terraform apply
+```

.test-infra/terraform/google-cloud-platform/utility-cluster/gke.tf

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "gke" {
+  source = "../google-kubernetes-engine"
+  project    = "apache-beam-testing"
+  network    = "default"
+  subnetwork = "default-f91f013bcf8bd369"
+  region     = "us-central1"
+  cluster_name_prefix = "beam-utility"
+  service_account_id = "beam-github-actions@apache-beam-testing.iam.gserviceaccount.com"
+  cluster_name_override = "beam-utility"
+
+}

.test-infra/terraform/google-cloud-platform/utility-cluster/provider.tf

@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+terraform {
+   backend "gcs" {
+    prefix = "beam-utility"
+   }
+
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+      version = "~> 5.22.0"
+    }
+  }
+}
+
+provider "helm" {
+  kubernetes {
+    host                    = "https://${module.gke.kubernetes_api_endpoint}"
+    token                   = data.google_client_config.provider.access_token
+    cluster_ca_certificate  = base64decode(module.gke.cluster_ca_certificate)
+  }
+}
+
+data "google_client_config" "provider" {}

.test-infra/terraform/google-cloud-platform/utility-cluster/strimzi-contoller.tf

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "helm_release" "strimzi-helm-release" {
+  name       = "strimzi"
+  namespace  = "strimzi"
+  create_namespace = true
+  repository = "https://strimzi.io/charts/"
+  chart      = "strimzi-kafka-operator"
+  version = "0.40.0"
+
+  atomic = "true"
+  timeout = 500
+
+  set {
+    name  = "watchAnyNamespace"
+    value = "true"
+  }
+  depends_on = [ module.gke.google_container_cluster ]
+}
+