Use OpenTofu to create GCP infrastructure

.editorconfig

@@ -8,6 +8,6 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = tab
 
-[*.{md,yml,yaml}]
+[*.{md,yml,yaml,tf}]
 indent_size = 2
 indent_style = space

.github/workflows/nfr.yml

@@ -87,43 +87,35 @@ jobs:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           install_components: kubectl
 
+
+      - name: Setup OpenTofu
+        uses: opentofu/setup-opentofu@592200bd4b9bbf4772ace78f887668b1aee8f716 # v1.0.5
+        with:
+          tofu_version: 1.8.7 # renovate: datasource=github-tags depName=opentofu/opentofu
+
       - name: Setup dotenv file
         working-directory: ./tests/scripts
         run: |
-          echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
           echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env
           echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
           echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
           echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
-          echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
-          echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env
-          echo "GKE_CLUSTER_REGION=us-west1" >> vars.env
-          echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
-          echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
-          echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
-          echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
-          echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
-          echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
-          echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
           echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env
           echo "GINKGO_LABEL=" >> vars.env
           echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env
-          echo "GKE_NUM_NODES=12" >> vars.env
-          echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
-
-      - name: Setup license file for plus
-        if: matrix.type == 'plus'
-        env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
-        run: echo "${PLUS_LICENSE}" > license.jwt
 
-      - name: Create GKE cluster
-        working-directory: ./tests
-        run: make create-gke-cluster CI=true
-
-      - name: Create and setup VM
-        working-directory: ./tests
-        run: make create-and-setup-vm
+      - name: Create GKE cluster and VM
+        working-directory: ./tests/tofu
+        run: |
+          tofu apply --auto-approve \
+          -var="project_id=${{ secrets.GCP_PROJECT_ID }}" \
+          -var="gke_cluster_name=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" \
+          -var="vm_service_account=${{ secrets.GCP_SERVICE_ACCOUNT }}"
+          -var="gke_nodes_service_account=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" \
+          -var="gke_num_nodes=12" \
+          -var="gke_machine_type=n2d-standard-16" \
+          -var="gke_project=${{ secrets.GCP_PROJECT_ID }}" \
+          -var="ngf_branch=${{ github.ref_name }}"
 
       - name: Run Tests
         working-directory: ./tests

.gitignore

@@ -61,3 +61,32 @@ internal/mode/static/nginx/modules/coverage
 # temporary files used for manifest generation
 config/base/deploy.yaml
 config/base/deploy.yaml.bak
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

.pre-commit-config.yaml

@@ -95,6 +95,12 @@ repos:
           - "--skip-auto-generation=required,additionalProperties"
           - --append-newline
 
+  - repo: https://github.com/tofuutils/pre-commit-opentofu
+    rev: v2.1.0
+    hooks:
+      - id: tofu_fmt
+      # - id: tofu_docs
+
 ci:
   skip: [golangci-lint-full, prettier, markdownlint-cli2, yamllint]
   autofix_prs: false

tests/Makefile

@@ -90,19 +90,11 @@ ifeq ($(PLUS_ENABLED),true)
 endif
 
 .PHONY: setup-gcp-and-run-nfr-tests
-setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
+setup-gcp-and-run-nfr-tests: create-gcp-resources nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
 
 .PHONY: create-gke-cluster
-create-gke-cluster: ## Create a GKE cluster
-	./scripts/create-gke-cluster.sh $(CI)
-
-.PHONY: create-and-setup-vm
-create-and-setup-vm: ## Create and setup a GCP VM for tests
-	./scripts/create-and-setup-gcp-vm.sh
-
-.PHONY: create-gke-router
-create-gke-router: ## Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
-	./scripts/create-gke-router.sh
+create-gcp-resources: ## Create a GKE cluster and a GCP VM for tests
+	tofu -chdir=tofu apply
 
 .PHONY: sync-files-to-vm
 sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
@@ -150,23 +142,8 @@ test-with-plus: PLUS_ENABLED=true
 test-with-plus: test ## Runs the functional tests for NGF with NGINX Plus on your default k8s cluster
 
 .PHONY: cleanup-gcp
-cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources
-
-.PHONY: cleanup-router
-cleanup-router: ## Delete the GKE router
-	./scripts/cleanup-router.sh
-
-.PHONY: cleanup-vm
-cleanup-vm: ## Delete the test GCP VM and delete the firewall rule
-	./scripts/cleanup-vm.sh
-
-.PHONY: delete-gke-cluster
-delete-gke-cluster: ## Delete the GKE cluster
-	./scripts/delete-gke-cluster.sh
-
-.PHONY: add-local-ip-to-cluster
-add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks
-	./scripts/add-local-ip-auth-networks.sh
+cleanup-gcp: ## Cleanup all GCP resources
+	tofu -chdir=tofu destroy
 
 HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set service.create=false --skip-schema-validation
 

tests/scripts/run-tests-gcp-vm.sh

@@ -4,11 +4,14 @@ set -eo pipefail
 
 SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
-source scripts/vars.env
+GKE_CLUSTER_ZONE=$(tofu -chdir=tofu output -raw k8s_cluster_zone)
+GKE_CLUSTER_NAME=$(tofu -chdir=tofu output -raw k8s_cluster_name)
+GKE_PROJECT=$(tofu -chdir=tofu output -raw project_id)
+VM_NAME=$(tofu -chdir=tofu output -raw vm_name)
 
-gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~
+gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${VM_NAME}":~
 
-gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \
+gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${VM_NAME}" \
     --command="export START_LONGEVITY=${START_LONGEVITY} &&\
         export STOP_LONGEVITY=${STOP_LONGEVITY} &&\
         export CI=${CI} &&\
@@ -23,9 +26,9 @@ fi
 ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config
 if [ "${CI}" = "false" ]; then
     gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null
-    rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results .
+    rsync -ave 'ssh -F ngf-gcp.ssh' username@"${VM_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results .
 else
-    gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results .
+    gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${VM_NAME}":~/nginx-gateway-fabric/tests/results .
 fi
 
 ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results
@@ -44,15 +47,15 @@ if [ "${STOP_LONGEVITY}" = "true" ]; then
     printf "\n## Error Logs\n\n" >>"${results}"
 
     ## ngf error logs
-    ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)")
-    ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)")
+    ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${GKE_CLUSTER_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)")
+    ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${GKE_CLUSTER_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)")
     printf "### nginx-gateway\n%s\n%s\n\n" "${ngfErrText}" "${ngfErrJSON}" >>"${results}"
 
     ## nginx error logs
-    ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)")
+    ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${GKE_CLUSTER_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)")
     printf "### nginx\n%s\n\n" "${ngxErr}" >>"${results}"
 
     ## nginx non-200 responses (also filter out 499 since wrk cancels connections)
-    ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)")
+    ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${GKE_CLUSTER_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)")
     printf "%s\n\n" "${ngxNon200}" >>"${results}"
 fi