Use OpenTofu to create GCP infrastructure

.editorconfig

@@ -8,6 +8,6 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = tab
 
-[*.{md,yml,yaml}]
+[*.{md,yml,yaml,tf}]
 indent_size = 2
 indent_style = space

.github/workflows/nfr.yml

@@ -77,7 +77,6 @@ jobs:
         id: auth
         uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
         with:
-          token_format: access_token
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
@@ -87,43 +86,41 @@ jobs:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           install_components: kubectl
 
+
+      - name: Setup OpenTofu
+        uses: opentofu/setup-opentofu@592200bd4b9bbf4772ace78f887668b1aee8f716 # v1.0.5
+        with:
+          tofu_version: 1.8.7 # renovate: datasource=github-tags depName=opentofu/opentofu
+          tofu_wrapper: false
+
+      - name: Initialize OpenTofu
+        working-directory: ./tests/tofu
+        run: |
+          tofu version
+          tofu init
+
       - name: Setup dotenv file
         working-directory: ./tests/scripts
         run: |
-          echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
           echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env
           echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
           echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
           echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
-          echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
-          echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env
-          echo "GKE_CLUSTER_REGION=us-west1" >> vars.env
-          echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
-          echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
-          echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
-          echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
-          echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
-          echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
-          echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
           echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env
           echo "GINKGO_LABEL=" >> vars.env
           echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env
-          echo "GKE_NUM_NODES=12" >> vars.env
-          echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
-
-      - name: Setup license file for plus
-        if: matrix.type == 'plus'
-        env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
-        run: echo "${PLUS_LICENSE}" > license.jwt
 
-      - name: Create GKE cluster
-        working-directory: ./tests
-        run: make create-gke-cluster CI=true
-
-      - name: Create and setup VM
-        working-directory: ./tests
-        run: make create-and-setup-vm
+      - name: Create GKE cluster and VM
+        working-directory: ./tests/tofu
+        run: |
+          tofu apply --auto-approve \
+          -var="gke_cluster_name=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" \
+          -var="vm_service_account=${{ secrets.GCP_SERVICE_ACCOUNT }}" \
+          -var="gke_nodes_service_account=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" \
+          -var="gke_num_nodes=12" \
+          -var="gke_machine_type=n2d-standard-16" \
+          -var="gke_project=${{ secrets.GCP_PROJECT_ID }}" \
+          -var="ngf_branch=${{ github.ref_name }}"
 
       - name: Run Tests
         working-directory: ./tests
@@ -141,12 +138,10 @@ jobs:
           path: tests/results/**/*-${{ matrix.type }}.*
 
       - name: Cleanup
-        working-directory: ./tests
+        working-directory: ./tests/tofu
         if: always()
         run: |
-          bash scripts/cleanup-vm.sh true
-          make delete-gke-cluster
-          rm -rf scripts/vars.env
+          tofu destroy --auto-approve
 
   pr-results:
     name: Open PR with results

.gitignore

@@ -61,3 +61,32 @@ internal/mode/static/nginx/modules/coverage
 # temporary files used for manifest generation
 config/base/deploy.yaml
 config/base/deploy.yaml.bak
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

.pre-commit-config.yaml

@@ -95,6 +95,12 @@ repos:
           - "--skip-auto-generation=required,additionalProperties"
           - --append-newline
 
+  - repo: https://github.com/tofuutils/pre-commit-opentofu
+    rev: v2.1.0
+    hooks:
+      - id: tofu_fmt
+      # - id: tofu_docs
+
 ci:
   skip: [golangci-lint-full, prettier, markdownlint-cli2, yamllint]
   autofix_prs: false

tests/Makefile

@@ -90,19 +90,11 @@ ifeq ($(PLUS_ENABLED),true)
 endif
 
 .PHONY: setup-gcp-and-run-nfr-tests
-setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
+setup-gcp-and-run-nfr-tests: create-gcp-resources nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
 
 .PHONY: create-gke-cluster
-create-gke-cluster: ## Create a GKE cluster
-	./scripts/create-gke-cluster.sh $(CI)
-
-.PHONY: create-and-setup-vm
-create-and-setup-vm: ## Create and setup a GCP VM for tests
-	./scripts/create-and-setup-gcp-vm.sh
-
-.PHONY: create-gke-router
-create-gke-router: ## Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
-	./scripts/create-gke-router.sh
+create-gcp-resources: ## Create a GKE cluster and a GCP VM for tests
+	tofu -chdir=tofu apply
 
 .PHONY: sync-files-to-vm
 sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
@@ -150,23 +142,8 @@ test-with-plus: PLUS_ENABLED=true
 test-with-plus: test ## Runs the functional tests for NGF with NGINX Plus on your default k8s cluster
 
 .PHONY: cleanup-gcp
-cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources
-
-.PHONY: cleanup-router
-cleanup-router: ## Delete the GKE router
-	./scripts/cleanup-router.sh
-
-.PHONY: cleanup-vm
-cleanup-vm: ## Delete the test GCP VM and delete the firewall rule
-	./scripts/cleanup-vm.sh
-
-.PHONY: delete-gke-cluster
-delete-gke-cluster: ## Delete the GKE cluster
-	./scripts/delete-gke-cluster.sh
-
-.PHONY: add-local-ip-to-cluster
-add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks
-	./scripts/add-local-ip-auth-networks.sh
+cleanup-gcp: ## Cleanup all GCP resources
+	tofu -chdir=tofu destroy
 
 HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set service.create=false --skip-schema-validation