tfexec: Initial test command support (#400)

Reference: #398
Reference: hashicorp/terraform#33454

Adds support for the `terraform test` command, which currently supports JSON machine-readable output and one flag for configuring the tests directory away from the command default of `tests`. The command will return a non-zero status if any of the tests fail, which returns an error back to callers of the `Test` function. If consumers need access to the pass/fail test results, the terraform-json Go module will need to be enhanced to support the test summary JSON, e.g.

```
{"@Level":"info","@message":"Failure! 0 passed, 1 failed.","@module":"terraform.ui","@timestamp":"2023-07-25T10:03:42.980799-04:00","test_summary":{"status":"fail","passed":0,"failed":1,"errored":0,"skipped":0},"type":"test_summary"}
```

Output of new end-to-end testing:

```
$ TFEXEC_E2ETEST_VERSIONS=1.5.3,1.6.0-alpha20230719 go test -count=1 -run='TestTest' -v ./tfexec/internal/e2etest
...
--- PASS: TestTest (9.50s)
    --- SKIP: TestTest/test_command_passing-1.5.3 (4.06s)
    --- PASS: TestTest/test_command_passing-1.6.0-alpha20230719 (5.44s)
...
--- PASS: TestTestError (0.48s)
    --- SKIP: TestTestError/test_command_failing-1.5.3 (0.27s)
    --- PASS: TestTestError/test_command_failing-1.6.0-alpha20230719 (0.21s)
```

tfexec/internal/e2etest/test_test.go

@@ -0,0 +1,56 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package e2etest
+
+import (
+	"context"
+	"io"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+
+	"github.com/hashicorp/terraform-exec/tfexec"
+)
+
+var (
+	testMinVersion = version.Must(version.NewVersion("1.6.0"))
+)
+
+func TestTest(t *testing.T) {
+	runTest(t, "test_command_passing", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+		// Use Core() to enable pre-release support
+		if tfv.Core().LessThan(testMinVersion) {
+			t.Skip("test command is not available in this Terraform version")
+		}
+
+		err := tf.Test(context.Background(), nil)
+
+		if err != nil {
+			t.Fatalf("error running test command: %s", err)
+		}
+	})
+}
+
+func TestTestError(t *testing.T) {
+	runTest(t, "test_command_failing", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
+		// Use Core() to enable pre-release support
+		if tfv.Core().LessThan(testMinVersion) {
+			t.Skip("test command is not available in this Terraform version")
+		}
+
+		err := tf.Test(context.Background(), io.Discard)
+
+		if err == nil {
+			t.Fatal("expected error, got none")
+		}
+
+		got := err.Error()
+		expected := regexp.MustCompile("exit status 1")
+
+		if !expected.MatchString(got) {
+			t.Fatalf("expected error matching '%s', got: %s", expected, got)
+		}
+	})
+}

tfexec/internal/e2etest/testdata/test_command_failing/main.tf

@@ -0,0 +1,11 @@
+variable "test" {
+  type = string
+}
+
+resource "terraform_data" "test" {
+  input = var.test
+}
+
+output "test" {
+  value = terraform_data.test.output
+}

tfexec/internal/e2etest/testdata/test_command_failing/tests/passthrough.tftest

@@ -0,0 +1,12 @@
+variables {
+  test = "test value"
+}
+
+run "variable_output_passthrough" {
+  command = apply
+
+  assert {
+    condition     = output.test == "not test value" # intentionally incorrect
+    error_message = "variable was not passed through to output"
+  }
+}

tfexec/internal/e2etest/testdata/test_command_passing/main.tf

@@ -0,0 +1,11 @@
+variable "test" {
+  type = string
+}
+
+resource "terraform_data" "test" {
+  input = var.test
+}
+
+output "test" {
+  value = terraform_data.test.output
+}

tfexec/internal/e2etest/testdata/test_command_passing/tests/passthrough.tftest

@@ -0,0 +1,12 @@
+variables {
+  test = "test value"
+}
+
+run "variable_output_passthrough" {
+  command = apply
+
+  assert {
+    condition     = output.test == "test value"
+    error_message = "variable was not passed through to output"
+  }
+}

tfexec/internal/testutil/tfcache.go

@@ -22,6 +22,8 @@ const (
 	Latest015   = "0.15.5"
 	Latest_v1   = "1.0.11"
 	Latest_v1_1 = "1.1.9"
+	Latest_v1_5 = "1.5.3"
+	Latest_v1_6 = "1.6.0-alpha20230719"
 )
 
 const appendUserAgent = "tfexec-testutil"

tfexec/options.go

@@ -365,6 +365,15 @@ func Target(resource string) *TargetOption {
 	return &TargetOption{resource}
 }
 
+type TestsDirectoryOption struct {
+	testsDirectory string
+}
+
+// TestsDirectory represents the -tests-directory option (path to tests files)
+func TestsDirectory(testsDirectory string) *TestsDirectoryOption {
+	return &TestsDirectoryOption{testsDirectory}
+}
+
 type GraphTypeOption struct {
 	graphType string
 }

tfexec/test.go

@@ -0,0 +1,66 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package tfexec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+)
+
+type testConfig struct {
+	testsDirectory string
+}
+
+var defaultTestOptions = testConfig{}
+
+type TestOption interface {
+	configureTest(*testConfig)
+}
+
+func (opt *TestsDirectoryOption) configureTest(conf *testConfig) {
+	conf.testsDirectory = opt.testsDirectory
+}
+
+// Test represents the terraform test -json subcommand.
+//
+// The given io.Writer, if specified, will receive
+// [machine-readable](https://developer.hashicorp.com/terraform/internals/machine-readable-ui)
+// JSON from Terraform including test results.
+func (tf *Terraform) Test(ctx context.Context, w io.Writer, opts ...TestOption) error {
+	err := tf.compatible(ctx, tf1_6_0, nil)
+
+	if err != nil {
+		return fmt.Errorf("terraform test was added in 1.6.0: %w", err)
+	}
+
+	tf.SetStdout(w)
+
+	testCmd := tf.testCmd(ctx)
+
+	err = tf.runTerraformCmd(ctx, testCmd)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (tf *Terraform) testCmd(ctx context.Context, opts ...TestOption) *exec.Cmd {
+	c := defaultTestOptions
+
+	for _, o := range opts {
+		o.configureTest(&c)
+	}
+
+	args := []string{"test", "-json"}
+
+	if c.testsDirectory != "" {
+		args = append(args, "-tests-directory="+c.testsDirectory)
+	}
+
+	return tf.buildTerraformCmd(ctx, nil, args...)
+}

tfexec/test_test.go

@@ -0,0 +1,43 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package tfexec
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
+)
+
+func TestTestCmd(t *testing.T) {
+	td := t.TempDir()
+
+	tf, err := NewTerraform(td, tfVersion(t, testutil.Latest_v1_6))
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// empty env, to avoid environ mismatch in testing
+	tf.SetEnv(map[string]string{})
+
+	t.Run("defaults", func(t *testing.T) {
+		testCmd := tf.testCmd(context.Background())
+
+		assertCmd(t, []string{
+			"test",
+			"-json",
+		}, nil, testCmd)
+	})
+
+	t.Run("override all defaults", func(t *testing.T) {
+		testCmd := tf.testCmd(context.Background(), TestsDirectory("test"))
+
+		assertCmd(t, []string{
+			"test",
+			"-json",
+			"-tests-directory=test",
+		}, nil, testCmd)
+	})
+}

tfexec/version.go

@@ -31,6 +31,7 @@ var (
 	tf0_15_3 = version.Must(version.NewVersion("0.15.3"))
 	tf1_1_0  = version.Must(version.NewVersion("1.1.0"))
 	tf1_4_0  = version.Must(version.NewVersion("1.4.0"))
+	tf1_6_0  = version.Must(version.NewVersion("1.6.0"))
 )
 
 // Version returns structured output from the terraform version command including both the Terraform CLI version