fix(misconf): load full Terraform module (#7925)

Signed-off-by: nikpivkin <nikita.pivkin@smartforce.io>

pkg/iac/scanners/terraform/parser/resolvers/cache_integration_test.go

@@ -1,119 +1,172 @@
+//go:build unix
+
 package resolvers_test
 
 import (
 	"context"
+	"crypto/tls"
 	"io/fs"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/aquasecurity/trivy/internal/gittest"
 	"github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser/resolvers"
+	"github.com/aquasecurity/trivy/pkg/log"
 )
 
 type moduleResolver interface {
 	Resolve(context.Context, fs.FS, resolvers.Options) (fs.FS, string, string, bool, error)
 }
 
-func TestResolveModuleFromCache(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
+func testOptions(t *testing.T, source string) resolvers.Options {
+	return resolvers.Options{
+		Source:          source,
+		OriginalSource:  source,
+		Version:         "",
+		OriginalVersion: "",
+		AllowDownloads:  true,
+		CacheDir:        t.TempDir(),
+		Logger:          log.WithPrefix("test"),
 	}
+}
+
+func newRegistry(repoURL string) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/modules/terraform-aws-modules/s3-bucket/aws/download", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Terraform-Get", repoURL)
+		w.WriteHeader(http.StatusNoContent)
+	})
+
+	return httptest.NewTLSServer(mux)
+}
+
+func buildGitSource(repoURL string) string { return "git::" + repoURL }
+
+func TestResolveModuleFromCache(t *testing.T) {
+
+	repo := "terraform-aws-s3-bucket"
+	gs := gittest.NewServer(t, repo, "testdata/terraform-aws-s3-bucket")
+	defer gs.Close()
+
+	repoURL := gs.URL + "/" + repo + ".git"
+
+	registry := newRegistry(buildGitSource(repoURL))
+	defer registry.Close()
+
+	registryAddress := strings.TrimPrefix(registry.URL, "https://")
 
 	tests := []struct {
 		name           string
 		opts           resolvers.Options
 		firstResolver  moduleResolver
 		expectedSubdir string
+		expectedString string
 	}{
 		{
 			name: "registry",
 			opts: resolvers.Options{
-				Name:    "bucket",
-				Source:  "terraform-aws-modules/s3-bucket/aws",
-				Version: "4.1.2",
+				Source: registryAddress + "/terraform-aws-modules/s3-bucket/aws",
+				Client: &http.Client{
+					Transport: &http.Transport{
+						TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+					},
+				},
 			},
 			firstResolver:  resolvers.Registry,
 			expectedSubdir: ".",
+			expectedString: "# AWS S3 bucket Terraform module",
 		},
 		{
 			name: "registry with subdir",
 			opts: resolvers.Options{
-				Name:    "object",
-				Source:  "terraform-aws-modules/s3-bucket/aws//modules/object",
-				Version: "4.1.2",
+				Source: registryAddress + "/terraform-aws-modules/s3-bucket/aws//modules/object",
+				Client: &http.Client{
+					Transport: &http.Transport{
+						TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+					},
+				},
 			},
 			firstResolver:  resolvers.Registry,
 			expectedSubdir: "modules/object",
+			expectedString: "# S3 bucket object",
 		},
 		{
 			name: "remote",
 			opts: resolvers.Options{
-				Name:   "bucket",
-				Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git?ref=v4.1.2",
+				Source: buildGitSource(repoURL),
 			},
 			firstResolver:  resolvers.Remote,
 			expectedSubdir: ".",
+			expectedString: "# AWS S3 bucket Terraform module",
 		},
 		{
 			name: "remote with subdir",
 			opts: resolvers.Options{
-				Name:   "object",
-				Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2",
+				Source: buildGitSource(repoURL) + "//modules/object",
 			},
 			firstResolver:  resolvers.Remote,
 			expectedSubdir: "modules/object",
+			expectedString: "# S3 bucket object",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 
-			tt.opts.AllowDownloads = true
 			tt.opts.OriginalSource = tt.opts.Source
-			tt.opts.OriginalVersion = tt.opts.Version
+			tt.opts.AllowDownloads = true
 			tt.opts.CacheDir = t.TempDir()
+			tt.opts.Logger = log.WithPrefix("test")
+
+			fsys, _, dir, _, err := tt.firstResolver.Resolve(context.Background(), nil, tt.opts)
+			require.NoError(t, err)
+			assert.Equal(t, tt.expectedSubdir, dir)
 
-			fsys, _, _, applies, err := tt.firstResolver.Resolve(context.Background(), nil, tt.opts)
+			b, err := fs.ReadFile(fsys, path.Join(dir, "README.md"))
 			require.NoError(t, err)
-			assert.True(t, applies)
+			assert.Equal(t, tt.expectedString, string(b))
 
-			_, err = fs.Stat(fsys, "main.tf")
+			_, _, dir, _, err = resolvers.Cache.Resolve(context.Background(), fsys, tt.opts)
 			require.NoError(t, err)
+			assert.Equal(t, tt.expectedSubdir, dir)
 
-			_, _, _, applies, err = resolvers.Cache.Resolve(context.Background(), fsys, tt.opts)
+			b, err = fs.ReadFile(fsys, path.Join(dir, "README.md"))
 			require.NoError(t, err)
-			assert.True(t, applies)
+			assert.Equal(t, tt.expectedString, string(b))
 		})
 	}
 }
 
 func TestResolveModuleFromCacheWithDifferentSubdir(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
+	repo := "terraform-aws-s3-bucket"
+	gs := gittest.NewServer(t, repo, "testdata/terraform-aws-s3-bucket")
+	defer gs.Close()
 
-	cacheDir := t.TempDir()
+	repoURL := gs.URL + "/" + repo + ".git"
 
-	fsys, _, _, applies, err := resolvers.Remote.Resolve(context.Background(), nil, resolvers.Options{
-		Name:           "object",
-		Source:         "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2",
-		OriginalSource: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2",
-		AllowDownloads: true,
-		CacheDir:       cacheDir,
-	})
+	fsys, _, dir, _, err := resolvers.Remote.Resolve(
+		context.Background(), nil,
+		testOptions(t, "git::"+repoURL+"//modules/object"),
+	)
 	require.NoError(t, err)
-	assert.True(t, applies)
 
-	_, err = fs.Stat(fsys, "main.tf")
+	b, err := fs.ReadFile(fsys, path.Join(dir, "README.md"))
 	require.NoError(t, err)
+	assert.Equal(t, "# S3 bucket object", string(b))
 
-	_, _, _, applies, err = resolvers.Cache.Resolve(context.Background(), nil, resolvers.Options{
-		Name:           "notification",
-		Source:         "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/notification?ref=v4.1.2",
-		OriginalSource: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/notification?ref=v4.1.2",
-		CacheDir:       cacheDir,
-	})
+	fsys, _, dir, _, err = resolvers.Remote.Resolve(
+		context.Background(), nil,
+		testOptions(t, "git::"+repoURL+"//modules/notification"),
+	)
+	require.NoError(t, err)
+
+	b, err = fs.ReadFile(fsys, path.Join(dir, "README.md"))
 	require.NoError(t, err)
-	assert.True(t, applies)
+	assert.Equal(t, "# S3 bucket notification", string(b))
 }

pkg/iac/scanners/terraform/parser/resolvers/options.go

@@ -1,6 +1,7 @@
 package resolvers
 
 import (
+	"net/http"
 	"strings"
 
 	"github.com/aquasecurity/trivy/pkg/log"
@@ -13,6 +14,7 @@ type Options struct {
 	SkipCache                                                                      bool
 	RelativePath                                                                   string
 	CacheDir                                                                       string
+	Client                                                                         *http.Client
 }
 
 func (o *Options) hasPrefix(prefixes ...string) bool {

pkg/iac/scanners/terraform/parser/resolvers/registry.go

@@ -41,12 +41,17 @@ const registryHostname = "registry.terraform.io"
 // nolint
 func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Options) (filesystem fs.FS, prefix string, downloadPath string, applies bool, err error) {
 
+	client := r.client
+	if opt.Client != nil {
+		client = opt.Client
+	}
+
 	if !opt.AllowDownloads {
 		return
 	}
 
 	inputVersion := opt.Version
-	source, _ := splitPackageSubdirRaw(opt.Source)
+	source, _ := splitPackageSubdirRaw(opt.OriginalSource)
 	parts := strings.Split(source, "/")
 	if len(parts) < 3 || len(parts) > 4 {
 		return
@@ -81,7 +86,7 @@ func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Option
 		if token != "" {
 			req.Header.Set("Authorization", "Bearer "+token)
 		}
-		resp, err := r.client.Do(req)
+		resp, err := client.Do(req)
 		if err != nil {
 			return nil, "", "", true, err
 		}
@@ -122,7 +127,7 @@ func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Option
 		req.Header.Set("X-Terraform-Version", opt.Version)
 	}
 
-	resp, err := r.client.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, "", "", true, err
 	}

pkg/iac/scanners/terraform/parser/resolvers/registry_integration_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser/resolvers"
+	"github.com/aquasecurity/trivy/pkg/log"
 )
 
 func TestResolveModuleFromOpenTofuRegistry(t *testing.T) {
@@ -17,12 +18,15 @@ func TestResolveModuleFromOpenTofuRegistry(t *testing.T) {
 	}
 
 	fsys, _, path, _, err := resolvers.Registry.Resolve(context.Background(), nil, resolvers.Options{
-		Source:         "registry.opentofu.org/terraform-aws-modules/s3-bucket/aws",
-		RelativePath:   "test",
-		Name:           "bucket",
-		Version:        "4.1.2",
-		AllowDownloads: true,
-		SkipCache:      true,
+		Source:          "registry.opentofu.org/terraform-aws-modules/s3-bucket/aws",
+		OriginalSource:  "registry.opentofu.org/terraform-aws-modules/s3-bucket/aws",
+		RelativePath:    "test",
+		Name:            "bucket",
+		Version:         "4.1.2",
+		OriginalVersion: "4.1.2",
+		AllowDownloads:  true,
+		SkipCache:       true,
+		Logger:          log.WithPrefix("test"),
 	})
 	require.NoError(t, err)
 

pkg/iac/scanners/terraform/parser/resolvers/remote.go

@@ -40,25 +40,30 @@ func (r *remoteResolver) Resolve(ctx context.Context, _ fs.FS, opt Options) (fil
 		return nil, "", "", false, nil
 	}
 
-	src, subdir := splitPackageSubdirRaw(opt.OriginalSource)
-	key := cacheKey(src, opt.OriginalVersion)
+	origSrc, subdir := splitPackageSubdirRaw(opt.OriginalSource)
+	key := cacheKey(origSrc, opt.OriginalVersion)
 	opt.Logger.Debug("Caching module", log.String("key", key))
 
 	baseCacheDir, err := locateCacheDir(opt.CacheDir)
 	if err != nil {
 		return nil, "", "", true, fmt.Errorf("failed to locate cache directory: %w", err)
 	}
+
 	cacheDir := filepath.Join(baseCacheDir, key)
+
+	src, _ := splitPackageSubdirRaw(opt.Source)
+
+	opt.Source = src
 	if err := r.download(ctx, opt, cacheDir); err != nil {
 		return nil, "", "", true, err
 	}
 
 	r.incrementCount(opt)
 	opt.Logger.Debug("Successfully resolve module via remote download",
 		log.String("name", opt.Name),
-		log.String("source", opt.Source),
+		log.String("source", opt.OriginalSource),
 	)
-	return os.DirFS(cacheDir), opt.Source, subdir, true, nil
+	return os.DirFS(cacheDir), opt.OriginalSource, subdir, true, nil
 }
 
 func (r *remoteResolver) download(ctx context.Context, opt Options, dst string) error {

pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/README.md

@@ -0,0 +1 @@
+# AWS S3 bucket Terraform module

pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/notification/README.md

@@ -0,0 +1 @@
+# S3 bucket notification

pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/object/README.md

@@ -0,0 +1 @@
+# S3 bucket object