From fbc42a04ea24e2246f81491434a965846d55ed69 Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Tue, 26 Nov 2024 05:27:50 +0600 Subject: [PATCH] fix(misconf): load full Terraform module (#7925) Signed-off-by: nikpivkin --- .../resolvers/cache_integration_test.go | 133 ++++++++++++------ .../terraform/parser/resolvers/options.go | 2 + .../terraform/parser/resolvers/registry.go | 11 +- .../resolvers/registry_integration_test.go | 16 ++- .../terraform/parser/resolvers/remote.go | 13 +- .../terraform-aws-s3-bucket/README.md | 1 + .../modules/notification/README.md | 1 + .../modules/object/README.md | 1 + 8 files changed, 125 insertions(+), 53 deletions(-) create mode 100644 pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/README.md create mode 100644 pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/notification/README.md create mode 100644 pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/object/README.md diff --git a/pkg/iac/scanners/terraform/parser/resolvers/cache_integration_test.go b/pkg/iac/scanners/terraform/parser/resolvers/cache_integration_test.go index 43ad7f06b15b..6bfe812519bd 100644 --- a/pkg/iac/scanners/terraform/parser/resolvers/cache_integration_test.go +++ b/pkg/iac/scanners/terraform/parser/resolvers/cache_integration_test.go @@ -1,119 +1,172 @@ +//go:build unix + package resolvers_test import ( "context" + "crypto/tls" "io/fs" + "net/http" + "net/http/httptest" + "path" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/aquasecurity/trivy/internal/gittest" "github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser/resolvers" + "github.com/aquasecurity/trivy/pkg/log" ) type moduleResolver interface { Resolve(context.Context, fs.FS, resolvers.Options) (fs.FS, string, string, bool, error) } -func TestResolveModuleFromCache(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") +func testOptions(t *testing.T, source string) resolvers.Options { + return resolvers.Options{ + Source: source, + OriginalSource: source, + Version: "", + OriginalVersion: "", + AllowDownloads: true, + CacheDir: t.TempDir(), + Logger: log.WithPrefix("test"), } +} + +func newRegistry(repoURL string) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/v1/modules/terraform-aws-modules/s3-bucket/aws/download", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Terraform-Get", repoURL) + w.WriteHeader(http.StatusNoContent) + }) + + return httptest.NewTLSServer(mux) +} + +func buildGitSource(repoURL string) string { return "git::" + repoURL } + +func TestResolveModuleFromCache(t *testing.T) { + + repo := "terraform-aws-s3-bucket" + gs := gittest.NewServer(t, repo, "testdata/terraform-aws-s3-bucket") + defer gs.Close() + + repoURL := gs.URL + "/" + repo + ".git" + + registry := newRegistry(buildGitSource(repoURL)) + defer registry.Close() + + registryAddress := strings.TrimPrefix(registry.URL, "https://") tests := []struct { name string opts resolvers.Options firstResolver moduleResolver expectedSubdir string + expectedString string }{ { name: "registry", opts: resolvers.Options{ - Name: "bucket", - Source: "terraform-aws-modules/s3-bucket/aws", - Version: "4.1.2", + Source: registryAddress + "/terraform-aws-modules/s3-bucket/aws", + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, }, firstResolver: resolvers.Registry, expectedSubdir: ".", + expectedString: "# AWS S3 bucket Terraform module", }, { name: "registry with subdir", opts: resolvers.Options{ - Name: "object", - Source: "terraform-aws-modules/s3-bucket/aws//modules/object", - Version: "4.1.2", + Source: registryAddress + "/terraform-aws-modules/s3-bucket/aws//modules/object", + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, }, firstResolver: resolvers.Registry, expectedSubdir: "modules/object", + expectedString: "# S3 bucket object", }, { name: "remote", opts: resolvers.Options{ - Name: "bucket", - Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git?ref=v4.1.2", + Source: buildGitSource(repoURL), }, firstResolver: resolvers.Remote, expectedSubdir: ".", + expectedString: "# AWS S3 bucket Terraform module", }, { name: "remote with subdir", opts: resolvers.Options{ - Name: "object", - Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2", + Source: buildGitSource(repoURL) + "//modules/object", }, firstResolver: resolvers.Remote, expectedSubdir: "modules/object", + expectedString: "# S3 bucket object", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.opts.AllowDownloads = true tt.opts.OriginalSource = tt.opts.Source - tt.opts.OriginalVersion = tt.opts.Version + tt.opts.AllowDownloads = true tt.opts.CacheDir = t.TempDir() + tt.opts.Logger = log.WithPrefix("test") + + fsys, _, dir, _, err := tt.firstResolver.Resolve(context.Background(), nil, tt.opts) + require.NoError(t, err) + assert.Equal(t, tt.expectedSubdir, dir) - fsys, _, _, applies, err := tt.firstResolver.Resolve(context.Background(), nil, tt.opts) + b, err := fs.ReadFile(fsys, path.Join(dir, "README.md")) require.NoError(t, err) - assert.True(t, applies) + assert.Equal(t, tt.expectedString, string(b)) - _, err = fs.Stat(fsys, "main.tf") + _, _, dir, _, err = resolvers.Cache.Resolve(context.Background(), fsys, tt.opts) require.NoError(t, err) + assert.Equal(t, tt.expectedSubdir, dir) - _, _, _, applies, err = resolvers.Cache.Resolve(context.Background(), fsys, tt.opts) + b, err = fs.ReadFile(fsys, path.Join(dir, "README.md")) require.NoError(t, err) - assert.True(t, applies) + assert.Equal(t, tt.expectedString, string(b)) }) } } func TestResolveModuleFromCacheWithDifferentSubdir(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") - } + repo := "terraform-aws-s3-bucket" + gs := gittest.NewServer(t, repo, "testdata/terraform-aws-s3-bucket") + defer gs.Close() - cacheDir := t.TempDir() + repoURL := gs.URL + "/" + repo + ".git" - fsys, _, _, applies, err := resolvers.Remote.Resolve(context.Background(), nil, resolvers.Options{ - Name: "object", - Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2", - OriginalSource: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/object?ref=v4.1.2", - AllowDownloads: true, - CacheDir: cacheDir, - }) + fsys, _, dir, _, err := resolvers.Remote.Resolve( + context.Background(), nil, + testOptions(t, "git::"+repoURL+"//modules/object"), + ) require.NoError(t, err) - assert.True(t, applies) - _, err = fs.Stat(fsys, "main.tf") + b, err := fs.ReadFile(fsys, path.Join(dir, "README.md")) require.NoError(t, err) + assert.Equal(t, "# S3 bucket object", string(b)) - _, _, _, applies, err = resolvers.Cache.Resolve(context.Background(), nil, resolvers.Options{ - Name: "notification", - Source: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/notification?ref=v4.1.2", - OriginalSource: "git::https://github.com/terraform-aws-modules/terraform-aws-s3-bucket.git//modules/notification?ref=v4.1.2", - CacheDir: cacheDir, - }) + fsys, _, dir, _, err = resolvers.Remote.Resolve( + context.Background(), nil, + testOptions(t, "git::"+repoURL+"//modules/notification"), + ) + require.NoError(t, err) + + b, err = fs.ReadFile(fsys, path.Join(dir, "README.md")) require.NoError(t, err) - assert.True(t, applies) + assert.Equal(t, "# S3 bucket notification", string(b)) } diff --git a/pkg/iac/scanners/terraform/parser/resolvers/options.go b/pkg/iac/scanners/terraform/parser/resolvers/options.go index 73fd39689e84..937f89709dc3 100644 --- a/pkg/iac/scanners/terraform/parser/resolvers/options.go +++ b/pkg/iac/scanners/terraform/parser/resolvers/options.go @@ -1,6 +1,7 @@ package resolvers import ( + "net/http" "strings" "github.com/aquasecurity/trivy/pkg/log" @@ -13,6 +14,7 @@ type Options struct { SkipCache bool RelativePath string CacheDir string + Client *http.Client } func (o *Options) hasPrefix(prefixes ...string) bool { diff --git a/pkg/iac/scanners/terraform/parser/resolvers/registry.go b/pkg/iac/scanners/terraform/parser/resolvers/registry.go index 471416463cad..1af19f753209 100644 --- a/pkg/iac/scanners/terraform/parser/resolvers/registry.go +++ b/pkg/iac/scanners/terraform/parser/resolvers/registry.go @@ -41,12 +41,17 @@ const registryHostname = "registry.terraform.io" // nolint func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Options) (filesystem fs.FS, prefix string, downloadPath string, applies bool, err error) { + client := r.client + if opt.Client != nil { + client = opt.Client + } + if !opt.AllowDownloads { return } inputVersion := opt.Version - source, _ := splitPackageSubdirRaw(opt.Source) + source, _ := splitPackageSubdirRaw(opt.OriginalSource) parts := strings.Split(source, "/") if len(parts) < 3 || len(parts) > 4 { return @@ -81,7 +86,7 @@ func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Option if token != "" { req.Header.Set("Authorization", "Bearer "+token) } - resp, err := r.client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, "", "", true, err } @@ -122,7 +127,7 @@ func (r *registryResolver) Resolve(ctx context.Context, target fs.FS, opt Option req.Header.Set("X-Terraform-Version", opt.Version) } - resp, err := r.client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, "", "", true, err } diff --git a/pkg/iac/scanners/terraform/parser/resolvers/registry_integration_test.go b/pkg/iac/scanners/terraform/parser/resolvers/registry_integration_test.go index e2d87104da2d..b8be4b10e0f2 100644 --- a/pkg/iac/scanners/terraform/parser/resolvers/registry_integration_test.go +++ b/pkg/iac/scanners/terraform/parser/resolvers/registry_integration_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser/resolvers" + "github.com/aquasecurity/trivy/pkg/log" ) func TestResolveModuleFromOpenTofuRegistry(t *testing.T) { @@ -17,12 +18,15 @@ func TestResolveModuleFromOpenTofuRegistry(t *testing.T) { } fsys, _, path, _, err := resolvers.Registry.Resolve(context.Background(), nil, resolvers.Options{ - Source: "registry.opentofu.org/terraform-aws-modules/s3-bucket/aws", - RelativePath: "test", - Name: "bucket", - Version: "4.1.2", - AllowDownloads: true, - SkipCache: true, + Source: "registry.opentofu.org/terraform-aws-modules/s3-bucket/aws", + OriginalSource: "registry.opentofu.org/terraform-aws-modules/s3-bucket/aws", + RelativePath: "test", + Name: "bucket", + Version: "4.1.2", + OriginalVersion: "4.1.2", + AllowDownloads: true, + SkipCache: true, + Logger: log.WithPrefix("test"), }) require.NoError(t, err) diff --git a/pkg/iac/scanners/terraform/parser/resolvers/remote.go b/pkg/iac/scanners/terraform/parser/resolvers/remote.go index 467f2cee6970..d70edde8c3b7 100644 --- a/pkg/iac/scanners/terraform/parser/resolvers/remote.go +++ b/pkg/iac/scanners/terraform/parser/resolvers/remote.go @@ -40,15 +40,20 @@ func (r *remoteResolver) Resolve(ctx context.Context, _ fs.FS, opt Options) (fil return nil, "", "", false, nil } - src, subdir := splitPackageSubdirRaw(opt.OriginalSource) - key := cacheKey(src, opt.OriginalVersion) + origSrc, subdir := splitPackageSubdirRaw(opt.OriginalSource) + key := cacheKey(origSrc, opt.OriginalVersion) opt.Logger.Debug("Caching module", log.String("key", key)) baseCacheDir, err := locateCacheDir(opt.CacheDir) if err != nil { return nil, "", "", true, fmt.Errorf("failed to locate cache directory: %w", err) } + cacheDir := filepath.Join(baseCacheDir, key) + + src, _ := splitPackageSubdirRaw(opt.Source) + + opt.Source = src if err := r.download(ctx, opt, cacheDir); err != nil { return nil, "", "", true, err } @@ -56,9 +61,9 @@ func (r *remoteResolver) Resolve(ctx context.Context, _ fs.FS, opt Options) (fil r.incrementCount(opt) opt.Logger.Debug("Successfully resolve module via remote download", log.String("name", opt.Name), - log.String("source", opt.Source), + log.String("source", opt.OriginalSource), ) - return os.DirFS(cacheDir), opt.Source, subdir, true, nil + return os.DirFS(cacheDir), opt.OriginalSource, subdir, true, nil } func (r *remoteResolver) download(ctx context.Context, opt Options, dst string) error { diff --git a/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/README.md b/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/README.md new file mode 100644 index 000000000000..26e6186c9cbe --- /dev/null +++ b/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/README.md @@ -0,0 +1 @@ +# AWS S3 bucket Terraform module \ No newline at end of file diff --git a/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/notification/README.md b/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/notification/README.md new file mode 100644 index 000000000000..9f2f884a9586 --- /dev/null +++ b/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/notification/README.md @@ -0,0 +1 @@ +# S3 bucket notification \ No newline at end of file diff --git a/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/object/README.md b/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/object/README.md new file mode 100644 index 000000000000..a016e2bb23cb --- /dev/null +++ b/pkg/iac/scanners/terraform/parser/resolvers/testdata/terraform-aws-s3-bucket/modules/object/README.md @@ -0,0 +1 @@ +# S3 bucket object \ No newline at end of file