support for terraform registry remote modules

go.sum

@@ -553,6 +553,7 @@ github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -683,6 +684,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI=
 github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@@ -1296,8 +1298,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/tools v0.0.0-20201028111035-eafbe7b904eb/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963 h1:K+NlvTLy0oONtRtkl1jRD9xIhnItbG2PiE7YOdjPb+k=
-golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210115202250-e0d201561e39 h1:BTs2GMGSMWpgtCpv1CE7vkJTv7XcHdcLLnAMu7UbgTY=
 golang.org/x/tools v0.0.0-20210115202250-e0d201561e39/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1496,11 +1496,11 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
-sigs.k8s.io/kustomize/api v0.7.1 h1:/cjDi4Pk/hqRSeCCj/Xum66rYrEtc7osM2/O+lvYKkM=
-sigs.k8s.io/kustomize/api v0.7.1/go.mod h1:XOt24UrCkv0x63eT5JVaph4Kqf5EVU2UBAXo6SPBaAY=
+sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs=
+sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs=
 sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE=
-sigs.k8s.io/kustomize/kyaml v0.10.5 h1:PbJcsZsEM7O3hHtUWTR+4WkHVbQRW9crSy75or1gRbI=
-sigs.k8s.io/kustomize/kyaml v0.10.5/go.mod h1:P6Oy/ah/GZMKzJMIJA2a3/bc8YrBkuL5kJji13PSIzY=
+sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE=
+sigs.k8s.io/kustomize/kyaml v0.10.6 h1:xUJxc/k8JoWqHUahaB8DTqY0KwEPxTbTGStvW8TOcDc=
 sigs.k8s.io/kustomize/kyaml v0.10.6/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=

pkg/cli/run_test.go

@@ -50,6 +50,8 @@ func TestRun(t *testing.T) {
 	testDirPath := "testdata/run-test"
 	kustomizeTestDirPath := testDirPath + "/kustomize-test"
 	testTerraformFilePath := testDirPath + "/config-only.tf"
+	testRemoteModuleFilePath := testDirPath + "/remote-modules.tf"
+
 	ruleSlice := []string{"AWS.ECR.DataSecurity.High.0579", "AWS.SecurityGroup.NetworkPortsSecurity.Low.0561"}
 
 	table := []struct {
@@ -171,6 +173,15 @@ func TestRun(t *testing.T) {
 				configFile: "testdata/configFile.toml",
 			},
 		},
+		{
+			name: "config file with remote module",
+			scanOptions: &ScanOptions{
+				policyType:  []string{"all"},
+				iacFilePath: testRemoteModuleFilePath,
+				outputType:  "human",
+				configFile:  "testdata/configFile.toml",
+			},
+		},
 	}
 
 	for _, tt := range table {

pkg/cli/testdata/run-test/remote-modules.tf

@@ -0,0 +1,18 @@
+#file added for fix for #418
+
+# source https://registry.terraform.io/
+
+module "network" {
+  source  = "Azure/network/azurerm"
+  version = "3.2.1"
+}
+
+module "eks" {
+  source = "terraform-aws-modules/eks/aws"
+}
+
+## contains local modules
+module "rds" {
+  source  = "terraform-aws-modules/rds/aws"
+  version = "2.20.0"
+}

pkg/iac-providers/terraform/commons/load-dir.go

@@ -18,14 +18,18 @@ package commons
 
 import (
 	"fmt"
+	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/accurics/terrascan/pkg/iac-providers/output"
 	"github.com/accurics/terrascan/pkg/utils"
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl/v2"
 	hclConfigs "github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/registry/regsrc"
 	"github.com/spf13/afero"
 	"go.uber.org/zap"
 )
@@ -53,6 +57,14 @@ type ModuleConfig struct {
 // resources present in rootDir and descendant modules
 func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) {
 
+	// disable terraform logs when TF_LOG env variable is not set
+	if os.Getenv("TF_LOG") == "" {
+		log.SetOutput(ioutil.Discard)
+	}
+
+	// map to hold the download paths of remote modules
+	remoteModPaths := make(map[string]string)
+
 	// create a new config parser
 	parser := hclConfigs.NewParser(afero.NewOsFs())
 
@@ -92,8 +104,36 @@ func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs
 				for p := req.Parent; p != nil; p = p.Parent {
 					pathToModule = filepath.Join(p.SourceAddr, pathToModule)
 				}
-				pathToModule = filepath.Join(absRootDir, pathToModule)
+
+				// check if pathToModule consists of a remote module downloaded
+				// if yes, then update the module path, with the remote module
+				// download path
+				keyFound := false
+				for remoteSourceAddr, downloadPath := range remoteModPaths {
+					if strings.Contains(pathToModule, remoteSourceAddr) {
+						pathToModule = strings.Replace(pathToModule, remoteSourceAddr, "", 1)
+						pathToModule = downloadPath + pathToModule
+						keyFound = true
+						break
+					}
+				}
+				if !keyFound {
+					pathToModule = filepath.Join(absRootDir, pathToModule)
+				}
 				zap.S().Debugf("processing local module %q", pathToModule)
+			} else if isRegistrySourceAddr(req.SourceAddr) {
+				// regsrc.ParseModuleSource func returns a terraform registry module source
+				// error check is not required as the source address is already validated above
+				module, _ := regsrc.ParseModuleSource(req.SourceAddr)
+
+				tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6))
+				pathToModule, err = r.DownloadRemoteModule(req.VersionConstraint, tempDir, module)
+				if err != nil {
+					zap.S().Errorf("failed to download remote module %q. error: '%v'", req.SourceAddr, err)
+				} else {
+					// add the entry of remote module's source address to the map
+					remoteModPaths[req.SourceAddr] = pathToModule
+				}
 			} else {
 				// temp dir to download the remote repo
 				tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6))

pkg/iac-providers/terraform/commons/module-download.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/accurics/terrascan/pkg/downloader"
 	"go.uber.org/zap"
+
+	"github.com/hashicorp/terraform/registry/regsrc"
 )
 
 var localSourcePrefixes = []string{
@@ -41,6 +43,15 @@ func isLocalSourceAddr(addr string) bool {
 	return false
 }
 
+// isRegistrySourceAddr will validate if the source address is a valid registry
+// module or not.
+// a valid source address is of the form <HOSTNAME>/NAMESPACE>/<NAME>/<PROVIDER>
+// regsrc.ParseModuleSource func returns a terraform registry module source.
+func isRegistrySourceAddr(addr string) bool {
+	_, err := regsrc.ParseModuleSource(addr)
+	return err == nil
+}
+
 // RemoteModuleInstaller helps in downloading remote modules, it also maintains a
 // cache of all the installed modules and their respective resolved addresses
 // (URL)

pkg/iac-providers/terraform/commons/remote-module-download.go

@@ -0,0 +1,150 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package commons
+
+import (
+	"fmt"
+	"path/filepath"
+
+	version "github.com/hashicorp/go-version"
+	hclConfigs "github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/registry/regsrc"
+	"github.com/hashicorp/terraform/registry/response"
+	"go.uber.org/zap"
+)
+
+// DownloadRemoteModule will download remote modules from public and private terraform registries
+// this function takes similar approach taken by terraform init for downloading terraform registry modules
+func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs.VersionConstraint, destPath string, module *regsrc.Module) (string, error) {
+	// Terraform doesn't allow the hostname to contain Punycode
+	// module.SvcHost returns an error for such case
+	_, err := module.SvcHost()
+	if err != nil {
+		zap.S().Errorf("hostname for the module %s is invalid", module.String())
+		return "", err
+	}
+
+	// get terraform registry client.
+	// terraform registry client provides methods for querying the terraform module registry
+	regClient := registry.NewClient(nil, nil)
+
+	// get all the available module versions from the terraform registry
+	moduleVersions, err := regClient.ModuleVersions(module)
+	if err != nil {
+		if registry.IsModuleNotFound(err) {
+			zap.S().Errorf("module: %s, not be found at registry: %s", module.String(), module.Host().Display())
+		} else {
+			zap.S().Errorf("error while fetching available modules for module: %s, at registry: %s", module.String(), module.Host().Display())
+		}
+		return "", err
+	}
+
+	// get the version to download
+	versionToDownload, err := getVersionToDownload(moduleVersions, requiredVersion, module)
+	if err != nil {
+		zap.S().Error("error while fetching the version to download,", zap.Error(err))
+		return "", err
+	}
+
+	// get the source location for the matched version
+	sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String())
+	if err != nil {
+		zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display())
+		return "", err
+	}
+
+	downloadLocation, err := r.DownloadModule(sourceLocation, destPath)
+	if err != nil {
+		zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation)
+		return "", nil
+	}
+
+	if module.RawSubmodule != "" {
+		// Append the user's requested subdirectory
+		downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule)
+	}
+
+	return downloadLocation, nil
+}
+
+// helper func to compare and update the version
+func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version {
+	if latestVersion == nil || currentVersion.GreaterThan(latestVersion) {
+		latestVersion = currentVersion
+	}
+	return latestVersion
+}
+
+// helper func to get the module version to download
+func getVersionToDownload(moduleVersions *response.ModuleVersions, requiredVersion hclConfigs.VersionConstraint, module *regsrc.Module) (*version.Version, error) {
+	// terraform init command pulls all the available versions of a module,
+	// and downloads the latest non pre-release (unless a pre-release version was
+	// specified in tf file) version, if a version constraint is not provided in the tf file.
+	// we are following what terraform does
+	allModules := moduleVersions.Modules[0]
+
+	var latestMatch *version.Version
+	var latestVersion *version.Version
+	var versionToDownload *version.Version
+	for _, moduleVersion := range allModules.Versions {
+		currentVersion, err := version.NewVersion(moduleVersion.Version)
+		if err != nil {
+			// if error is received for a version, then skip the current version
+			zap.S().Errorf("invalid version: %s, for module: %s, at registry: %s", moduleVersion.Version, module.String(), module.Host().Display())
+			continue
+		}
+
+		if requiredVersion.Required == nil {
+			// skip the pre release version
+			if currentVersion.Prerelease() != "" {
+				continue
+			}
+
+			// update the latest version
+			latestVersion = getGreaterVersion(latestVersion, currentVersion)
+		} else {
+			// skip the pre-release version, unless specified in the tf file
+			if currentVersion.Prerelease() != "" && requiredVersion.Required.String() != currentVersion.String() {
+				continue
+			}
+
+			// update the latest version
+			latestVersion = getGreaterVersion(latestVersion, currentVersion)
+
+			// update latest match
+			if requiredVersion.Required.Check(currentVersion) {
+				latestMatch = getGreaterVersion(latestMatch, currentVersion)
+			}
+		}
+	}
+
+	if latestVersion == nil {
+		return nil, fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display())
+	}
+
+	if requiredVersion.Required != nil && latestMatch == nil {
+		return nil, fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String())
+	}
+
+	versionToDownload = latestVersion
+	if latestMatch != nil {
+		versionToDownload = latestMatch
+	}
+
+	return versionToDownload, nil
+}