merge v0.8.0 into main

.circleci/config.yml

@@ -1,6 +1,5 @@
 version: 2.1
 orbs:
-  aws-cli: circleci/aws-cli@1.3.0
   go: circleci/go@1.5.0
 jobs:
   build:
@@ -167,15 +166,18 @@ jobs:
             git commit -m "Updated to version ${CIRCLE_TAG}"
             git push
   update-lambda:
-    executor: aws-cli/default
     environment:
         FUNCTION_NAME: driftctl-version
+    docker:
+        - image: cimg/base:2021.04
     steps:
-      - aws-cli/install
       - run:
           name: "Update Lambda version"
           command: |
-            aws lambda update-function-configuration --function-name $FUNCTION_NAME --environment "{\"Variables\":{\"LATEST_VERSION\":\"$CIRCLE_TAG\"}}"
+              wget "https://github.com/cloudskiff/lambda-env-updater/releases/download/v1.0.0/lambda-env-updater_linux_amd64" && chmod +x lambda-env-updater_linux_amd64
+              ./lambda-env-updater_linux_amd64\
+                -name ${FUNCTION_NAME}\
+                -env "LATEST_VERSION=${CIRCLE_TAG}"
 workflows:
   nightly:
     jobs:

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/aws/aws-sdk-go v1.34.2
+	github.com/bmatcuk/doublestar/v4 v4.0.1 // indirect
 	github.com/eapache/go-resiliency v1.2.0
 	github.com/fatih/color v1.9.0
 	github.com/getsentry/sentry-go v0.10.0

go.sum

@@ -162,6 +162,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
 github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v4 v4.0.1 h1:v5DFrvGpNnIKPlG7gcF4TlceHwBTvHdmjgDEkbDk9t8=
+github.com/bmatcuk/doublestar/v4 v4.0.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=

pkg/analyser/analysis.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/r3labs/diff/v2"
 
@@ -39,6 +40,7 @@ type Analysis struct {
 	differences []Difference
 	summary     Summary
 	alerts      alerter.Alerts
+	Duration    time.Duration
 }
 
 type serializableDifference struct {

pkg/cmd/driftctl.go

@@ -62,6 +62,7 @@ func NewDriftctlCmd(build build.BuildInterface) *DriftctlCmd {
 
 	cmd.PersistentFlags().BoolP("help", "h", false, "Display help for command")
 	cmd.PersistentFlags().BoolP("no-version-check", "", false, "Disable the version check")
+	cmd.PersistentFlags().BoolP("disable-telemetry", "", false, "Disable telemetry")
 	cmd.PersistentFlags().BoolP("send-crash-report", "", false, "Enable error reporting. Crash data will be sent to us via Sentry.\nWARNING: may leak sensitive data (please read the documentation for more details)\nThis flag should be used only if an error occurs during execution")
 
 	cmd.AddCommand(NewScanCmd())

pkg/cmd/driftctl_test.go

@@ -99,7 +99,7 @@ func TestDriftctlCmd_Scan(t *testing.T) {
 			env: map[string]string{
 				"DCTL_FROM": "test",
 			},
-			err: fmt.Errorf("Unable to parse from flag 'test': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://"),
+			err: fmt.Errorf("Unable to parse from flag 'test': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://,tfstate+tfcloud://"),
 		},
 		{
 			env: map[string]string{

pkg/cmd/scan.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/cloudskiff/driftctl/pkg/telemetry"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -75,6 +76,7 @@ func NewScanCmd() *cobra.Command {
 			}
 
 			opts.Quiet, _ = cmd.Flags().GetBool("quiet")
+			opts.DisableTelemetry, _ = cmd.Flags().GetBool("disable-telemetry")
 
 			return nil
 		},
@@ -128,6 +130,12 @@ func NewScanCmd() *cobra.Command {
 		"Use those HTTP headers to query the provided URL.\n"+
 			"Only used with tfstate+http(s) backend for now.\n",
 	)
+	fl.StringVar(&opts.BackendOptions.TFCloudToken,
+		"tfc-token",
+		"",
+		"Terraform Cloud / Enterprise API token.\n"+
+			"Only used with tfstate+tfcloud backend.\n",
+	)
 	fl.BoolVar(&opts.StrictMode,
 		"strict",
 		false,
@@ -147,11 +155,12 @@ func scanRun(opts *pkg.ScanOptions) error {
 	providerLibrary := terraform.NewProviderLibrary()
 	supplierLibrary := resource.NewSupplierLibrary()
 
-	progress := globaloutput.NewProgress()
+	iacProgress := globaloutput.NewProgress("Scanning states", "Scanned states", true)
+	scanProgress := globaloutput.NewProgress("Scanning resources", "Scanned resources", true)
 
 	resourceSchemaRepository := resource.NewSchemaRepository()
 
-	err := remote.Activate(opts.To, alerter, providerLibrary, supplierLibrary, progress, resourceSchemaRepository)
+	err := remote.Activate(opts.To, alerter, providerLibrary, supplierLibrary, scanProgress, resourceSchemaRepository)
 	if err != nil {
 		return err
 	}
@@ -165,25 +174,22 @@ func scanRun(opts *pkg.ScanOptions) error {
 
 	scanner := pkg.NewScanner(supplierLibrary.Suppliers(), alerter, resourceSchemaRepository)
 
-	iacSupplier, err := supplier.GetIACSupplier(opts.From, providerLibrary, opts.BackendOptions, resourceSchemaRepository)
+	iacSupplier, err := supplier.GetIACSupplier(opts.From, providerLibrary, opts.BackendOptions, iacProgress, resourceSchemaRepository)
 	if err != nil {
 		return err
 	}
 
 	resFactory := terraform.NewTerraformResourceFactory(providerLibrary, resourceSchemaRepository)
 
-	ctl := pkg.NewDriftCTL(scanner, iacSupplier, alerter, resFactory, opts, resourceSchemaRepository)
+	ctl := pkg.NewDriftCTL(scanner, iacSupplier, alerter, resFactory, opts, scanProgress, iacProgress, resourceSchemaRepository)
 
 	go func() {
 		<-c
 		logrus.Warn("Detected interrupt, cleanup ...")
 		ctl.Stop()
 	}()
 
-	progress.Start()
 	analysis, err := ctl.Run()
-	progress.Stop()
-
 	if err != nil {
 		return err
 	}
@@ -193,6 +199,10 @@ func scanRun(opts *pkg.ScanOptions) error {
 		return err
 	}
 
+	if !opts.DisableTelemetry {
+		telemetry.SendTelemetry(analysis)
+	}
+
 	if !analysis.IsSync() {
 		return cmderrors.InfrastructureNotInSync{}
 	}

pkg/cmd/scan_test.go

@@ -39,6 +39,8 @@ func TestScanCmd_Valid(t *testing.T) {
 		{args: []string{"scan", "-t", "aws+tf", "-f", "tfstate://test"}},
 		{args: []string{"scan", "--to", "aws+tf", "--from", "tfstate://test"}},
 		{args: []string{"scan", "--to", "aws+tf", "--from", "tfstate+https://github.com/state.tfstate"}},
+		{args: []string{"scan", "--to", "aws+tf", "--from", "tfstate+tfcloud://workspace_id"}},
+		{args: []string{"scan", "--tfc-token", "token"}},
 		{args: []string{"scan", "--filter", "Type=='aws_s3_bucket'"}},
 		{args: []string{"scan", "--strict"}},
 	}
@@ -69,14 +71,14 @@ func TestScanCmd_Invalid(t *testing.T) {
 		{args: []string{"scan", "-f"}, expected: `flag needs an argument: 'f' in -f`},
 		{args: []string{"scan", "--from"}, expected: `flag needs an argument: --from`},
 		{args: []string{"scan", "--from"}, expected: `flag needs an argument: --from`},
-		{args: []string{"scan", "--from", "tosdgjhgsdhgkjs"}, expected: "Unable to parse from flag 'tosdgjhgsdhgkjs': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://"},
-		{args: []string{"scan", "--from", "://"}, expected: "Unable to parse from flag '://': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://"},
-		{args: []string{"scan", "--from", "://test"}, expected: "Unable to parse from flag '://test': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://"},
-		{args: []string{"scan", "--from", "tosdgjhgsdhgkjs://"}, expected: "Unable to parse from flag 'tosdgjhgsdhgkjs://': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://"},
-		{args: []string{"scan", "--from", "terraform+foo+bar://test"}, expected: "Unable to parse from scheme 'terraform+foo+bar': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://"},
+		{args: []string{"scan", "--from", "tosdgjhgsdhgkjs"}, expected: "Unable to parse from flag 'tosdgjhgsdhgkjs': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://,tfstate+tfcloud://"},
+		{args: []string{"scan", "--from", "://"}, expected: "Unable to parse from flag '://': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://,tfstate+tfcloud://"},
+		{args: []string{"scan", "--from", "://test"}, expected: "Unable to parse from flag '://test': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://,tfstate+tfcloud://"},
+		{args: []string{"scan", "--from", "tosdgjhgsdhgkjs://"}, expected: "Unable to parse from flag 'tosdgjhgsdhgkjs://': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://,tfstate+tfcloud://"},
+		{args: []string{"scan", "--from", "terraform+foo+bar://test"}, expected: "Unable to parse from scheme 'terraform+foo+bar': \nAccepted schemes are: tfstate://,tfstate+s3://,tfstate+http://,tfstate+https://,tfstate+tfcloud://"},
 		{args: []string{"scan", "--from", "unsupported://test"}, expected: "Unsupported IaC source 'unsupported': \nAccepted values are: tfstate"},
-		{args: []string{"scan", "--from", "tfstate+foobar://test"}, expected: "Unsupported IaC backend 'foobar': \nAccepted values are: s3,http,https"},
-		{args: []string{"scan", "--from", "tfstate:///tmp/test", "--from", "tfstate+toto://test"}, expected: "Unsupported IaC backend 'toto': \nAccepted values are: s3,http,https"},
+		{args: []string{"scan", "--from", "tfstate+foobar://test"}, expected: "Unsupported IaC backend 'foobar': \nAccepted values are: s3,http,https,tfcloud"},
+		{args: []string{"scan", "--from", "tfstate:///tmp/test", "--from", "tfstate+toto://test"}, expected: "Unsupported IaC backend 'toto': \nAccepted values are: s3,http,https,tfcloud"},
 		{args: []string{"scan", "--filter", "Type='test'"}, expected: "unable to parse filter expression: SyntaxError: Expected tRbracket, received: tUnknown"},
 		{args: []string{"scan", "--filter", "Type='test'", "--filter", "Type='test2'"}, expected: "Filter flag should be specified only once"},
 	}

pkg/driftctl.go

@@ -2,7 +2,9 @@ package pkg
 
 import (
 	"fmt"
+	"time"
 
+	globaloutput "github.com/cloudskiff/driftctl/pkg/output"
 	"github.com/jmespath/go-jmespath"
 	"github.com/sirupsen/logrus"
 
@@ -17,15 +19,16 @@ import (
 )
 
 type ScanOptions struct {
-	Coverage       bool
-	Detect         bool
-	From           []config.SupplierConfig
-	To             string
-	Output         output.OutputConfig
-	Filter         *jmespath.JMESPath
-	Quiet          bool
-	BackendOptions *backend.Options
-	StrictMode     bool
+	Coverage         bool
+	Detect           bool
+	From             []config.SupplierConfig
+	To               string
+	Output           output.OutputConfig
+	Filter           *jmespath.JMESPath
+	Quiet            bool
+	BackendOptions   *backend.Options
+	StrictMode       bool
+	DisableTelemetry bool
 }
 
 type DriftCTL struct {
@@ -36,10 +39,12 @@ type DriftCTL struct {
 	filter                   *jmespath.JMESPath
 	resourceFactory          resource.ResourceFactory
 	strictMode               bool
+	scanProgress             globaloutput.Progress
+	iacProgress              globaloutput.Progress
 	resourceSchemaRepository resource.SchemaRepositoryInterface
 }
 
-func NewDriftCTL(remoteSupplier resource.Supplier, iacSupplier resource.Supplier, alerter *alerter.Alerter, resFactory resource.ResourceFactory, opts *ScanOptions, resourceSchemaRepository resource.SchemaRepositoryInterface) *DriftCTL {
+func NewDriftCTL(remoteSupplier resource.Supplier, iacSupplier resource.Supplier, alerter *alerter.Alerter, resFactory resource.ResourceFactory, opts *ScanOptions, scanProgress globaloutput.Progress, iacProgress globaloutput.Progress, resourceSchemaRepository resource.SchemaRepositoryInterface) *DriftCTL {
 	return &DriftCTL{
 		remoteSupplier,
 		iacSupplier,
@@ -48,11 +53,14 @@ func NewDriftCTL(remoteSupplier resource.Supplier, iacSupplier resource.Supplier
 		opts.Filter,
 		resFactory,
 		opts.StrictMode,
+		scanProgress,
+		iacProgress,
 		resourceSchemaRepository,
 	}
 }
 
 func (d DriftCTL) Run() (*analyser.Analysis, error) {
+	start := time.Now()
 	remoteResources, resourcesFromState, err := d.scan()
 	if err != nil {
 		return nil, err
@@ -108,6 +116,7 @@ func (d DriftCTL) Run() (*analyser.Analysis, error) {
 	driftIgnore := filter.NewDriftIgnore()
 
 	analysis, err := d.analyzer.Analyze(remoteResources, resourcesFromState, driftIgnore)
+	analysis.Duration = time.Since(start)
 
 	if err != nil {
 		return nil, err
@@ -133,12 +142,16 @@ func (d DriftCTL) Stop() {
 
 func (d DriftCTL) scan() (remoteResources []resource.Resource, resourcesFromState []resource.Resource, err error) {
 	logrus.Info("Start reading IaC")
+	d.iacProgress.Start()
 	resourcesFromState, err = d.iacSupplier.Resources()
+	d.iacProgress.Stop()
 	if err != nil {
 		return nil, nil, err
 	}
 
 	logrus.Info("Start scanning cloud provider")
+	d.scanProgress.Start()
+	defer d.scanProgress.Stop()
 	remoteResources, err = d.remoteSupplier.Resources()
 	if err != nil {
 		return nil, nil, err

pkg/driftctl_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/cloudskiff/driftctl/pkg/alerter"
 	"github.com/cloudskiff/driftctl/pkg/analyser"
 	"github.com/cloudskiff/driftctl/pkg/filter"
+	"github.com/cloudskiff/driftctl/pkg/output"
 	"github.com/cloudskiff/driftctl/pkg/resource"
 	"github.com/cloudskiff/driftctl/pkg/resource/aws"
 	"github.com/cloudskiff/driftctl/pkg/resource/github"
@@ -72,11 +73,20 @@ func runTest(t *testing.T, cases TestCases) {
 				c.mocks(resourceFactory)
 			}
 
-			driftctl := pkg.NewDriftCTL(remoteSupplier, stateSupplier, testAlerter, resourceFactory, c.options, repo)
+			scanProgress := &output.MockProgress{}
+			scanProgress.On("Start").Return().Once()
+			scanProgress.On("Stop").Return().Once()
+
+			iacProgress := &output.MockProgress{}
+			iacProgress.On("Start").Return().Once()
+			iacProgress.On("Stop").Return().Once()
+
+			driftctl := pkg.NewDriftCTL(remoteSupplier, stateSupplier, testAlerter, resourceFactory, c.options, scanProgress, iacProgress, repo)
 
 			analysis, err := driftctl.Run()
 
 			c.assert(test.NewScanResult(t, analysis), err)
+			scanProgress.AssertExpectations(t)
 		})
 	}
 }
@@ -93,6 +103,15 @@ func matchByAttributes(input, attrs map[string]interface{}) bool {
 func TestDriftctlRun_BasicBehavior(t *testing.T) {
 
 	cases := TestCases{
+		{
+			name:            "analysis duration is set",
+			stateResources:  []resource.Resource{},
+			remoteResources: []resource.Resource{},
+			assert: func(result *test.ScanResult, err error) {
+				result.NotZero(result.Duration)
+			},
+			options: &pkg.ScanOptions{},
+		},
 		{
 			name: "infrastructure should be in sync",
 			stateResources: []resource.Resource{
@@ -296,7 +315,7 @@ func TestDriftctlRun_BasicBehavior(t *testing.T) {
 			},
 			assert: func(result *test.ScanResult, err error) {
 				result.AssertManagedCount(2)
-				result.AssertUnmanagedCount(1)
+				result.AssertUnmanagedCount(2)
 				result.AssertDeletedCount(0)
 				result.AssertDriftCountTotal(0)
 			},

pkg/http/client.go

@@ -0,0 +1,8 @@
+package http
+
+import "net/http"
+
+// HTTPClient is an interface for http.Client type
+type HTTPClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}