Merge branch 'dev'

go.mod

@@ -25,11 +25,11 @@ require (
 	github.com/linode/linodego v1.22.0
 	github.com/namecheap/go-namecheap-sdk/v2 v2.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/goflags v0.1.40
+	github.com/projectdiscovery/goflags v0.1.45
 	github.com/projectdiscovery/gologger v1.1.12
-	github.com/projectdiscovery/utils v0.0.79
+	github.com/projectdiscovery/utils v0.0.85
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.14
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/oauth2 v0.12.0
 	google.golang.org/api v0.126.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -64,13 +64,13 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.4 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
@@ -107,6 +107,7 @@ require (
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
@@ -129,35 +130,45 @@ require (
 	github.com/pborman/uuid v1.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
+	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
 	github.com/prometheus/client_golang v1.15.1 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
+	github.com/shirou/gopsutil/v3 v3.23.7 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/gjson v1.14.3 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yuin/goldmark v1.5.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	google.golang.org/grpc v1.55.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.6 // indirect

internal/runner/banner.go

@@ -13,7 +13,7 @@ const banner = `
 `
 
 // version is the current version of cloudlist
-const version = `1.0.7`
+const version = `1.0.8`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

pkg/inventory/inventory.go

@@ -21,7 +21,6 @@ import (
 	"github.com/projectdiscovery/cloudlist/pkg/providers/scaleway"
 	"github.com/projectdiscovery/cloudlist/pkg/providers/terraform"
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
-	"github.com/projectdiscovery/gologger"
 )
 
 // Inventory is an inventory of providers
@@ -38,11 +37,9 @@ func New(options schema.Options) (*Inventory, error) {
 		if !ok {
 			continue
 		}
-		id, _ := block.GetMetadata("id")
 		provider, err := nameToProvider(value, block)
 		if err != nil {
-			gologger.Warning().Msgf("Could not initialize provider %s %s: %s\n", value, id, err)
-			continue
+			return nil, fmt.Errorf("could not create provider %s: %s", value, err)
 		}
 		inventory.Providers = append(inventory.Providers, provider)
 	}

pkg/providers/gcp/auth.go

@@ -47,7 +47,7 @@ func register(ctx context.Context, data []byte) (option.ClientOption, error) {
 
 	// Authenticate with the token
 	// If it's nil use Google ADC
-	err = rest.RegisterAuthProviderPlugin(googleAuthPlugin,
+	_ = rest.RegisterAuthProviderPlugin(googleAuthPlugin,
 		func(clusterAddress string, config map[string]string, persister rest.AuthProviderConfigPersister) (rest.AuthProvider, error) {
 			var err error
 			if tokenSource == nil {
@@ -58,9 +58,6 @@ func register(ctx context.Context, data []byte) (option.ClientOption, error) {
 			}
 			return &googleAuthProvider{tokenSource: tokenSource}, nil
 		})
-	if err != nil {
-		return nil, errorutil.NewWithErr(err).Msgf("failed to register %s auth plugin", googleAuthPlugin)
-	}
 	// return clioptions
 	return option.WithCredentialsJSON(data), nil
 }

pkg/providers/gcp/gcp.go

@@ -4,8 +4,10 @@ import (
 	"context"
 
 	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"github.com/projectdiscovery/gologger"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	"google.golang.org/api/cloudresourcemanager/v1"
+	"google.golang.org/api/compute/v1"
 	container "google.golang.org/api/container/v1beta1"
 	"google.golang.org/api/dns/v1"
 )
@@ -14,6 +16,7 @@ import (
 type Provider struct {
 	dns      *dns.Service
 	gke      *container.Service
+	compute  *compute.Service
 	id       string
 	projects []string
 }
@@ -47,7 +50,10 @@ func New(options schema.OptionBlock) (*Provider, error) {
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("could not create dns service with api key")
 	}
-
+	computeService, err := compute.NewService(context.Background(), creds)
+	if err != nil {
+		return nil, errorutil.NewWithErr(err).Msgf("could not create compute service with api key")
+	}
 	containerService, err := container.NewService(context.Background(), creds)
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("could not create container service with api key")
@@ -65,23 +71,33 @@ func New(options schema.OptionBlock) (*Provider, error) {
 		}
 		return nil
 	})
-	return &Provider{dns: dnsService, gke: containerService, projects: projects, id: id}, err
+	return &Provider{dns: dnsService, gke: containerService, projects: projects, id: id, compute: computeService}, err
 }
 
 // Resources returns the provider for an resource deployment source.
 func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) {
 	finalList := schema.NewResources()
+
 	cloudDNSProvider := &cloudDNSProvider{dns: p.dns, id: p.id, projects: p.projects}
 	zones, err := cloudDNSProvider.GetResource(ctx)
 	if err != nil {
 		return nil, err
 	}
 	finalList.Merge(zones)
+
 	GKEProvider := &gkeProvider{svc: p.gke, id: p.id, projects: p.projects}
 	gkeData, err := GKEProvider.GetResource(ctx)
 	if err != nil {
-		return nil, err
+		gologger.Warning().Msgf("Could not get GKE resources: %s\n", err)
 	}
 	finalList.Merge(gkeData)
+
+	VMProvider := &cloudVMProvider{compute: p.compute, id: p.id, projects: p.projects}
+	vmData, err := VMProvider.GetResource(ctx)
+	if err != nil {
+		return nil, err
+	}
+	finalList.Merge(vmData)
+
 	return finalList, nil
 }

pkg/providers/gcp/gke.go

@@ -43,7 +43,10 @@ func (d *gkeProvider) GetResource(ctx context.Context) (*schema.Resources, error
 			if err != nil {
 				return nil, fmt.Errorf("failed to create Kubernetes client cluster=%s: %w", clusterName, err)
 			}
-			ingress, err := k8sClient.NetworkingV1().Ingresses("").List(ctx, metav1.ListOptions{})
+			timeoutSeconds := int64(10)
+			ingress, err := k8sClient.NetworkingV1().Ingresses("").List(ctx, metav1.ListOptions{
+				TimeoutSeconds: &timeoutSeconds,
+			})
 			if err != nil {
 				return nil, errors.Wrap(err, "could not list kubernetes ingress")
 			}

pkg/providers/gcp/vms.go

@@ -0,0 +1,53 @@
+package gcp
+
+import (
+	"context"
+	"log"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"google.golang.org/api/compute/v1"
+)
+
+type cloudVMProvider struct {
+	id       string
+	compute  *compute.Service
+	projects []string
+}
+
+// GetResource returns all the resources in the store for a provider.
+func (d *cloudVMProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
+	list := schema.NewResources()
+
+	for _, project := range d.projects {
+		instances := d.compute.Instances.AggregatedList(project)
+		err := instances.Pages(context.Background(), func(ial *compute.InstanceAggregatedList) error {
+			for _, instancesScopedList := range ial.Items {
+				for _, instance := range instancesScopedList.Instances {
+					instance := instance
+
+					if len(instance.NetworkInterfaces) == 0 {
+						continue
+					}
+					nic := instance.NetworkInterfaces[0]
+					if len(nic.AccessConfigs) == 0 {
+						continue
+					}
+					cfg := nic.AccessConfigs[0]
+
+					list.Append(&schema.Resource{
+						ID:         d.id,
+						Public:     true,
+						Provider:   providerName,
+						PublicIPv4: cfg.NatIP,
+					})
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			log.Printf("Could not get all instances for project %s: %s\n", project, err)
+			continue
+		}
+	}
+	return list, nil
+}

pkg/schema/schema.go

@@ -97,6 +97,9 @@ func (r *Resources) Append(resource *Resource) {
 
 // Merge merges a list of resources into the main list
 func (r *Resources) Merge(resources *Resources) {
+	if resources == nil {
+		return
+	}
 	mergeUniqueMap := &sync.Map{}
 	for _, item := range resources.Items {
 		r.appendResource(item, mergeUniqueMap)