Support additional services for gcp provider (#528)

* Support additional services for gcp provider

* gcp services refactor

* Add service field to json output

internal/runner/options.go

@@ -183,6 +183,14 @@ const defaultProviderConfigFile = `#  #Provider configuration file for cloudlist
 #  # aws_session_token session token for temporary security credentials retrieved via STS (optional)
 #  aws_session_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
+#- # provider is the name of the provider
+#  provider: gcp
+#  # id is the name of the provider id
+#  id: staging
+#  # gcp_service_account_key is the service account key for gcp account
+#  gcp_service_account_key: |
+#    json data here
+
 #- # provider is the name of the provider
 #  provider: azure
 #  # id is the name of the provider id

pkg/providers/alibaba/instances.go

@@ -13,6 +13,10 @@ type instanceProvider struct {
 	client *ecs.Client
 }
 
+func (d *instanceProvider) name() string {
+	return "instance"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	list := schema.NewResources()
@@ -39,6 +43,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources,
 			PublicIPv4:  ipv4,
 			PrivateIpv4: privateIPv4,
 			Public:      ipv4 != "",
+			Service:     d.name(),
 		})
 	}
 

pkg/providers/aws/instances.go

@@ -19,6 +19,10 @@ type instanceProvider struct {
 	regions   *ec2.DescribeRegionsOutput
 }
 
+func (d *instanceProvider) name() string {
+	return "instance"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	list := schema.NewResources()
@@ -53,13 +57,15 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources,
 							Provider:    providerName,
 							PrivateIpv4: privateIp4,
 							Public:      false,
+							Service:    d.name(),
 						})
 					}
 					list.Append(&schema.Resource{
 						ID:         d.id,
 						Provider:   providerName,
 						PublicIPv4: ip4,
 						Public:     true,
+						Service:   d.name(),
 					})
 				}
 			}

pkg/providers/aws/route53.go

@@ -18,6 +18,10 @@ type route53Provider struct {
 	session *session.Session
 }
 
+func (d *route53Provider) name() string {
+	return "route53"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (d *route53Provider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	list := schema.NewResources()
@@ -69,12 +73,14 @@ func (d *route53Provider) listResourceRecords(zone *route53.HostedZone) (*schema
 				Public:   public,
 				DNSName:  name,
 				Provider: providerName,
+				Service: d.name(),
 			})
 			list.Append(&schema.Resource{
 				ID:         d.id,
 				Public:     public,
 				PublicIPv4: ip4,
 				Provider:   providerName,
+				Service:    d.name(),
 			})
 		}
 		if aws.BoolValue(sets.IsTruncated) && *sets.NextRecordName != "" {

pkg/providers/aws/s3.go

@@ -18,6 +18,10 @@ type s3Provider struct {
 	session *session.Session
 }
 
+func (d *s3Provider) name() string {
+	return "s3"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (d *s3Provider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	list := schema.NewResources()
@@ -38,6 +42,7 @@ func (d *s3Provider) GetResource(ctx context.Context) (*schema.Resources, error)
 			Public:   true,
 			DNSName:  endpointBuilder.String(),
 			Provider: providerName,
+			Service:  d.name(),
 		})
 	}
 

pkg/providers/azure/publicips.go

@@ -15,6 +15,10 @@ type publicIPProvider struct {
 	Authorizer     autorest.Authorizer
 }
 
+func (pip *publicIPProvider) name() string {
+	return "publicip"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (pip *publicIPProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 
@@ -36,6 +40,7 @@ func (pip *publicIPProvider) GetResource(ctx context.Context) (*schema.Resources
 			PublicIPv4: *ip.IPAddress,
 			ID:         pip.id,
 			Public:     true,
+			Service:    pip.name(),
 		})
 	}
 	return list, nil

pkg/providers/azure/vm.go

@@ -19,6 +19,10 @@ type vmProvider struct {
 	Authorizer     autorest.Authorizer
 }
 
+func (d *vmProvider) name() string {
+	return "vm"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 
@@ -77,6 +81,7 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error)
 						PublicIPv4:  *publicIP.IPAddress,
 						ID:          d.id,
 						PrivateIpv4: *privateIP,
+						Service:    d.name(),
 					})
 				}
 			}

pkg/providers/cloudflare/dns.go

@@ -14,6 +14,10 @@ type dnsProvider struct {
 	client *cloudflare.API
 }
 
+func (d *dnsProvider) name() string {
+	return "dns"
+}
+
 // GetResource returns all the resources in the store for a provider.
 func (d *dnsProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	list := schema.NewResources()
@@ -37,6 +41,7 @@ func (d *dnsProvider) GetResource(ctx context.Context) (*schema.Resources, error
 				Provider: providerName,
 				DNSName:  record.Name,
 				ID:       d.id,
+				Service: d.name(),
 			})
 			// Skip CNAME records values to discard duplidate data
 			if record.Type == "CNAME" {
@@ -47,6 +52,7 @@ func (d *dnsProvider) GetResource(ctx context.Context) (*schema.Resources, error
 				Provider:   providerName,
 				PublicIPv4: record.Content,
 				ID:         d.id,
+				Service:   d.name(),
 			})
 		}
 	}

pkg/providers/consul/resources.go

@@ -36,6 +36,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources,
 			list.Append(&schema.Resource{
 				Provider:   providerName,
 				ID:         d.id,
+				Service:    "consul_node",
 				PublicIPv4: node.Address,
 			})
 		}
@@ -63,6 +64,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources,
 				list.Append(&schema.Resource{
 					Provider:   providerName,
 					ID:         d.id,
+					Service:    item.ServiceName,
 					PublicIPv4: net.JoinHostPort(nodeIP, strconv.Itoa(port)),
 				})
 			}

pkg/providers/digitalocean/apps.go

@@ -13,6 +13,10 @@ type appsProvider struct {
 	client *godo.Client
 }
 
+func (d *appsProvider) name() string {
+	return "app"
+}
+
 // GetInstances returns all the instances in the store for a provider.
 func (d *appsProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	opt := &godo.ListOptions{PerPage: 200}
@@ -32,6 +36,7 @@ func (d *appsProvider) GetResource(ctx context.Context) (*schema.Resources, erro
 				ID:       d.id,
 				DNSName:  dnsname,
 				Public:   true,
+				Service:  d.name(),
 			})
 		}
 		if resp.Links == nil || resp.Links.IsLastPage() {

pkg/providers/digitalocean/instances.go

@@ -13,6 +13,10 @@ type instanceProvider struct {
 	client *godo.Client
 }
 
+func (d *instanceProvider) name() string {
+	return "instance"
+}
+
 // GetInstances returns all the instances in the store for a provider.
 func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
 	opt := &godo.ListOptions{PerPage: 200}
@@ -33,13 +37,15 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources,
 					Provider:    providerName,
 					ID:          d.id,
 					PrivateIpv4: privateIP4,
+					Service:     d.name(),
 				})
 			}
 			list.Append(&schema.Resource{
 				Provider:   providerName,
 				ID:         d.id,
 				PublicIPv4: ip4,
 				Public:     true,
+				Service:    d.name(),
 			})
 		}
 		if resp.Links == nil || resp.Links.IsLastPage() {

pkg/providers/fastly/services.go

@@ -33,6 +33,7 @@ func (d *serviceProvider) GetResource(ctx context.Context) (*schema.Resources, e
 				Provider: providerName,
 				DNSName:  domain.Name,
 				ID:       d.id,
+				Service: service.Name,
 			})
 		}
 	}

pkg/providers/gcp/bucket.go

@@ -0,0 +1,68 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"google.golang.org/api/storage/v1"
+)
+
+type cloudStorageProvider struct {
+	id       string
+	storage  *storage.Service
+	projects []string
+}
+
+func (d *cloudStorageProvider) name() string {
+	return "s3"
+}
+
+// GetResource returns all the storage resources in the store for a provider.
+func (d *cloudStorageProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
+	list := schema.NewResources()
+
+	buckets, err := d.getBuckets()
+	if err != nil {
+		return nil, fmt.Errorf("could not get buckets: %s", err)
+	}
+	for _, bucket := range buckets {
+		resource := &schema.Resource{
+			ID:       d.id,
+			Provider: providerName,
+			DNSName:  fmt.Sprintf("%s.storage.googleapis.com", bucket.Name),
+			Public:   d.isBucketPublic(bucket.Name),
+			Service:  d.name(),
+		}
+		list.Append(resource)
+	}
+	return list, nil
+}
+
+func (d *cloudStorageProvider) getBuckets() ([]*storage.Bucket, error) {
+	var buckets []*storage.Bucket
+	for _, project := range d.projects {
+		bucketsService := d.storage.Buckets.List(project)
+		_ = bucketsService.Pages(context.Background(), func(bal *storage.Buckets) error {
+			buckets = append(buckets, bal.Items...)
+			return nil
+		})
+	}
+	return buckets, nil
+}
+
+func (d *cloudStorageProvider) isBucketPublic(bucketName string) bool {
+	bucketIAMPolicy, err := d.storage.Buckets.GetIamPolicy(bucketName).Do()
+	if err == nil {
+		for _, binding := range bucketIAMPolicy.Bindings {
+			if binding.Role == "roles/storage.objectViewer" {
+				for _, member := range binding.Members {
+					if member == "allUsers" || member == "allAuthenticatedUsers" {
+						return true
+					}
+				}
+			}
+		}
+	}
+	return false
+}

pkg/providers/gcp/cloud-run.go

@@ -0,0 +1,79 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	run "google.golang.org/api/run/v1"
+)
+
+type cloudRunProvider struct {
+	id       string
+	run      *run.APIService
+	projects []string
+}
+
+func (d *cloudRunProvider) name() string {
+	return "cloud-run"
+}
+
+// GetResource returns all the Cloud Run resources in the store for a provider.
+func (d *cloudRunProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
+	list := schema.NewResources()
+	services, err := d.getServices()
+	if err != nil {
+		return nil, fmt.Errorf("could not get services: %s", err)
+	}
+
+	for _, service := range services {
+		serviceUrl, _ := url.Parse(service.Status.Url)
+		resource := &schema.Resource{
+			ID:       d.id,
+			Provider: providerName,
+			DNSName:  serviceUrl.Hostname(),
+			Public:   d.isPublicService(service.Metadata.Name),
+			Service:  d.name(),
+		}
+		list.Append(resource)
+	}
+	return list, nil
+}
+
+func (d *cloudRunProvider) getServices() ([]*run.Service, error) {
+	var services []*run.Service
+	for _, project := range d.projects {
+		locationsService := d.run.Projects.Locations.List(fmt.Sprintf("projects/%s", project))
+		locationsResponse, err := locationsService.Do()
+		if err != nil {
+			continue
+		}
+
+		for _, location := range locationsResponse.Locations {
+			servicesService := d.run.Projects.Locations.Services.List(location.Name)
+			servicesResponse, err := servicesService.Do()
+			if err != nil {
+				continue
+			}
+			services = append(services, servicesResponse.Items...)
+		}
+	}
+	return services, nil
+}
+
+func (d *cloudRunProvider) isPublicService(serviceName string) bool {
+	serviceIAMPolicy, err := d.run.Projects.Locations.Services.GetIamPolicy(serviceName).Do()
+	if err == nil {
+		for _, binding := range serviceIAMPolicy.Bindings {
+			if binding.Role == "roles/run.invoker" {
+				for _, member := range binding.Members {
+					if member == "allUsers" || member == "allAuthenticatedUsers" {
+						return true
+					}
+				}
+			}
+		}
+	}
+	return false
+}