From a42a1a0a72ca9125e661870909e104d1802aba5f Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 13 Jun 2024 01:59:02 +0530 Subject: [PATCH 1/3] Support additional services for gcp provider --- internal/runner/options.go | 8 +++++ pkg/providers/gcp/bucket.go | 52 ++++++++++++++++++++++++++++ pkg/providers/gcp/cloud-run.go | 62 ++++++++++++++++++++++++++++++++++ pkg/providers/gcp/function.go | 54 +++++++++++++++++++++++++++++ pkg/providers/gcp/gcp.go | 51 ++++++++++++++++++++++++---- 5 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 pkg/providers/gcp/bucket.go create mode 100644 pkg/providers/gcp/cloud-run.go create mode 100644 pkg/providers/gcp/function.go diff --git a/internal/runner/options.go b/internal/runner/options.go index 4d5c5cda..3e5ba803 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -183,6 +183,14 @@ const defaultProviderConfigFile = `# #Provider configuration file for cloudlist # # aws_session_token session token for temporary security credentials retrieved via STS (optional) # aws_session_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +#- # provider is the name of the provider +# provider: gcp +# # id is the name of the provider id +# id: staging +# # gcp_service_account_key is the service account key for gcp account +# gcp_service_account_key: | +# json data here + #- # provider is the name of the provider # provider: azure # # id is the name of the provider id diff --git a/pkg/providers/gcp/bucket.go b/pkg/providers/gcp/bucket.go new file mode 100644 index 00000000..7ead73ef --- /dev/null +++ b/pkg/providers/gcp/bucket.go @@ -0,0 +1,52 @@ +package gcp + +import ( + "context" + "fmt" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + "google.golang.org/api/storage/v1" +) + +type cloudStorageProvider struct { + id string + storage *storage.Service + projects []string +} + +// GetResource returns all the storage resources in the store for a provider. +func (d *cloudStorageProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + for _, project := range d.projects { + bucketsService := d.storage.Buckets.List(project) + err := bucketsService.Pages(context.Background(), func(bal *storage.Buckets) error { + for _, bucket := range bal.Items { + resource := &schema.Resource{ + ID: d.id, + Provider: providerName, + DNSName: fmt.Sprintf("%s.storage.googleapis.com", bucket.Name), + } + bucketIAMPolicy, err := d.storage.Buckets.GetIamPolicy(bucket.Name).Do() + if err == nil { + for _, binding := range bucketIAMPolicy.Bindings { + if binding.Role == "roles/storage.objectViewer" { + for _, member := range binding.Members { + if member == "allUsers" || member == "allAuthenticatedUsers" { + resource.Public = true + break + } + } + } + } + } + list.Append(resource) + } + return nil + }) + if err != nil { + continue + } + } + return list, nil +} diff --git a/pkg/providers/gcp/cloud-run.go b/pkg/providers/gcp/cloud-run.go new file mode 100644 index 00000000..5f2fe9fb --- /dev/null +++ b/pkg/providers/gcp/cloud-run.go @@ -0,0 +1,62 @@ +package gcp + +import ( + "context" + "fmt" + "net/url" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + run "google.golang.org/api/run/v1" +) + +type cloudRunProvider struct { + id string + run *run.APIService + projects []string +} + +// GetResource returns all the Cloud Run resources in the store for a provider. +func (d *cloudRunProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + for _, project := range d.projects { + locationsService := d.run.Projects.Locations.List(fmt.Sprintf("projects/%s", project)) + locationsResponse, err := locationsService.Do() + if err != nil { + continue + } + + for _, location := range locationsResponse.Locations { + servicesService := d.run.Projects.Locations.Services.List(location.Name) + servicesResponse, err := servicesService.Do() + if err != nil { + continue + } + + for _, service := range servicesResponse.Items { + serviceUrl, _ := url.Parse(service.Status.Url) + resource := &schema.Resource{ + ID: d.id, + Provider: providerName, + DNSName: serviceUrl.Hostname(), + } + + serviceIAMPolicy, err := d.run.Projects.Locations.Services.GetIamPolicy(service.Metadata.Name).Do() + if err == nil { + for _, binding := range serviceIAMPolicy.Bindings { + if binding.Role == "roles/run.invoker" { + for _, member := range binding.Members { + if member == "allUsers" || member == "allAuthenticatedUsers" { + resource.Public = true + break + } + } + } + } + } + list.Append(resource) + } + } + } + return list, nil +} diff --git a/pkg/providers/gcp/function.go b/pkg/providers/gcp/function.go new file mode 100644 index 00000000..c36be391 --- /dev/null +++ b/pkg/providers/gcp/function.go @@ -0,0 +1,54 @@ +package gcp + +import ( + "context" + "fmt" + "net/url" + + "github.com/projectdiscovery/cloudlist/pkg/schema" + "google.golang.org/api/cloudfunctions/v1" +) + +type cloudFunctionsProvider struct { + id string + functions *cloudfunctions.Service + projects []string +} + +// GetResource returns all the Cloud Function resources in the store for a provider. +func (d *cloudFunctionsProvider) GetResource(ctx context.Context) (*schema.Resources, error) { + list := schema.NewResources() + + for _, project := range d.projects { + functionsService := d.functions.Projects.Locations.Functions.List(fmt.Sprintf("projects/%s/locations/-", project)) + err := functionsService.Pages(context.Background(), func(fal *cloudfunctions.ListFunctionsResponse) error { + for _, function := range fal.Functions { + funcUrl, _ := url.Parse(function.HttpsTrigger.Url) + resource := &schema.Resource{ + ID: d.id, + Provider: providerName, + DNSName: funcUrl.Hostname(), + } + functionIAMPolicy, err := d.functions.Projects.Locations.Functions.GetIamPolicy(function.Name).Do() + if err == nil { + for _, binding := range functionIAMPolicy.Bindings { + if binding.Role == "roles/cloudfunctions.invoker" { + for _, member := range binding.Members { + if member == "allUsers" || member == "allAuthenticatedUsers" { + resource.Public = true + break + } + } + } + } + } + list.Append(resource) + } + return nil + }) + if err != nil { + continue + } + } + return list, nil +} diff --git a/pkg/providers/gcp/gcp.go b/pkg/providers/gcp/gcp.go index 92160cbe..c8831d8a 100644 --- a/pkg/providers/gcp/gcp.go +++ b/pkg/providers/gcp/gcp.go @@ -6,19 +6,25 @@ import ( "github.com/projectdiscovery/cloudlist/pkg/schema" "github.com/projectdiscovery/gologger" errorutil "github.com/projectdiscovery/utils/errors" + "google.golang.org/api/cloudfunctions/v1" "google.golang.org/api/cloudresourcemanager/v1" "google.golang.org/api/compute/v1" container "google.golang.org/api/container/v1beta1" "google.golang.org/api/dns/v1" + run "google.golang.org/api/run/v1" + "google.golang.org/api/storage/v1" ) // Provider is a data provider for gcp API type Provider struct { - dns *dns.Service - gke *container.Service - compute *compute.Service - id string - projects []string + dns *dns.Service + gke *container.Service + compute *compute.Service + storage *storage.Service + functions *cloudfunctions.Service + run *run.APIService + id string + projects []string } const serviceAccountJSON = "gcp_service_account_key" @@ -58,6 +64,18 @@ func New(options schema.OptionBlock) (*Provider, error) { if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create container service with api key") } + storageService, err := storage.NewService(context.Background(), creds) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create storage service with api key") + } + functionsService, err := cloudfunctions.NewService(context.Background(), creds) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create functions service with api key") + } + cloudRunService, err := run.NewService(context.Background(), creds) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create cloud run service with api key") + } projects := []string{} manager, err := cloudresourcemanager.NewService(context.Background(), creds) @@ -71,7 +89,7 @@ func New(options schema.OptionBlock) (*Provider, error) { } return nil }) - return &Provider{dns: dnsService, gke: containerService, projects: projects, id: id, compute: computeService}, err + return &Provider{dns: dnsService, gke: containerService, projects: projects, id: id, compute: computeService, storage: storageService, functions: functionsService, run: cloudRunService}, err } // Resources returns the provider for an resource deployment source. @@ -99,5 +117,26 @@ func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) { } finalList.Merge(vmData) + cloudStorageProvider := &cloudStorageProvider{id: p.id, storage: p.storage, projects: p.projects} + storageData, err := cloudStorageProvider.GetResource(ctx) + if err != nil { + return nil, err + } + finalList.Merge(storageData) + + cloudFunctionsProvider := &cloudFunctionsProvider{id: p.id, functions: p.functions, projects: p.projects} + functionsData, err := cloudFunctionsProvider.GetResource(ctx) + if err != nil { + return nil, err + } + finalList.Merge(functionsData) + + cloudRunProvider := &cloudRunProvider{id: p.id, run: p.run, projects: p.projects} + cloudRunData, err := cloudRunProvider.GetResource(ctx) + if err != nil { + return nil, err + } + finalList.Merge(cloudRunData) + return finalList, nil } From f771d9df7bb6bfefad87eb9a3ca2ba30c3229c17 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Tue, 25 Jun 2024 19:36:36 +0530 Subject: [PATCH 2/3] gcp services refactor --- pkg/providers/gcp/bucket.go | 59 +++++++++++++++++++-------------- pkg/providers/gcp/cloud-run.go | 54 ++++++++++++++++++------------ pkg/providers/gcp/function.go | 60 ++++++++++++++++++++-------------- 3 files changed, 103 insertions(+), 70 deletions(-) diff --git a/pkg/providers/gcp/bucket.go b/pkg/providers/gcp/bucket.go index 7ead73ef..143b380c 100644 --- a/pkg/providers/gcp/bucket.go +++ b/pkg/providers/gcp/bucket.go @@ -18,35 +18,46 @@ type cloudStorageProvider struct { func (d *cloudStorageProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() + buckets, err := d.getBuckets() + if err != nil { + return nil, fmt.Errorf("could not get buckets: %s", err) + } + for _, bucket := range buckets { + resource := &schema.Resource{ + ID: d.id, + Provider: providerName, + DNSName: fmt.Sprintf("%s.storage.googleapis.com", bucket.Name), + Public: d.isBucketPublic(bucket.Name), + } + list.Append(resource) + } + return list, nil +} + +func (d *cloudStorageProvider) getBuckets() ([]*storage.Bucket, error) { + var buckets []*storage.Bucket for _, project := range d.projects { bucketsService := d.storage.Buckets.List(project) - err := bucketsService.Pages(context.Background(), func(bal *storage.Buckets) error { - for _, bucket := range bal.Items { - resource := &schema.Resource{ - ID: d.id, - Provider: providerName, - DNSName: fmt.Sprintf("%s.storage.googleapis.com", bucket.Name), - } - bucketIAMPolicy, err := d.storage.Buckets.GetIamPolicy(bucket.Name).Do() - if err == nil { - for _, binding := range bucketIAMPolicy.Bindings { - if binding.Role == "roles/storage.objectViewer" { - for _, member := range binding.Members { - if member == "allUsers" || member == "allAuthenticatedUsers" { - resource.Public = true - break - } - } - } + _ = bucketsService.Pages(context.Background(), func(bal *storage.Buckets) error { + buckets = append(buckets, bal.Items...) + return nil + }) + } + return buckets, nil +} + +func (d *cloudStorageProvider) isBucketPublic(bucketName string) bool { + bucketIAMPolicy, err := d.storage.Buckets.GetIamPolicy(bucketName).Do() + if err == nil { + for _, binding := range bucketIAMPolicy.Bindings { + if binding.Role == "roles/storage.objectViewer" { + for _, member := range binding.Members { + if member == "allUsers" || member == "allAuthenticatedUsers" { + return true } } - list.Append(resource) } - return nil - }) - if err != nil { - continue } } - return list, nil + return false } diff --git a/pkg/providers/gcp/cloud-run.go b/pkg/providers/gcp/cloud-run.go index 5f2fe9fb..790e125a 100644 --- a/pkg/providers/gcp/cloud-run.go +++ b/pkg/providers/gcp/cloud-run.go @@ -18,7 +18,26 @@ type cloudRunProvider struct { // GetResource returns all the Cloud Run resources in the store for a provider. func (d *cloudRunProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() + services, err := d.getServices() + if err != nil { + return nil, fmt.Errorf("could not get services: %s", err) + } + + for _, service := range services { + serviceUrl, _ := url.Parse(service.Status.Url) + resource := &schema.Resource{ + ID: d.id, + Provider: providerName, + DNSName: serviceUrl.Hostname(), + Public: d.isPublicService(service.Metadata.Name), + } + list.Append(resource) + } + return list, nil +} +func (d *cloudRunProvider) getServices() ([]*run.Service, error) { + var services []*run.Service for _, project := range d.projects { locationsService := d.run.Projects.Locations.List(fmt.Sprintf("projects/%s", project)) locationsResponse, err := locationsService.Do() @@ -32,31 +51,24 @@ func (d *cloudRunProvider) GetResource(ctx context.Context) (*schema.Resources, if err != nil { continue } + services = append(services, servicesResponse.Items...) + } + } + return services, nil +} - for _, service := range servicesResponse.Items { - serviceUrl, _ := url.Parse(service.Status.Url) - resource := &schema.Resource{ - ID: d.id, - Provider: providerName, - DNSName: serviceUrl.Hostname(), - } - - serviceIAMPolicy, err := d.run.Projects.Locations.Services.GetIamPolicy(service.Metadata.Name).Do() - if err == nil { - for _, binding := range serviceIAMPolicy.Bindings { - if binding.Role == "roles/run.invoker" { - for _, member := range binding.Members { - if member == "allUsers" || member == "allAuthenticatedUsers" { - resource.Public = true - break - } - } - } +func (d *cloudRunProvider) isPublicService(serviceName string) bool { + serviceIAMPolicy, err := d.run.Projects.Locations.Services.GetIamPolicy(serviceName).Do() + if err == nil { + for _, binding := range serviceIAMPolicy.Bindings { + if binding.Role == "roles/run.invoker" { + for _, member := range binding.Members { + if member == "allUsers" || member == "allAuthenticatedUsers" { + return true } } - list.Append(resource) } } } - return list, nil + return false } diff --git a/pkg/providers/gcp/function.go b/pkg/providers/gcp/function.go index c36be391..9d39a35c 100644 --- a/pkg/providers/gcp/function.go +++ b/pkg/providers/gcp/function.go @@ -18,37 +18,47 @@ type cloudFunctionsProvider struct { // GetResource returns all the Cloud Function resources in the store for a provider. func (d *cloudFunctionsProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() + functions, err := d.getFunctions() + if err != nil { + return nil, fmt.Errorf("could not get functions: %s", err) + } + for _, function := range functions { + funcUrl, _ := url.Parse(function.HttpsTrigger.Url) + resource := &schema.Resource{ + ID: d.id, + Provider: providerName, + DNSName: funcUrl.Hostname(), + Public: d.isPublicFunction(function.Name), + } + list.Append(resource) + } + return list, nil +} +func (d *cloudFunctionsProvider) getFunctions() ([]*cloudfunctions.CloudFunction, error) { + var functions []*cloudfunctions.CloudFunction for _, project := range d.projects { functionsService := d.functions.Projects.Locations.Functions.List(fmt.Sprintf("projects/%s/locations/-", project)) - err := functionsService.Pages(context.Background(), func(fal *cloudfunctions.ListFunctionsResponse) error { - for _, function := range fal.Functions { - funcUrl, _ := url.Parse(function.HttpsTrigger.Url) - resource := &schema.Resource{ - ID: d.id, - Provider: providerName, - DNSName: funcUrl.Hostname(), - } - functionIAMPolicy, err := d.functions.Projects.Locations.Functions.GetIamPolicy(function.Name).Do() - if err == nil { - for _, binding := range functionIAMPolicy.Bindings { - if binding.Role == "roles/cloudfunctions.invoker" { - for _, member := range binding.Members { - if member == "allUsers" || member == "allAuthenticatedUsers" { - resource.Public = true - break - } - } - } + _ = functionsService.Pages(context.Background(), func(fal *cloudfunctions.ListFunctionsResponse) error { + functions = append(functions, fal.Functions...) + return nil + }) + } + return functions, nil +} + +func (d *cloudFunctionsProvider) isPublicFunction(functionName string) bool { + functionIAMPolicy, err := d.functions.Projects.Locations.Functions.GetIamPolicy(functionName).Do() + if err == nil { + for _, binding := range functionIAMPolicy.Bindings { + if binding.Role == "roles/cloudfunctions.invoker" { + for _, member := range binding.Members { + if member == "allUsers" || member == "allAuthenticatedUsers" { + return true } } - list.Append(resource) } - return nil - }) - if err != nil { - continue } } - return list, nil + return false } From 8a0b7e34a1933c6626c302233784030d70131e05 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 27 Jun 2024 10:12:38 +0530 Subject: [PATCH 3/3] Add service field to json output --- pkg/providers/alibaba/instances.go | 5 +++++ pkg/providers/aws/instances.go | 6 ++++++ pkg/providers/aws/route53.go | 6 ++++++ pkg/providers/aws/s3.go | 5 +++++ pkg/providers/azure/publicips.go | 5 +++++ pkg/providers/azure/vm.go | 5 +++++ pkg/providers/cloudflare/dns.go | 6 ++++++ pkg/providers/consul/resources.go | 2 ++ pkg/providers/digitalocean/apps.go | 5 +++++ pkg/providers/digitalocean/instances.go | 6 ++++++ pkg/providers/fastly/services.go | 1 + pkg/providers/gcp/bucket.go | 5 +++++ pkg/providers/gcp/cloud-run.go | 5 +++++ pkg/providers/gcp/dns.go | 5 +++++ pkg/providers/gcp/function.go | 5 +++++ pkg/providers/gcp/gke.go | 7 +++++++ pkg/providers/gcp/vms.go | 5 +++++ pkg/providers/heroku/applications.go | 5 +++++ pkg/providers/hetzner/instances.go | 6 ++++++ pkg/providers/k8s/ingress.go | 6 ++++++ pkg/providers/k8s/services.go | 8 ++++++++ pkg/providers/linode/instances.go | 5 +++++ pkg/providers/namecheap/domains.go | 5 +++++ pkg/providers/nomad/resources.go | 3 +++ pkg/providers/openstack/instances.go | 5 +++++ pkg/providers/scaleway/instances.go | 6 ++++++ pkg/providers/terraform/state_file.go | 5 +++++ pkg/schema/schema.go | 11 +++++++---- 28 files changed, 145 insertions(+), 4 deletions(-) diff --git a/pkg/providers/alibaba/instances.go b/pkg/providers/alibaba/instances.go index eaa2495a..67e0c587 100644 --- a/pkg/providers/alibaba/instances.go +++ b/pkg/providers/alibaba/instances.go @@ -13,6 +13,10 @@ type instanceProvider struct { client *ecs.Client } +func (d *instanceProvider) name() string { + return "instance" +} + // GetResource returns all the resources in the store for a provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -39,6 +43,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, PublicIPv4: ipv4, PrivateIpv4: privateIPv4, Public: ipv4 != "", + Service: d.name(), }) } diff --git a/pkg/providers/aws/instances.go b/pkg/providers/aws/instances.go index a460ce40..772ee5cc 100644 --- a/pkg/providers/aws/instances.go +++ b/pkg/providers/aws/instances.go @@ -19,6 +19,10 @@ type instanceProvider struct { regions *ec2.DescribeRegionsOutput } +func (d *instanceProvider) name() string { + return "instance" +} + // GetResource returns all the resources in the store for a provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -53,6 +57,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, PrivateIpv4: privateIp4, Public: false, + Service: d.name(), }) } list.Append(&schema.Resource{ @@ -60,6 +65,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, PublicIPv4: ip4, Public: true, + Service: d.name(), }) } } diff --git a/pkg/providers/aws/route53.go b/pkg/providers/aws/route53.go index b0cb7762..82ea0ccc 100644 --- a/pkg/providers/aws/route53.go +++ b/pkg/providers/aws/route53.go @@ -18,6 +18,10 @@ type route53Provider struct { session *session.Session } +func (d *route53Provider) name() string { + return "route53" +} + // GetResource returns all the resources in the store for a provider. func (d *route53Provider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -69,12 +73,14 @@ func (d *route53Provider) listResourceRecords(zone *route53.HostedZone) (*schema Public: public, DNSName: name, Provider: providerName, + Service: d.name(), }) list.Append(&schema.Resource{ ID: d.id, Public: public, PublicIPv4: ip4, Provider: providerName, + Service: d.name(), }) } if aws.BoolValue(sets.IsTruncated) && *sets.NextRecordName != "" { diff --git a/pkg/providers/aws/s3.go b/pkg/providers/aws/s3.go index f628c5e3..3c444bfc 100644 --- a/pkg/providers/aws/s3.go +++ b/pkg/providers/aws/s3.go @@ -18,6 +18,10 @@ type s3Provider struct { session *session.Session } +func (d *s3Provider) name() string { + return "s3" +} + // GetResource returns all the resources in the store for a provider. func (d *s3Provider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -38,6 +42,7 @@ func (d *s3Provider) GetResource(ctx context.Context) (*schema.Resources, error) Public: true, DNSName: endpointBuilder.String(), Provider: providerName, + Service: d.name(), }) } diff --git a/pkg/providers/azure/publicips.go b/pkg/providers/azure/publicips.go index 316bca29..be9b6b39 100644 --- a/pkg/providers/azure/publicips.go +++ b/pkg/providers/azure/publicips.go @@ -15,6 +15,10 @@ type publicIPProvider struct { Authorizer autorest.Authorizer } +func (pip *publicIPProvider) name() string { + return "publicip" +} + // GetResource returns all the resources in the store for a provider. func (pip *publicIPProvider) GetResource(ctx context.Context) (*schema.Resources, error) { @@ -36,6 +40,7 @@ func (pip *publicIPProvider) GetResource(ctx context.Context) (*schema.Resources PublicIPv4: *ip.IPAddress, ID: pip.id, Public: true, + Service: pip.name(), }) } return list, nil diff --git a/pkg/providers/azure/vm.go b/pkg/providers/azure/vm.go index 4871977b..702be09a 100644 --- a/pkg/providers/azure/vm.go +++ b/pkg/providers/azure/vm.go @@ -19,6 +19,10 @@ type vmProvider struct { Authorizer autorest.Authorizer } +func (d *vmProvider) name() string { + return "vm" +} + // GetResource returns all the resources in the store for a provider. func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) { @@ -77,6 +81,7 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) PublicIPv4: *publicIP.IPAddress, ID: d.id, PrivateIpv4: *privateIP, + Service: d.name(), }) } } diff --git a/pkg/providers/cloudflare/dns.go b/pkg/providers/cloudflare/dns.go index 3f0bf8e9..090dacdc 100644 --- a/pkg/providers/cloudflare/dns.go +++ b/pkg/providers/cloudflare/dns.go @@ -14,6 +14,10 @@ type dnsProvider struct { client *cloudflare.API } +func (d *dnsProvider) name() string { + return "dns" +} + // GetResource returns all the resources in the store for a provider. func (d *dnsProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -37,6 +41,7 @@ func (d *dnsProvider) GetResource(ctx context.Context) (*schema.Resources, error Provider: providerName, DNSName: record.Name, ID: d.id, + Service: d.name(), }) // Skip CNAME records values to discard duplidate data if record.Type == "CNAME" { @@ -47,6 +52,7 @@ func (d *dnsProvider) GetResource(ctx context.Context) (*schema.Resources, error Provider: providerName, PublicIPv4: record.Content, ID: d.id, + Service: d.name(), }) } } diff --git a/pkg/providers/consul/resources.go b/pkg/providers/consul/resources.go index 9367521a..cf112925 100644 --- a/pkg/providers/consul/resources.go +++ b/pkg/providers/consul/resources.go @@ -36,6 +36,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources, list.Append(&schema.Resource{ Provider: providerName, ID: d.id, + Service: "consul_node", PublicIPv4: node.Address, }) } @@ -63,6 +64,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources, list.Append(&schema.Resource{ Provider: providerName, ID: d.id, + Service: item.ServiceName, PublicIPv4: net.JoinHostPort(nodeIP, strconv.Itoa(port)), }) } diff --git a/pkg/providers/digitalocean/apps.go b/pkg/providers/digitalocean/apps.go index 37afc058..89cd622f 100644 --- a/pkg/providers/digitalocean/apps.go +++ b/pkg/providers/digitalocean/apps.go @@ -13,6 +13,10 @@ type appsProvider struct { client *godo.Client } +func (d *appsProvider) name() string { + return "app" +} + // GetInstances returns all the instances in the store for a provider. func (d *appsProvider) GetResource(ctx context.Context) (*schema.Resources, error) { opt := &godo.ListOptions{PerPage: 200} @@ -32,6 +36,7 @@ func (d *appsProvider) GetResource(ctx context.Context) (*schema.Resources, erro ID: d.id, DNSName: dnsname, Public: true, + Service: d.name(), }) } if resp.Links == nil || resp.Links.IsLastPage() { diff --git a/pkg/providers/digitalocean/instances.go b/pkg/providers/digitalocean/instances.go index aa89d767..13d65f91 100644 --- a/pkg/providers/digitalocean/instances.go +++ b/pkg/providers/digitalocean/instances.go @@ -13,6 +13,10 @@ type instanceProvider struct { client *godo.Client } +func (d *instanceProvider) name() string { + return "instance" +} + // GetInstances returns all the instances in the store for a provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { opt := &godo.ListOptions{PerPage: 200} @@ -33,6 +37,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, ID: d.id, PrivateIpv4: privateIP4, + Service: d.name(), }) } list.Append(&schema.Resource{ @@ -40,6 +45,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, ID: d.id, PublicIPv4: ip4, Public: true, + Service: d.name(), }) } if resp.Links == nil || resp.Links.IsLastPage() { diff --git a/pkg/providers/fastly/services.go b/pkg/providers/fastly/services.go index d775380c..8566b79e 100644 --- a/pkg/providers/fastly/services.go +++ b/pkg/providers/fastly/services.go @@ -33,6 +33,7 @@ func (d *serviceProvider) GetResource(ctx context.Context) (*schema.Resources, e Provider: providerName, DNSName: domain.Name, ID: d.id, + Service: service.Name, }) } } diff --git a/pkg/providers/gcp/bucket.go b/pkg/providers/gcp/bucket.go index 143b380c..1b8bf842 100644 --- a/pkg/providers/gcp/bucket.go +++ b/pkg/providers/gcp/bucket.go @@ -14,6 +14,10 @@ type cloudStorageProvider struct { projects []string } +func (d *cloudStorageProvider) name() string { + return "s3" +} + // GetResource returns all the storage resources in the store for a provider. func (d *cloudStorageProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -28,6 +32,7 @@ func (d *cloudStorageProvider) GetResource(ctx context.Context) (*schema.Resourc Provider: providerName, DNSName: fmt.Sprintf("%s.storage.googleapis.com", bucket.Name), Public: d.isBucketPublic(bucket.Name), + Service: d.name(), } list.Append(resource) } diff --git a/pkg/providers/gcp/cloud-run.go b/pkg/providers/gcp/cloud-run.go index 790e125a..a3f64eb4 100644 --- a/pkg/providers/gcp/cloud-run.go +++ b/pkg/providers/gcp/cloud-run.go @@ -15,6 +15,10 @@ type cloudRunProvider struct { projects []string } +func (d *cloudRunProvider) name() string { + return "cloud-run" +} + // GetResource returns all the Cloud Run resources in the store for a provider. func (d *cloudRunProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -30,6 +34,7 @@ func (d *cloudRunProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, DNSName: serviceUrl.Hostname(), Public: d.isPublicService(service.Metadata.Name), + Service: d.name(), } list.Append(resource) } diff --git a/pkg/providers/gcp/dns.go b/pkg/providers/gcp/dns.go index 5db4ecbf..22293c97 100644 --- a/pkg/providers/gcp/dns.go +++ b/pkg/providers/gcp/dns.go @@ -15,6 +15,10 @@ type cloudDNSProvider struct { projects []string } +func (d *cloudDNSProvider) name() string { + return "dns" +} + // GetResource returns all the resources in the store for a provider. func (d *cloudDNSProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -60,6 +64,7 @@ func (d *cloudDNSProvider) parseRecordsForResourceSet(r *dns.ResourceRecordSetsL PublicIPv4: data, ID: d.id, Provider: providerName, + Service: d.name(), }) } } diff --git a/pkg/providers/gcp/function.go b/pkg/providers/gcp/function.go index 9d39a35c..f80ae6f0 100644 --- a/pkg/providers/gcp/function.go +++ b/pkg/providers/gcp/function.go @@ -15,6 +15,10 @@ type cloudFunctionsProvider struct { projects []string } +func (d *cloudFunctionsProvider) name() string { + return "cloud-functions" +} + // GetResource returns all the Cloud Function resources in the store for a provider. func (d *cloudFunctionsProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -29,6 +33,7 @@ func (d *cloudFunctionsProvider) GetResource(ctx context.Context) (*schema.Resou Provider: providerName, DNSName: funcUrl.Hostname(), Public: d.isPublicFunction(function.Name), + Service: d.name(), } list.Append(resource) } diff --git a/pkg/providers/gcp/gke.go b/pkg/providers/gcp/gke.go index 0842835c..03da0b2d 100644 --- a/pkg/providers/gcp/gke.go +++ b/pkg/providers/gcp/gke.go @@ -23,6 +23,10 @@ type gkeProvider struct { projects []string } +func (d *gkeProvider) name() string { + return "gke" +} + // GetResource returns all the resources in the store for a provider. func (d *gkeProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -52,6 +56,9 @@ func (d *gkeProvider) GetResource(ctx context.Context) (*schema.Resources, error } k8sIngressProvider := k8s.NewK8sIngressProvider(d.id, ingress) ingressHosts, _ := k8sIngressProvider.GetResource(ctx) + for _, ingressHost := range ingressHosts.Items { + ingressHost.Service = d.name() + } list.Merge(ingressHosts) } } diff --git a/pkg/providers/gcp/vms.go b/pkg/providers/gcp/vms.go index d9d65fb5..39337335 100644 --- a/pkg/providers/gcp/vms.go +++ b/pkg/providers/gcp/vms.go @@ -14,6 +14,10 @@ type cloudVMProvider struct { projects []string } +func (d *cloudVMProvider) name() string { + return "vms" +} + // GetResource returns all the resources in the store for a provider. func (d *cloudVMProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -39,6 +43,7 @@ func (d *cloudVMProvider) GetResource(ctx context.Context) (*schema.Resources, e Public: true, Provider: providerName, PublicIPv4: cfg.NatIP, + Service: d.name(), }) } } diff --git a/pkg/providers/heroku/applications.go b/pkg/providers/heroku/applications.go index 5b222e56..c6e1ee18 100644 --- a/pkg/providers/heroku/applications.go +++ b/pkg/providers/heroku/applications.go @@ -13,6 +13,10 @@ type instanceProvider struct { client *heroku.Service } +func (d *instanceProvider) name() string { + return "application" +} + // GetResource returns all the applications for the Heroku provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -35,6 +39,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Public: isPublic, ID: d.id, Provider: providerName, + Service: d.name(), }) } diff --git a/pkg/providers/hetzner/instances.go b/pkg/providers/hetzner/instances.go index 2bb8708b..2853cf93 100644 --- a/pkg/providers/hetzner/instances.go +++ b/pkg/providers/hetzner/instances.go @@ -12,6 +12,10 @@ type instanceProvider struct { client *hetzner.Client } +func (d *instanceProvider) name() string { + return "instance" +} + // GetResource returns all the instances in the store for a provider. func (p *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -28,6 +32,7 @@ func (p *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, ID: p.id, PublicIPv4: server.PublicNet.IPv4.IP.String(), Public: true, + Service: p.name(), }) } for _, privateNet := range server.PrivateNet { @@ -35,6 +40,7 @@ func (p *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, ID: p.id, PrivateIpv4: privateNet.IP.String(), + Service: p.name(), }) } } diff --git a/pkg/providers/k8s/ingress.go b/pkg/providers/k8s/ingress.go index d5c79828..a631f117 100644 --- a/pkg/providers/k8s/ingress.go +++ b/pkg/providers/k8s/ingress.go @@ -20,6 +20,10 @@ func NewK8sIngressProvider(id string, ingress *v1.IngressList) *K8sIngressProvid } } +func (k *K8sIngressProvider) name() string { + return "ingress" +} + // GetResource returns all the resources in the store for a provider. func (k *K8sIngressProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -39,6 +43,7 @@ func (k *K8sIngressProvider) GetResource(ctx context.Context) (*schema.Resources Provider: providerName, ID: k.id, PublicIPv4: ip.IP, + Service: k.name(), }) } if ip.Hostname == "" { @@ -47,6 +52,7 @@ func (k *K8sIngressProvider) GetResource(ctx context.Context) (*schema.Resources Provider: providerName, ID: k.id, DNSName: ip.Hostname, + Service: k.name(), }) } } diff --git a/pkg/providers/k8s/services.go b/pkg/providers/k8s/services.go index 580dc412..63db8857 100644 --- a/pkg/providers/k8s/services.go +++ b/pkg/providers/k8s/services.go @@ -13,6 +13,10 @@ type K8sServiceProvider struct { serviceClient *v1.ServiceList } +func (k *K8sServiceProvider) name() string { + return "service" +} + // GetResource returns all the resources in the store for a provider. func (k *K8sServiceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -23,6 +27,7 @@ func (k *K8sServiceProvider) GetResource(ctx context.Context) (*schema.Resources Provider: providerName, ID: k.id, PublicIPv4: service.Spec.LoadBalancerIP, + Service: k.name(), }) } if service.Spec.Type == "LoadBalancer" { @@ -33,6 +38,7 @@ func (k *K8sServiceProvider) GetResource(ctx context.Context) (*schema.Resources ID: k.id, PublicIPv4: ip.IP, DNSName: ip.Hostname, + Service: "load_balancer", }) } } @@ -44,6 +50,7 @@ func (k *K8sServiceProvider) GetResource(ctx context.Context) (*schema.Resources PublicIPv4: ip, PrivateIpv4: "", DNSName: "", + Service: "external_ip", }) } for _, ip := range service.Spec.ClusterIPs { @@ -55,6 +62,7 @@ func (k *K8sServiceProvider) GetResource(ctx context.Context) (*schema.Resources Provider: providerName, ID: k.id, PrivateIpv4: ip, + Service: "cluster_ip", }) } } diff --git a/pkg/providers/linode/instances.go b/pkg/providers/linode/instances.go index b299fa36..ca31dd4a 100644 --- a/pkg/providers/linode/instances.go +++ b/pkg/providers/linode/instances.go @@ -13,6 +13,10 @@ type instanceProvider struct { client *linodego.Client } +func (d *instanceProvider) name() string { + return "instance" +} + // GetResource returns all the instance resources for a provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { @@ -34,6 +38,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, PublicIPv4: ip4, ID: d.id, Public: ip4 != "", + Service: d.name(), }) } diff --git a/pkg/providers/namecheap/domains.go b/pkg/providers/namecheap/domains.go index 4c938c48..75464f88 100644 --- a/pkg/providers/namecheap/domains.go +++ b/pkg/providers/namecheap/domains.go @@ -13,6 +13,10 @@ type domainProvider struct { client *namecheap.Client } +func (d *domainProvider) name() string { + return "domain" +} + // GetResource returns all the applications for the NameCheap provider. func (d *domainProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -35,6 +39,7 @@ func (d *domainProvider) GetResource(ctx context.Context) (*schema.Resources, er Provider: providerName, DNSName: *domain.Name, ID: d.id, + Service: d.name(), }) } if *domainList.Paging.TotalItems <= (*domainList.Paging.PageSize * *domainList.Paging.CurrentPage) { diff --git a/pkg/providers/nomad/resources.go b/pkg/providers/nomad/resources.go index d7afc876..e7335e83 100644 --- a/pkg/providers/nomad/resources.go +++ b/pkg/providers/nomad/resources.go @@ -41,6 +41,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, ID: d.id, PublicIPv4: node.Address, + Service: "nomad_node", }) nodeAddressMap[node.ID] = node.Address } @@ -71,6 +72,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources, for _, port := range network.ReservedPorts { list.Append(&schema.Resource{ Provider: providerName, + Service: job.Name, ID: d.id, PublicIPv4: net.JoinHostPort(nodeAddress, strconv.Itoa(port.Value)), }) @@ -78,6 +80,7 @@ func (d *resourceProvider) GetResource(ctx context.Context) (*schema.Resources, for _, port := range network.DynamicPorts { list.Append(&schema.Resource{ Provider: providerName, + Service: job.Name, ID: d.id, PublicIPv4: net.JoinHostPort(nodeAddress, strconv.Itoa(port.Value)), }) diff --git a/pkg/providers/openstack/instances.go b/pkg/providers/openstack/instances.go index 60a7fe02..13ed3bd2 100644 --- a/pkg/providers/openstack/instances.go +++ b/pkg/providers/openstack/instances.go @@ -16,6 +16,10 @@ type instanceProvider struct { client *gophercloud.ServiceClient } +func (d *instanceProvider) name() string { + return "instance" +} + // GetResource returns all the instances in the store for a provider. func (p *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -35,6 +39,7 @@ func (p *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, ID: p.id, PrivateIpv4: address["addr"].(string), + Service: p.name(), }) } } diff --git a/pkg/providers/scaleway/instances.go b/pkg/providers/scaleway/instances.go index c3cb8b9d..d7940049 100644 --- a/pkg/providers/scaleway/instances.go +++ b/pkg/providers/scaleway/instances.go @@ -14,6 +14,10 @@ type instanceProvider struct { instanceAPI *instance.API } +func (d *instanceProvider) name() string { + return "instance" +} + // GetInstances returns all the instances in the store for a provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { list := schema.NewResources() @@ -44,6 +48,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, Provider: providerName, ID: d.id, PrivateIpv4: privateIP4, + Service: d.name(), }) } list.Append(&schema.Resource{ @@ -51,6 +56,7 @@ func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, ID: d.id, PublicIPv4: ip4, Public: true, + Service: d.name(), }) } if resp.TotalCount == totalResults { diff --git a/pkg/providers/terraform/state_file.go b/pkg/providers/terraform/state_file.go index 2eea68a5..6e025073 100644 --- a/pkg/providers/terraform/state_file.go +++ b/pkg/providers/terraform/state_file.go @@ -16,6 +16,10 @@ type instanceProvider struct { id string } +func (d *instanceProvider) name() string { + return "instance" +} + // GetInstances returns all the instances in the store for a provider. func (d *instanceProvider) GetResource(ctx context.Context) (*schema.Resources, error) { file, err := os.Open(d.path) @@ -44,6 +48,7 @@ func (d *instanceProvider) extractIPsFromText(text string) *schema.Resources { Provider: providerName, ID: d.id, PublicIPv4: match[0], + Service: d.name(), }) } return resources diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index f07aab25..21bb008b 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -51,10 +51,11 @@ func init() { } // appendResourceWithTypeAndMeta appends a resource with a type and metadata -func (r *Resources) appendResourceWithTypeAndMeta(resourceType validate.ResourceType, item, id, provider string) { +func (r *Resources) appendResourceWithTypeAndMeta(resourceType validate.ResourceType, item, id, provider, service string) { resource := &Resource{ Provider: provider, ID: id, + Service: service, } switch resourceType { case validate.DNSName: @@ -75,17 +76,17 @@ func (r *Resources) appendResourceWithTypeAndMeta(resourceType validate.Resource func (r *Resources) appendResource(resource *Resource, uniqueMap *sync.Map) { if _, ok := uniqueMap.Load(resource.DNSName); !ok && resource.DNSName != "" { resourceType := validator.Identify(resource.DNSName) - r.appendResourceWithTypeAndMeta(resourceType, resource.DNSName, resource.ID, resource.Provider) + r.appendResourceWithTypeAndMeta(resourceType, resource.DNSName, resource.ID, resource.Provider, resource.Service) uniqueMap.Store(resource.DNSName, struct{}{}) } if _, ok := uniqueMap.Load(resource.PublicIPv4); !ok && resource.PublicIPv4 != "" { resourceType := validator.Identify(resource.PublicIPv4) - r.appendResourceWithTypeAndMeta(resourceType, resource.PublicIPv4, resource.ID, resource.Provider) + r.appendResourceWithTypeAndMeta(resourceType, resource.PublicIPv4, resource.ID, resource.Provider, resource.Service) uniqueMap.Store(resource.PublicIPv4, struct{}{}) } if _, ok := uniqueMap.Load(resource.PrivateIpv4); !ok && resource.PrivateIpv4 != "" { resourceType := validator.Identify(resource.PrivateIpv4) - r.appendResourceWithTypeAndMeta(resourceType, resource.PrivateIpv4, resource.ID, resource.Provider) + r.appendResourceWithTypeAndMeta(resourceType, resource.PrivateIpv4, resource.ID, resource.Provider, resource.Service) uniqueMap.Store(resource.PrivateIpv4, struct{}{}) } } @@ -112,6 +113,8 @@ type Resource struct { Public bool `json:"public"` // Provider is the name of provider for instance Provider string `json:"provider"` + // Service is the name of the service under the provider + Service string `json:"service,omitempty"` // ID is the id name of the resource provider ID string `json:"id,omitempty"` // PublicIPv4 is the public ipv4 address of the instance.