From 89c499013047d774d393800570bbcf8b08688e7a Mon Sep 17 00:00:00 2001 From: Brendan O'Leary Date: Sat, 30 Mar 2024 23:51:57 +0800 Subject: [PATCH 1/4] Remove items moved to the centralized docs (#494) * Remove items moved to the centralized docs * Update links --------- Co-authored-by: Brendan O'Leary --- README.md | 296 ++---------------------------------------------------- 1 file changed, 6 insertions(+), 290 deletions(-) diff --git a/README.md b/README.md index 3bfdea6b..f095e38b 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,12 @@

Features • - Installation • - Usage • - Configuration • - Running cloudlist • - Supported providers • - Library • + Installation • + Usage • + Configuration • + Running cloudlist • + Supported providers • + Library • Join Discord

@@ -75,255 +75,6 @@ OUTPUT: -silent display only results in output ``` -# Installation Instructions - - -Download the ready to use binary from [release page](https://github.com/projectdiscovery/cloudlist/releases/) or install/build using Go - -```sh -go install -v github.com/projectdiscovery/cloudlist/cmd/cloudlist@latest -``` - -# Running Cloudlist - -``` -cloudlist -``` - -This will list all the assets from configured providers in the configuration file. Specific providers and asset type can also be specified using `provider` and `id` filter. - -```console -cloudlist -provider aws,gcp - - ________ _____ __ - / ____/ /___ __ ______/ / (_)____/ /_ - / / / / __ \/ / / / __ / / / ___/ __/ -/ /___/ / /_/ / /_/ / /_/ / / (__ ) /_ -\____/_/\____/\__,_/\__,_/_/_/____/\__/ v0.0.1 - - projectdiscovery.io - -[WRN] Use with caution. You are responsible for your actions -[WRN] Developers assume no liability and are not responsible for any misuse or damage. -[INF] Listing assets from AWS (prod) provider. -example.com -example2.com -example3.com -1.1.1.1 -2.2.2.2 -3.3.3.3 -4.4.4.4 -5.5.5.5 -6.6.6.6 -[INF] Found 2 hosts and 6 IPs from AWS service (prod) -``` -## Running cloudlist with Nuclei - -Scanning assets from various cloud providers with nuclei for security assessments:- - -```bash -cloudlist -silent | httpx -silent | nuclei -t cves/ -``` - -# Supported providers - -- AWS (Amazon web services) - - EC2 - - Route53 - - S3 -- GCP (Google Cloud Platform) - - Cloud DNS - - GKE -- DO (DigitalOcean) - - Instances -- SCW (Scaleway) - - Instances -- Fastly - - Services -- Heroku - - Applications -- Linode - - Instances -- Azure - - Virtual Machines -- Namecheap - - Domain List -- Alibaba Cloud - - ECS Instances -- Cloudflare - - DNS -- Hashistack - - Nomad - - Consul - - Terraform -- Hetzner Cloud - - Instances -- Openstack - - Instances -- Kubernetes - - Services - - Ingresses - -# Configuration file - -The default provider config file should be located at `$HOME/.config/cloudlist/provider-config.yaml` and has the following contents as an example. In order to run this tool, the keys need to updated in the config file for the desired providers. - -
-Example Provider Config - -```yaml -- provider: do # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: xxxx - # digitalocean_token is the API key for digitalocean cloud platform - digitalocean_token: $DIGITALOCEAN_TOKEN - -- provider: scw # provider is the name of the provider - # scaleway_access_key is the access key for scaleway API - scaleway_access_key: $SCALEWAY_ACCESS_KEY - # scaleway_access_token is the access token for scaleway API - scaleway_access_token: $SCALEWAY_ACCESS_TOKEN - -- provider: aws # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # aws_access_key is the access key for AWS account - aws_access_key: $AWS_ACCESS_KEY - # aws_secret_key is the secret key for AWS account - aws_secret_key: $AWS_SECRET_KEY - # aws_session_token session token for temporary security credentials retrieved via STS (optional) - aws_session_token: $AWS_SESSION_TOKEN - -- provider: gcp # provider is the name of the provider - # profile is the name of the provider profile - id: logs - # gcp_service_account_key is the minified json of a google cloud service account with list permissions - gcp_service_account_key: '{xxxxxxxxxxxxx}' - -- provider: azure # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # client_id is the client ID of registered application of the azure account (not requuired if using cli auth) - client_id: $AZURE_CLIENT_ID - # client_secret is the secret ID of registered application of the zure account (not requuired if using cli uth) - client_secret: $AZURE_CLIENT_SECRET - # tenant_id is the tenant ID of registered application of the azure account (not requuired if using cli auth) - tenant_id: $AZURE_TENANT_ID - #subscription_id is the azure subscription id - subscription_id: $AZURE_SUBSCRIPTION_ID - #use_cli_auth if set to true cloudlist will use azure cli auth - use_cli_auth: true - -- provider: cloudflare # provider is the name of the provider - # email is the email for cloudflare - email: $CF_EMAIL - # api_key is the api_key for cloudflare - api_key: $CF_API_KEY - # api_token is the scoped_api_token for cloudflare (optional) - api_token: $CF_API_TOKEN - -- provider: heroku # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # heroku_api_token is the api key for Heroku account - heroku_api_token: $HEROKU_API_TOKEN - -- provider: linode # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # linode_personal_access_token is the personal access token for linode account - linode_personal_access_token: $LINODE_PERSONAL_ACCESS_TOKEN - -- provider: fastly # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # fastly_api_key is the personal API token for fastly account - fastly_api_key: $FASTLY_API_KEY - -- provider: alibaba # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # alibaba_region_id is the region id of the resources - alibaba_region_id: $ALIBABA_REGION_ID - # alibaba_access_key is the access key ID for alibaba cloud account - alibaba_access_key: $ALIBABA_ACCESS_KEY - # alibaba_access_key_secret is the secret access key for alibaba cloud account - alibaba_access_key_secret: $ALIBABA_ACCESS_KEY_SECRET - -- provider: namecheap # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # namecheap_api_key is the api key for namecheap account - namecheap_api_key: $NAMECHEAP_API_KEY - # namecheap_user_name is the username of the namecheap account - namecheap_user_name: $NAMECHEAP_USER_NAME - -- provider: terraform # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - #tf_state_file is the location of terraform state file (terraform.tfsate) - tf_state_file: path/to/terraform.tfstate - -- provider: hetzner # provider is the name of the provider - # id is the name defined by user for filtering (optional) - id: staging - # auth_token is the is the hetzner authentication token - auth_token: $HETZNER_AUTH_TOKEN - -- provider: nomad # provider is the name of the provider - # nomad_url is the url for nomad server - nomad_url: http:/127.0.0.1:4646/ - # nomad_ca_file is the path to nomad CA file - # nomad_ca_file: .pem - # nomad_cert_file is the path to nomad Certificate file - # nomad_cert_file: .pem - # nomad_key_file is the path to nomad Certificate Key file - # nomad_key_file: .pem - # nomad_token is the nomad authentication token - # nomad_token: - # nomad_http_auth is the nomad http auth value - # nomad_http_auth: - -- provider: consul # provider is the name of the provider - # consul_url is the url for consul server - consul_url: http://localhost:8500/ - # consul_ca_file is the path to consul CA file - # consul_ca_file: .pem - # consul_cert_file is the path to consul Certificate file - # consul_cert_file: .pem - # consul_key_file is the path to consul Certificate Key file - # consul_key_file: .pem - # consul_http_token is the consul authentication token - # consul_http_token: - # consul_http_auth is the consul http auth value - # consul_http_auth: - -- provider: openstack # provider is the name of the provider - # id is the name of the provider id - id: staging - # identity_endpoint is Openstack identity endpoint used to authenticate - identity_endpoint: $OS_IDENTITY_ENDPOINT - # domain_name is Openstack domain name used to authenticate - domain_name: $OS_DOMAIN_NAME - # tenant_name is Openstack project name - tenant_name: $OS_TENANT_NAME - # username is Openstack username used to authenticate - username: $OS_USERNAME - # password is Openstack password used to authenticate - password: $OS_PASSWORD - -- provider: kubernetes # provider is the name of the provider - # id is the name of the provider id - id: staging - # kubeconfig_file is the path of kubeconfig file - kubeconfig: path/to/kubeconfig - # context is the context to be used from kubeconfig file - context: - -``` - -
- # Contribution Please check [PROVIDERS.md](https://github.com/projectdiscovery/cloudlist/blob/main/PROVIDERS.md) and [DESIGN.md](https://github.com/projectdiscovery/cloudlist/blob/main/DESIGN.md) to include support for new cloud providers in Cloudlist. @@ -335,41 +86,6 @@ Please check [PROVIDERS.md](https://github.com/projectdiscovery/cloudlist/blob/m - Push to the branch (`git push origin new-provider`) - Create new Pull Request -# Cloudlist as a library - -It's possible to use the library directly in your go programs. The following code snippets outline how to list assets from all or given cloud provider. - -```go -package main - -import ( - "context" - "log" - - "github.com/projectdiscovery/cloudlist/pkg/inventory" - "github.com/projectdiscovery/cloudlist/pkg/schema" -) - -func main() { - inventory, err := inventory.New(schema.Options{ - schema.OptionBlock{"provider": "digitalocean", "digitalocean_token": "ec405badb974fd3d891c9223245f9ab5871c127fce9e632c8dc421edd46d7242"}, - }) - if err != nil { - log.Fatalf("%s\n", err) - } - - for _, provider := range inventory.Providers { - resources, err := provider.Resources(context.Background()) - if err != nil { - log.Fatalf("%s\n", err) - } - for _, resource := range resources.Items { - _ = resource // Do something with the resource - } - } -} -``` - ## Acknowledgments Thank you for inspiration From e2823a94eaf212f7d6be99f4bc5fea137314bbab Mon Sep 17 00:00:00 2001 From: Jeremy Galindo Date: Sat, 4 May 2024 08:39:43 -0400 Subject: [PATCH 2/4] Fixed nil fields being dereferenced for public ips --- pkg/providers/azure/publicips.go | 5 +++++ pkg/providers/azure/vm.go | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/pkg/providers/azure/publicips.go b/pkg/providers/azure/publicips.go index 4730fe80..316bca29 100644 --- a/pkg/providers/azure/publicips.go +++ b/pkg/providers/azure/publicips.go @@ -26,6 +26,11 @@ func (pip *publicIPProvider) GetResource(ctx context.Context) (*schema.Resources } for _, ip := range ips { + // The IPAddress field can be nil and so we want to prevent from dereferencing + // a nil field in the struct + if ip.IPAddress == nil { + continue + } list.Append(&schema.Resource{ Provider: providerName, PublicIPv4: *ip.IPAddress, diff --git a/pkg/providers/azure/vm.go b/pkg/providers/azure/vm.go index e19171ea..fb3b7262 100644 --- a/pkg/providers/azure/vm.go +++ b/pkg/providers/azure/vm.go @@ -51,6 +51,12 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) for _, ipConfig := range ipconfigList { privateIP := *ipConfig.PrivateIPAddress + // The PublicIPAddress field can be nil especially for SQL Server + // VMs that are pulled. + // In these cases, we just want to return an empty error + if ipConfig.PublicIPAddress == nil { + return nil, nil + } res, err := azure.ParseResourceID(*ipConfig.PublicIPAddress.ID) if err != nil { return nil, err From f026668e926892f578aa85cc0aa0b6b4dd6389b9 Mon Sep 17 00:00:00 2001 From: Jeremy Galindo Date: Tue, 7 May 2024 15:40:14 -0400 Subject: [PATCH 3/4] Changed to continue, added new nil check to prevent SEGV --- pkg/providers/azure/vm.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/providers/azure/vm.go b/pkg/providers/azure/vm.go index fb3b7262..3be2b5bc 100644 --- a/pkg/providers/azure/vm.go +++ b/pkg/providers/azure/vm.go @@ -55,8 +55,9 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) // VMs that are pulled. // In these cases, we just want to return an empty error if ipConfig.PublicIPAddress == nil { - return nil, nil + continue } + res, err := azure.ParseResourceID(*ipConfig.PublicIPAddress.ID) if err != nil { return nil, err @@ -67,6 +68,10 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) return nil, err } + if publicIP.IPAddress == nil { + return nil, nil + } + list.Append(&schema.Resource{ Provider: providerName, PublicIPv4: *publicIP.IPAddress, From 2210951fe287155cce0208e9b055aad87f80a10f Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 4 Jun 2024 16:39:50 +0530 Subject: [PATCH 4/4] misc --- pkg/providers/azure/vm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/providers/azure/vm.go b/pkg/providers/azure/vm.go index 3be2b5bc..4871977b 100644 --- a/pkg/providers/azure/vm.go +++ b/pkg/providers/azure/vm.go @@ -49,7 +49,7 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) return nil, err } for _, ipConfig := range ipconfigList { - privateIP := *ipConfig.PrivateIPAddress + privateIP := ipConfig.PrivateIPAddress // The PublicIPAddress field can be nil especially for SQL Server // VMs that are pulled. @@ -69,14 +69,14 @@ func (d *vmProvider) GetResource(ctx context.Context) (*schema.Resources, error) } if publicIP.IPAddress == nil { - return nil, nil + continue } list.Append(&schema.Resource{ Provider: providerName, PublicIPv4: *publicIP.IPAddress, ID: d.id, - PrivateIpv4: privateIP, + PrivateIpv4: *privateIP, }) } }