adding checkDnsResponse and fallback methods (#86)

* adding DnsResponse and fallback methods

adding CNAME check

fix typos
adding tests

* fix typo

* fix typos

* refactor with new logic

* add retry flag

* fix failing functional_test

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

.gitignore

@@ -1,3 +1,5 @@
 cmd/generate-index/generate-index
 cmd/cdncheck/cdncheck
+cmd/functional-test/cdncheck_dev
+cmd/functional-test/functional-test
 dist/

README.md

@@ -68,14 +68,14 @@ DETECTION:
    -waf    display only waf in cli output
 
 MATCHER:
-   -mcdn, -match-cdn string[]      match host with specified cdn provider (fastly, cloudfront)
-   -mcloud, -match-cloud string[]  match host with specified cloud provider (google, oracle, aws, azure)
-   -mwaf, -match-waf string[]      match host with specified waf provider (cloudflare, incapsula)
+   -mcdn, -match-cdn string[]      match host with specified cdn provider (cloudfront, fastly, google, leaseweb)
+   -mcloud, -match-cloud string[]  match host with specified cloud provider (aws, google, oracle)
+   -mwaf, -match-waf string[]      match host with specified waf provider (cloudflare, incapsula, sucuri, akamai)
 
 FILTER:
-   -fcdn, -filter-cdn string[]      filter host with specified cdn provider (fastly, cloudfront)
-   -fcloud, -filter-cloud string[]  filter host with specified cloud provider (google, oracle, aws, azure)
-   -fwaf, -filter-waf string[]      filter host with specified waf provider (cloudflare, incapsula)
+   -fcdn, -filter-cdn string[]      filter host with specified cdn provider (cloudfront, fastly, google, leaseweb)
+   -fcloud, -filter-cloud string[]  filter host with specified cloud provider (aws, google, oracle)
+   -fwaf, -filter-waf string[]      filter host with specified waf provider (cloudflare, incapsula, sucuri, akamai)
 
 OUTPUT:
    -resp               display technology name in cli output
@@ -89,6 +89,7 @@ OUTPUT:
 CONFIG:
    -r, -resolver string[]  list of resolvers to use (file or comma separated)
    -e, -exclude            exclude detected ip from output
+   -retry int              maximum number of retries for dns resolution (must be at least 1) (default 2)
 
 UPDATE:
    -up, -update                 update cdncheck to latest version

cdncheck.go

@@ -4,6 +4,8 @@ import (
 	"net"
 	"strings"
 	"sync"
+
+	"github.com/projectdiscovery/retryabledns"
 )
 
 var (
@@ -12,23 +14,50 @@ var (
 	DefaultCloudProviders string
 )
 
+// DefaultResolvers trusted (taken from fastdialer)
+var DefaultResolvers = []string{
+	"1.1.1.1:53",
+	"1.0.0.1:53",
+	"8.8.8.8:53",
+	"8.8.4.4:53",
+}
+
 // Client checks for CDN based IPs which should be excluded
 // during scans since they belong to third party firewalls.
 type Client struct {
 	sync.Once
-	cdn   *providerScraper
-	waf   *providerScraper
-	cloud *providerScraper
+	cdn          *providerScraper
+	waf          *providerScraper
+	cloud        *providerScraper
+	retriabledns *retryabledns.Client
 }
 
-// New creates a new firewall IP checking client.
+// New creates cdncheck client with default options
+// NewWithOpts should be preferred over this function
 func New() *Client {
+	client, _ := NewWithOpts(3, []string{})
+	return client
+}
+
+// NewWithOpts creates cdncheck client with custom options
+func NewWithOpts(MaxRetries int, resolvers []string) (*Client, error) {
+	if MaxRetries <= 0 {
+		MaxRetries = 3
+	}
+	if len(resolvers) == 0 {
+		resolvers = DefaultResolvers
+	}
+	retryabledns, err := retryabledns.New(resolvers, MaxRetries)
+	if err != nil {
+		return nil, err
+	}
 	client := &Client{
-		cdn:   newProviderScraper(generatedData.CDN),
-		waf:   newProviderScraper(generatedData.WAF),
-		cloud: newProviderScraper(generatedData.Cloud),
+		cdn:          newProviderScraper(generatedData.CDN),
+		waf:          newProviderScraper(generatedData.WAF),
+		cloud:        newProviderScraper(generatedData.Cloud),
+		retriabledns: retryabledns,
 	}
-	return client
+	return client, nil
 }
 
 // CheckCDN checks if an IP is contained in the cdn denylist
@@ -49,9 +78,7 @@ func (c *Client) CheckCloud(ip net.IP) (matched bool, value string, err error) {
 	return matched, value, err
 }
 
-// Check checks if an IP is contained in the denylist
-//
-// It includes CDN, WAF and Cloud. Basically all varaint of individual functions
+// Check checks if ip belongs to one of CDN, WAF and Cloud . It is generic method for Checkxxx methods
 func (c *Client) Check(ip net.IP) (matched bool, value string, itemType string, err error) {
 	if matched, value, err = c.cdn.Match(ip); err == nil && matched && value != "" {
 		return matched, value, "cdn", nil
@@ -65,6 +92,58 @@ func (c *Client) Check(ip net.IP) (matched bool, value string, itemType string,
 	return false, "", "", err
 }
 
+// Check Domain with fallback checks if domain belongs to one of CDN, WAF and Cloud . It is generic method for Checkxxx methods
+// Since input is domain, as a fallback it queries CNAME records and checks if domain is WAF
+func (c *Client) CheckDomainWithFallback(domain string) (matched bool, value string, itemType string, err error) {
+	dnsData, err := c.retriabledns.Resolve(domain)
+	if err != nil {
+		return false, "", "", err
+	}
+	matched, value, itemType, err = c.CheckDNSResponse(dnsData)
+	if err != nil {
+		return false, "", "", err
+	}
+	if matched {
+		return matched, value, itemType, nil
+	}
+	// resolve cname
+	dnsData, err = c.retriabledns.CNAME(domain)
+	if err != nil {
+		return false, "", "", err
+	}
+	return c.CheckDNSResponse(dnsData)
+}
+
+// CheckDNSResponse is same as CheckDomainWithFallback but takes DNS response as input
+func (c *Client) CheckDNSResponse(dnsResponse *retryabledns.DNSData) (matched bool, value string, itemType string, err error) {
+	if dnsResponse.A != nil {
+		for _, ip := range dnsResponse.A {
+			ipAddr := net.ParseIP(ip)
+			if ipAddr == nil {
+				continue
+			}
+			matched, value, itemType, err := c.Check(ipAddr)
+			if err != nil {
+				return false, "", "", err
+			}
+			if matched {
+				return matched, value, itemType, nil
+			}
+		}
+	}
+	if dnsResponse.CNAME != nil {
+		matched, discovered, itemType, err := c.CheckSuffix(dnsResponse.CNAME...)
+		if err != nil {
+			return false, "", itemType, err
+		}
+		if matched {
+			// for now checkSuffix only checks for wafs
+			return matched, discovered, itemType, nil
+		}
+	}
+	return false, "", "", err
+}
+
 func mapKeys(m map[string][]string) string {
 	keys := make([]string, 0, len(m))
 	for k := range m {

go.mod

@@ -7,10 +7,10 @@ require (
 	github.com/ipinfo/go/v2 v2.9.2
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/projectdiscovery/fastdialer v0.0.25
 	github.com/projectdiscovery/goflags v0.1.8
 	github.com/projectdiscovery/gologger v1.1.8
 	github.com/projectdiscovery/mapcidr v1.1.1
+	github.com/projectdiscovery/retryabledns v1.0.23
 	github.com/projectdiscovery/utils v0.0.26
 	github.com/stretchr/testify v1.8.2
 	github.com/weppos/publicsuffix-go v0.30.0
@@ -23,7 +23,6 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
-	github.com/akrylysov/pogreb v0.10.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
@@ -33,7 +32,6 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.2 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/fatih/color v1.14.1 // indirect
@@ -61,29 +59,14 @@ require (
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
-	github.com/projectdiscovery/hmap v0.0.11 // indirect
-	github.com/projectdiscovery/networkpolicy v0.0.5 // indirect
-	github.com/projectdiscovery/retryabledns v1.0.21 // indirect
-	github.com/projectdiscovery/retryablehttp-go v1.0.14 // indirect
+	github.com/projectdiscovery/retryablehttp-go v1.0.15 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
-	github.com/syndtr/goleveldb v1.0.0 // indirect
-	github.com/tidwall/btree v1.4.3 // indirect
-	github.com/tidwall/buntdb v1.2.10 // indirect
-	github.com/tidwall/gjson v1.14.3 // indirect
-	github.com/tidwall/grect v0.1.4 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/rtred v0.1.2 // indirect
-	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yuin/goldmark v1.5.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
-	github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
-	github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/exp v0.0.0-20230420155640-133eef4313cb // indirect