diff --git a/.gitignore b/.gitignore index 1ebffdb..28bd029 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ cmd/generate-index/generate-index cmd/cdncheck/cdncheck +cmd/functional-test/cdncheck_dev +cmd/functional-test/functional-test dist/ \ No newline at end of file diff --git a/README.md b/README.md index 22b532d..e0875af 100644 --- a/README.md +++ b/README.md @@ -68,14 +68,14 @@ DETECTION: -waf display only waf in cli output MATCHER: - -mcdn, -match-cdn string[] match host with specified cdn provider (fastly, cloudfront) - -mcloud, -match-cloud string[] match host with specified cloud provider (google, oracle, aws, azure) - -mwaf, -match-waf string[] match host with specified waf provider (cloudflare, incapsula) + -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, leaseweb) + -mcloud, -match-cloud string[] match host with specified cloud provider (aws, google, oracle) + -mwaf, -match-waf string[] match host with specified waf provider (cloudflare, incapsula, sucuri, akamai) FILTER: - -fcdn, -filter-cdn string[] filter host with specified cdn provider (fastly, cloudfront) - -fcloud, -filter-cloud string[] filter host with specified cloud provider (google, oracle, aws, azure) - -fwaf, -filter-waf string[] filter host with specified waf provider (cloudflare, incapsula) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, leaseweb) + -fcloud, -filter-cloud string[] filter host with specified cloud provider (aws, google, oracle) + -fwaf, -filter-waf string[] filter host with specified waf provider (cloudflare, incapsula, sucuri, akamai) OUTPUT: -resp display technology name in cli output @@ -89,6 +89,7 @@ OUTPUT: CONFIG: -r, -resolver string[] list of resolvers to use (file or comma separated) -e, -exclude exclude detected ip from output + -retry int maximum number of retries for dns resolution (must be at least 1) (default 2) UPDATE: -up, -update update cdncheck to latest version diff --git a/cdncheck.go b/cdncheck.go index 8d11ae5..294365c 100644 --- a/cdncheck.go +++ b/cdncheck.go @@ -4,6 +4,8 @@ import ( "net" "strings" "sync" + + "github.com/projectdiscovery/retryabledns" ) var ( @@ -12,23 +14,50 @@ var ( DefaultCloudProviders string ) +// DefaultResolvers trusted (taken from fastdialer) +var DefaultResolvers = []string{ + "1.1.1.1:53", + "1.0.0.1:53", + "8.8.8.8:53", + "8.8.4.4:53", +} + // Client checks for CDN based IPs which should be excluded // during scans since they belong to third party firewalls. type Client struct { sync.Once - cdn *providerScraper - waf *providerScraper - cloud *providerScraper + cdn *providerScraper + waf *providerScraper + cloud *providerScraper + retriabledns *retryabledns.Client } -// New creates a new firewall IP checking client. +// New creates cdncheck client with default options +// NewWithOpts should be preferred over this function func New() *Client { + client, _ := NewWithOpts(3, []string{}) + return client +} + +// NewWithOpts creates cdncheck client with custom options +func NewWithOpts(MaxRetries int, resolvers []string) (*Client, error) { + if MaxRetries <= 0 { + MaxRetries = 3 + } + if len(resolvers) == 0 { + resolvers = DefaultResolvers + } + retryabledns, err := retryabledns.New(resolvers, MaxRetries) + if err != nil { + return nil, err + } client := &Client{ - cdn: newProviderScraper(generatedData.CDN), - waf: newProviderScraper(generatedData.WAF), - cloud: newProviderScraper(generatedData.Cloud), + cdn: newProviderScraper(generatedData.CDN), + waf: newProviderScraper(generatedData.WAF), + cloud: newProviderScraper(generatedData.Cloud), + retriabledns: retryabledns, } - return client + return client, nil } // CheckCDN checks if an IP is contained in the cdn denylist @@ -49,9 +78,7 @@ func (c *Client) CheckCloud(ip net.IP) (matched bool, value string, err error) { return matched, value, err } -// Check checks if an IP is contained in the denylist -// -// It includes CDN, WAF and Cloud. Basically all varaint of individual functions +// Check checks if ip belongs to one of CDN, WAF and Cloud . It is generic method for Checkxxx methods func (c *Client) Check(ip net.IP) (matched bool, value string, itemType string, err error) { if matched, value, err = c.cdn.Match(ip); err == nil && matched && value != "" { return matched, value, "cdn", nil @@ -65,6 +92,58 @@ func (c *Client) Check(ip net.IP) (matched bool, value string, itemType string, return false, "", "", err } +// Check Domain with fallback checks if domain belongs to one of CDN, WAF and Cloud . It is generic method for Checkxxx methods +// Since input is domain, as a fallback it queries CNAME records and checks if domain is WAF +func (c *Client) CheckDomainWithFallback(domain string) (matched bool, value string, itemType string, err error) { + dnsData, err := c.retriabledns.Resolve(domain) + if err != nil { + return false, "", "", err + } + matched, value, itemType, err = c.CheckDNSResponse(dnsData) + if err != nil { + return false, "", "", err + } + if matched { + return matched, value, itemType, nil + } + // resolve cname + dnsData, err = c.retriabledns.CNAME(domain) + if err != nil { + return false, "", "", err + } + return c.CheckDNSResponse(dnsData) +} + +// CheckDNSResponse is same as CheckDomainWithFallback but takes DNS response as input +func (c *Client) CheckDNSResponse(dnsResponse *retryabledns.DNSData) (matched bool, value string, itemType string, err error) { + if dnsResponse.A != nil { + for _, ip := range dnsResponse.A { + ipAddr := net.ParseIP(ip) + if ipAddr == nil { + continue + } + matched, value, itemType, err := c.Check(ipAddr) + if err != nil { + return false, "", "", err + } + if matched { + return matched, value, itemType, nil + } + } + } + if dnsResponse.CNAME != nil { + matched, discovered, itemType, err := c.CheckSuffix(dnsResponse.CNAME...) + if err != nil { + return false, "", itemType, err + } + if matched { + // for now checkSuffix only checks for wafs + return matched, discovered, itemType, nil + } + } + return false, "", "", err +} + func mapKeys(m map[string][]string) string { keys := make([]string, 0, len(m)) for k := range m { diff --git a/go.mod b/go.mod index 1b241bc..f5aa10d 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ require ( github.com/ipinfo/go/v2 v2.9.2 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/fastdialer v0.0.25 github.com/projectdiscovery/goflags v0.1.8 github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/mapcidr v1.1.1 + github.com/projectdiscovery/retryabledns v1.0.23 github.com/projectdiscovery/utils v0.0.26 github.com/stretchr/testify v1.8.2 github.com/weppos/publicsuffix-go v0.30.0 @@ -23,7 +23,6 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect @@ -33,7 +32,6 @@ require ( github.com/cheggaaa/pb/v3 v3.1.2 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/fatih/color v1.14.1 // indirect @@ -61,29 +59,14 @@ require ( github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/hmap v0.0.11 // indirect - github.com/projectdiscovery/networkpolicy v0.0.5 // indirect - github.com/projectdiscovery/retryabledns v1.0.21 // indirect - github.com/projectdiscovery/retryablehttp-go v1.0.14 // indirect + github.com/projectdiscovery/retryablehttp-go v1.0.15 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect - github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/tidwall/btree v1.4.3 // indirect - github.com/tidwall/buntdb v1.2.10 // indirect - github.com/tidwall/gjson v1.14.3 // indirect - github.com/tidwall/grect v0.1.4 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect - github.com/tidwall/rtred v0.1.2 // indirect - github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect - github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect - github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect - github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect - go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/exp v0.0.0-20230420155640-133eef4313cb // indirect diff --git a/go.sum b/go.sum index cfda402..732304c 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,6 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= -github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= @@ -21,8 +19,6 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g= -github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= @@ -33,8 +29,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= @@ -45,15 +39,11 @@ github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -70,16 +60,12 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ipinfo/go/v2 v2.9.2 h1:wih7S6ifXAdGE7OH5fgTfC/yA/lFYRKaG5z4FNiE+MY= github.com/ipinfo/go/v2 v2.9.2/go.mod h1:tRDkYfM20b1XzNqorn1Q1O6Xtg7uzw3Wn3I2R0SyJh4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -113,7 +99,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= @@ -122,15 +107,8 @@ github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4AN github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -142,22 +120,16 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/fastdialer v0.0.25 h1:/j7M8sL35ZqpRYmz9hDjnWNBnN8Yj5iL0kab7qzp1nQ= -github.com/projectdiscovery/fastdialer v0.0.25/go.mod h1:jQLHYaNsDc3MhpySVjKIo4o0GaUb07ISlDdkKURAxSo= github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g= github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2nLSnHXVA= -github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI= github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e1WPXpluc0= github.com/projectdiscovery/mapcidr v1.1.1/go.mod h1:yyp9ghqmmC0+r5DySgDBXE4cf2QW8SBloVESCteWiAg= -github.com/projectdiscovery/networkpolicy v0.0.5 h1:yiW1+nqtZmVB4RRt20Rhxs0tAO7ghPy+OY5kv3iaAG8= -github.com/projectdiscovery/networkpolicy v0.0.5/go.mod h1:egI9+HL+Q80AzXs1PrOfARWZm1bAn6n3HMONM49uWS8= -github.com/projectdiscovery/retryabledns v1.0.21 h1:vOpPQR1q8Z824uoA8JXCI/RyvDAssPeD68Onz9hP/ds= -github.com/projectdiscovery/retryabledns v1.0.21/go.mod h1:6oTPKMRlKZ7lIIEzTH723K6RvNRjmm6fe9br4Dom3UI= -github.com/projectdiscovery/retryablehttp-go v1.0.14 h1:PJaHQtHWE00xrmryZwhtma2b72GbkfA9gJM0yIR9ENY= -github.com/projectdiscovery/retryablehttp-go v1.0.14/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= +github.com/projectdiscovery/retryabledns v1.0.23 h1:8X6HECevl6n7K7kxqfHLTcsCVXDdz9HR+1hS6fto7sw= +github.com/projectdiscovery/retryabledns v1.0.23/go.mod h1:tj8BEe0jWCB4m3aAhHq4S2dqfpod0h+BiCZmXkXAGac= +github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhEymvnZXS2Wp2Zs4rWk/c= +github.com/projectdiscovery/retryablehttp-go v1.0.15/go.mod h1:+OzSFUv3sQcPt+MgbNx6X/Q3ESxqPUQSphqG5kxoIgI= github.com/projectdiscovery/utils v0.0.26 h1:89t05av9r5gtHBZLCiEkid1rrUjHlwKppz5z5iYzeG0= github.com/projectdiscovery/utils v0.0.26/go.mod h1:lCiVsbo79iqy4ihUQWUY4ywz0J13Mqudirgk+IV767o= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -165,14 +137,12 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -180,38 +150,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= -github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= -github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= -github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM= -github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= -github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= -github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= -github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= -github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= -github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= -github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= -github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2ZvtT+vQLY= github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY= -github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= @@ -224,19 +169,9 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= -github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= -github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= -github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY= -github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -247,13 +182,11 @@ golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnL golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -265,18 +198,13 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -315,18 +243,11 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/runner/options.go b/internal/runner/options.go index d2d3a4f..c55ec45 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -75,6 +75,7 @@ type Options struct { FilterWaf goflags.StringSlice Resolvers goflags.StringSlice OnResult func(r Output) + MaxRetries int } // configureOutput configures the output logging levels to be displayed on the screen @@ -145,6 +146,7 @@ func readFlags() (*Options, error) { flagSet.CreateGroup("config", "CONFIG", flagSet.StringSliceVarP(&opts.Resolvers, "resolver", "r", nil, "list of resolvers to use (file or comma separated)", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&opts.Exclude, "exclude", "e", false, "exclude detected ip from output"), + flagSet.IntVar(&opts.MaxRetries, "retry", 2, "maximum number of retries for dns resolution (must be at least 1)"), ) flagSet.CreateGroup("update", "UPDATE", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 97efe2c..9b9a8cf 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -13,41 +13,29 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/cdncheck" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/mapcidr" - errorutils "github.com/projectdiscovery/utils/errors" iputils "github.com/projectdiscovery/utils/ip" - urlutils "github.com/projectdiscovery/utils/url" ) type Runner struct { - options *Options - cdnclient *cdncheck.Client - fastdialer *fastdialer.Dialer - aurora *aurora.Aurora - writer *OutputWriter + options *Options + cdnclient *cdncheck.Client + aurora *aurora.Aurora + writer *OutputWriter } func NewRunner(options *Options) *Runner { standardWriter := aurora.NewAurora(!options.NoColor) + client, err := cdncheck.NewWithOpts(options.MaxRetries, options.Resolvers) + if err != nil { + gologger.Fatal().Msgf("failed to create cdncheck client: %v", err) + } runner := &Runner{ options: options, - cdnclient: cdncheck.New(), + cdnclient: client, aurora: &standardWriter, } - fOption := fastdialer.DefaultOptions - if len(options.Resolvers) > 0 { - fOption.BaseResolvers = options.Resolvers - } - fdialer, err := fastdialer.NewDialer(fOption) - if err != nil { - if options.Verbose { - gologger.Error().Msgf("%v: failed to initialize dailer", err.Error()) - } - return runner - } - runner.fastdialer = fdialer return runner } @@ -196,50 +184,49 @@ func (r *Runner) processInputItemSingle(item string, output chan Output) { aurora: r.aurora, Input: item, } - if !iputils.IsIP(item) { - ipAddr, err := r.resolveToIP(item) - if err != nil { - if r.options.Verbose { - gologger.Error().Msgf("Could not parse domain/url %s: %s", item, err) - } - return - } - item = ipAddr - } - parsed := net.ParseIP(item) - if parsed == nil { + if iputils.IsIPv6(item) { + // TODO: IPv6 support refer issue #59 if r.options.Verbose { - gologger.Error().Msgf("Could not parse IP address: %s", item) + gologger.Error().Msgf("IPv6 is not supported: %s", item) } return } - isCDN, provider, itemType, err := r.cdnclient.Check(parsed) - if err != nil { - if r.options.Verbose { - gologger.Error().Msgf("Could not check IP cdn %s: %s", item, err) - } - return + + var matched bool + var provider, itemType string + var err error + + if iputils.IsIP(item) { + targetIP := net.ParseIP(item) + matched, provider, itemType, err = r.cdnclient.Check(targetIP) + } else { + matched, provider, itemType, err = r.cdnclient.CheckDomainWithFallback(item) } + if err != nil && r.options.Verbose { + gologger.Error().Msgf("Could not check domain cdn %s: %s", item, err) + } + data.itemType = itemType data.IP = item data.Timestamp = time.Now() if r.options.Exclude { - if !isCDN { + if !matched { output <- data } return } + switch itemType { case "cdn": - data.Cdn = isCDN + data.Cdn = matched data.CdnName = provider case "cloud": - data.Cloud = isCDN + data.Cloud = matched data.CloudName = provider case "waf": - data.Waf = isCDN + data.Waf = matched data.WafName = provider } if skipped := filterIP(r.options, data); skipped { @@ -249,34 +236,17 @@ func (r *Runner) processInputItemSingle(item string, output chan Output) { return } switch { - case r.options.Cdn && data.itemType == "cdn", - r.options.Cloud && data.itemType == "cloud", - r.options.Waf && data.itemType == "waf": + case r.options.Cdn && data.itemType == "cdn", r.options.Cloud && data.itemType == "cloud", r.options.Waf && data.itemType == "waf": { output <- data } - case (!r.options.Cdn && !r.options.Waf && !r.options.Cloud) && isCDN: + case (!r.options.Cdn && !r.options.Waf && !r.options.Cloud) && matched: { output <- data } } } -func (r *Runner) resolveToIP(domain string) (string, error) { - url, err := urlutils.Parse(domain) - if err != nil { - return domain, err - } - dsnData, err := r.fastdialer.GetDNSData(url.Host) - if err != nil { - return domain, err - } - if len(dsnData.A) < 1 { - return domain, errorutils.New("failed to resolve domain") - } - return dsnData.A[0], nil -} - func matchIP(options *Options, data Output) bool { if len(options.MatchCdn) == 0 && len(options.MatchCloud) == 0 && len(options.MatchWaf) == 0 { return true diff --git a/other.go b/other.go index df7e73e..41e8a63 100644 --- a/other.go +++ b/other.go @@ -19,7 +19,7 @@ var cdnWappalyzerTechnologies = map[string]string{ } // CheckFQDN checks if fqdns are known cloud ones -func (c *Client) CheckSuffix(fqdns ...string) (bool, string, error) { +func (c *Client) CheckSuffix(fqdns ...string) (isCDN bool, provider string, itemType string, err error) { c.Once.Do(func() { suffixToSource = make(map[string]string) for source, suffixes := range generatedData.Common { @@ -31,21 +31,21 @@ func (c *Client) CheckSuffix(fqdns ...string) (bool, string, error) { for _, fqdn := range fqdns { parsed, err := publicsuffix.Parse(fqdn) if err != nil { - return false, "", errors.Wrap(err, "could not parse fqdn") + return false, "", "", errors.Wrap(err, "could not parse fqdn") } if discovered, ok := suffixToSource[parsed.TLD]; ok { - return true, discovered, nil + return true, discovered, "waf", nil } domain := parsed.SLD + "." + parsed.TLD if discovered, ok := suffixToSource[domain]; ok { - return true, discovered, nil + return true, discovered, "waf", nil } } - return false, "", nil + return false, "", "", nil } // CheckWappalyzer checks if the wappalyzer detection are a part of CDN -func (c *Client) CheckWappalyzer(data map[string]struct{}) (bool, string, error) { +func (c *Client) CheckWappalyzer(data map[string]struct{}) (isCDN bool, provider string, err error) { for technology := range data { if strings.Contains(technology, ":") { if parts := strings.SplitN(technology, ":", 2); len(parts) == 2 { diff --git a/other_test.go b/other_test.go index 3a700dc..9b99fc8 100644 --- a/other_test.go +++ b/other_test.go @@ -3,18 +3,19 @@ package cdncheck import ( "testing" + "github.com/projectdiscovery/retryabledns" "github.com/stretchr/testify/require" ) func TestCheckSuffix(t *testing.T) { client := New() - valid, provider, err := client.CheckSuffix("test.cloudfront.net") + valid, provider, _, err := client.CheckSuffix("test.cloudfront.net") require.Nil(t, err, "could not check cname") require.True(t, valid, "could not get valid cname") require.Equal(t, "amazon", provider, "could not get correct provider") - valid, _, err = client.CheckSuffix("test.provider.net") + valid, _, _, err = client.CheckSuffix("test.provider.net") require.Nil(t, err, "could not check cname") require.False(t, valid, "could get valid cname") } @@ -36,3 +37,37 @@ func TestCheckWappalyzer(t *testing.T) { require.Nil(t, err, "could not check cname") require.False(t, valid, "could get valid cname") } + +func TestCheckDomainWithFallback(t *testing.T) { + client := New() + + valid, provider, itemType, err := client.CheckDomainWithFallback("www.gap.com") + require.Nil(t, err, "could not check") + require.True(t, valid, "could not check domain") + require.Equal(t, "akamai", provider, "could not get correct provider") + require.Equal(t, "waf", itemType, "could not get correct itemType") +} + +func TestCheckDNSResponse(t *testing.T) { + client := New() + defaultResolvers := []string{"8.8.8.8", "8.8.0.0"} + defaultMaxRetries := 3 + retryabledns, _ := retryabledns.New(defaultResolvers, defaultMaxRetries) + dnsData, _ := retryabledns.Resolve("hackerone.com") + + valid, provider, itemType, err := client.CheckDNSResponse(dnsData) + + require.Nil(t, err, "could not check cname") + require.True(t, valid, "could not get valid cname") + require.Equal(t, "cloudflare", provider, "could not get correct provider") + require.Equal(t, "waf", itemType, "could not get correct itemType") + + dnsData, _ = retryabledns.CNAME("www.gap.com") + + valid, provider, itemType, err = client.CheckDNSResponse(dnsData) + require.Nil(t, err, "could not check") + require.True(t, valid, "could not check domain") + require.Equal(t, "akamai", provider, "could not get correct provider") + require.Equal(t, "waf", itemType, "could not get correct itemType") + +}