diff --git a/README.md b/README.md index 0c0732d..070d178 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ PROBES: -lc, -line-count display response body line count -wc, -word-count display response body word count -title display page title + -bp, -body-preview display first N characters of response body (default 100) -server, -web-server display server name -td, -tech-detect display technology in use based on wappalyzer dataset -method display http request method @@ -111,7 +112,7 @@ PROBES: -ip display host ip -cname display host cname -asn display host asn information - -cdn display cdn in use + -cdn display cdn/waf in use -probe display probe status HEADLESS: @@ -146,6 +147,7 @@ FILTERS: -fcdn, -filter-cdn string[] filter host with specified cdn provider (google, leaseweb, stackpath, cloudfront, fastly) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition + -strip strips all tags in response. supported formats: html,xml (default html) RATE-LIMIT: -t, -threads int number of threads to use (default 50) @@ -222,7 +224,7 @@ OPTIMIZATIONS: -nf, -no-fallback display both probed protocol (HTTPS and HTTP) -nfs, -no-fallback-scheme probe with protocol scheme specified in input -maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30) - -ec, -exclude-cdn skip full port scans for CDNs (only checks for 80,443) + -ec, -exclude-cdn skip full port scans for CDN/WAF (only checks for 80,443) -retries int number of retries -timeout int timeout in seconds (default 5) -delay duration duration between each http request (eg: 200ms, 1s) (default -1ns) @@ -556,6 +558,16 @@ Screenshots are stored in the output/screenshot directory by default. To specify httpx -screenshot -srd /path/to/custom/directory -u https://example.com ``` +### Body Preview & Strip HTML +Body preview shows first N characters of response. And strip html tags in response +```console +httpx -u https://example.com -silent -bp -strip +https://example.com [ Example Domain This domain is for use in illustrative examples in documents. You may use this domai] + +httpx -u https://example.com -silent -bp=200 -strip=html +https://example.com [ Example Domain This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission. More information...] +``` + #### ⏳ Performance Considerations Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option. diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index e7553c7..8efc8e3 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -19,4 +19,5 @@ scanme.sh {{binary}} -silent -ztls scanme.sh {{binary}} -silent -jarm https://scanme.sh?a=1*1 {{binary}} -silent https://scanme.sh:443 {{binary}} -asn -scanme.sh {{binary}} -silent -tls-impersonate \ No newline at end of file +scanme.sh {{binary}} -silent -tls-impersonate +example.com {{binary}} -silent -bp -strip \ No newline at end of file diff --git a/common/httputilz/normalize.go b/common/httputilz/normalize.go new file mode 100644 index 0000000..e266767 --- /dev/null +++ b/common/httputilz/normalize.go @@ -0,0 +1,11 @@ +package httputilz + +import "regexp" + +var ( + normalizeSpacesRegex = regexp.MustCompile(`\s+`) +) + +func NormalizeSpaces(data string) string { + return normalizeSpacesRegex.ReplaceAllString(data, " ") +} diff --git a/common/httpx/cdn.go b/common/httpx/cdn.go index b117c52..6d0bb5f 100644 --- a/common/httpx/cdn.go +++ b/common/httpx/cdn.go @@ -5,11 +5,17 @@ import ( "net" ) -// CdnCheck verifies if the given ip is part of Cdn ranges +// CdnCheck verifies if the given ip is part of Cdn/WAF ranges func (h *HTTPX) CdnCheck(ip string) (bool, string, error) { if h.cdn == nil { return false, "", fmt.Errorf("cdn client not configured") } - return h.cdn.CheckCDN(net.ParseIP((ip))) + // the goal is to check if ip is part of cdn/waf to decide if target should be scanned or not + // since 'cloud' itemtype does not fit logic here , we consider target is not part of cdn/waf + matched, value, itemType, err := h.cdn.Check(net.ParseIP((ip))) + if itemType == "cloud" { + return false, "", err + } + return matched, value, err } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 2e59bb6..0eebf7b 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" + "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/generic" @@ -410,3 +411,14 @@ func (httpx *HTTPX) setCustomCookies(req *http.Request) { } } } + +func (httpx *HTTPX) Sanitize(respStr string, trimLine, normalizeSpaces bool) string { + respStr = httpx.htmlPolicy.Sanitize(respStr) + if trimLine { + respStr = strings.Replace(respStr, "\n", "", -1) + } + if normalizeSpaces { + respStr = httputilz.NormalizeSpaces(respStr) + } + return respStr +} diff --git a/go.mod b/go.mod index a8d2a87..578e6d4 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/clistats v0.0.19 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.11 + github.com/projectdiscovery/goflags v0.1.14-0.20230809193030-a634ac4b5c5d github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/hmap v0.0.13 github.com/projectdiscovery/mapcidr v1.1.2 @@ -26,9 +26,9 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/net v0.12.0 - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 + golang.org/x/net v0.14.0 + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 ) require github.com/spaolacci/murmur3 v1.1.0 @@ -47,11 +47,11 @@ require ( github.com/projectdiscovery/fastdialer v0.0.35 github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/tlsx v1.1.1 - github.com/projectdiscovery/utils v0.0.44 + github.com/projectdiscovery/utils v0.0.48 github.com/stretchr/testify v1.8.4 github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 go.uber.org/multierr v1.11.0 - golang.org/x/exp v0.0.0-20230420155640-133eef4313cb + golang.org/x/exp v0.0.0-20230810033253-352e893a4cad ) require ( @@ -112,7 +112,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/sashabaranov/go-openai v1.14.1 // indirect - github.com/shirou/gopsutil/v3 v3.23.6 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -140,10 +140,10 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - golang.org/x/crypto v0.11.0 // indirect - golang.org/x/mod v0.10.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/tools v0.8.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/tools v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index 7034627..b49814c 100644 --- a/go.sum +++ b/go.sum @@ -57,7 +57,6 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/gaukas/godicttls v0.0.3 h1:YNDIf0d9adcxOijiLrEzpfZGAkNwLRzPaG6OjU7EITk= @@ -125,7 +124,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -193,8 +191,6 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.14 h1:CAxCoYbIEBCuINiMR1UKA1v6ifmub3P5hCwzBmmkh0c= -github.com/projectdiscovery/dsl v0.0.14/go.mod h1:3K2GmExpriruVHsVJmsTugxR7H9wVpUo8/+jWXXbSSw= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= @@ -205,8 +201,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.11 h1:C4UTO3SM5Vfy1J2sdhukm7wONW/tljMpUMNKue5ie00= -github.com/projectdiscovery/goflags v0.1.11/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= +github.com/projectdiscovery/goflags v0.1.14-0.20230809193030-a634ac4b5c5d h1:kSDVdcwv/K8LKq9M7i6GRHpM9hrbQonCUAfFPuMJKO4= +github.com/projectdiscovery/goflags v0.1.14-0.20230809193030-a634ac4b5c5d/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= @@ -226,8 +222,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.19/go.mod h1:GurI1InT6ak7xklIi github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= -github.com/projectdiscovery/utils v0.0.44 h1:F/LNgBw53RNM/3mRZ1ji+prM1yDnehDRBf13TPk3WBM= -github.com/projectdiscovery/utils v0.0.44/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/utils v0.0.48 h1:eXJfOYQ3whDIo4uBX68UiPCLCmGE7Isv9348YukaCbY= +github.com/projectdiscovery/utils v0.0.48/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= github.com/projectdiscovery/wappalyzergo v0.0.105 h1:8Uag57UUzZSjzKgaOZKLB+vW1VlXUcwbDKgy2fE+VUs= github.com/projectdiscovery/wappalyzergo v0.0.105/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= @@ -243,12 +239,10 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.13.0 h1:EAusFfnhaMaaUspUZ2+MbB/ZcVeD4epJmTOlZ+8AcAE= -github.com/sashabaranov/go-openai v1.13.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/sashabaranov/go-openai v1.14.1 h1:jqfkdj8XHnBF84oi2aNtT8Ktp3EJ0MfuVjvcMkfI0LA= github.com/sashabaranov/go-openai v1.14.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= -github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= -github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -342,7 +336,6 @@ github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+ github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -356,15 +349,15 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -383,18 +376,18 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -418,9 +411,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -435,15 +428,15 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -459,7 +452,6 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -471,7 +463,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/runner/options.go b/runner/options.go index 22c2176..37a41dc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -229,6 +229,7 @@ type Options struct { Allow customlist.CustomList MaxResponseBodySizeToSave int MaxResponseBodySizeToRead int + ResponseBodyPreviewSize int OutputExtractRegexs goflags.StringSlice OutputExtractPresets goflags.StringSlice RateLimit int @@ -270,6 +271,7 @@ type Options struct { ListDSLVariable bool OutputFilterCondition string OutputMatchCondition string + StripFilter string //The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err. OnResult OnResultCallback DisableUpdateCheck bool @@ -308,6 +310,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"), flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"), flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"), + flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"), flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), @@ -356,6 +359,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"), + flagSet.DynamicVar(&options.StripFilter, "strip", "html", "strips all tags in response. supported formats: html,xml"), ) flagSet.CreateGroup("rate-limit", "Rate-Limit", @@ -479,6 +483,10 @@ func ParseOptions() *Options { options.ShowStatistics = true } + if options.ResponseBodyPreviewSize > 0 && options.StripFilter == "" { + options.StripFilter = "html" + } + // Read the inputs and configure the logging options.configureOutput() diff --git a/runner/runner.go b/runner/runner.go index 9700a08..106c95e 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1489,6 +1489,28 @@ retry: builder.WriteRune(']') } + var bodyPreview string + if r.options.ResponseBodyPreviewSize > 0 && resp != nil { + bodyPreview = string(resp.Data) + if stringsutil.EqualFoldAny(r.options.StripFilter, "html", "xml") { + bodyPreview = r.hp.Sanitize(bodyPreview, true, true) + } else { + bodyPreview = strings.ReplaceAll(bodyPreview, "\n", "\\n") + bodyPreview = httputilz.NormalizeSpaces(bodyPreview) + } + if len(bodyPreview) > r.options.ResponseBodyPreviewSize { + bodyPreview = bodyPreview[:r.options.ResponseBodyPreviewSize] + } + bodyPreview = strings.TrimSpace(bodyPreview) + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Blue(bodyPreview).String()) + } else { + builder.WriteString(bodyPreview) + } + builder.WriteRune(']') + } + serverHeader := resp.GetHeader("Server") if scanopts.OutputServerHeader { builder.WriteString(fmt.Sprintf(" [%s]", serverHeader)) @@ -1794,11 +1816,10 @@ retry: if len(respRaw) > scanopts.MaxResponseBodySizeToSave { respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] } - data := append([]byte(fullURL), append([]byte("\n\n"), reqRaw...)...) - if scanopts.StoreChain && resp.HasChain() { - data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...) - } - data = append(data, append([]byte("\n"), respRaw...)...) + data := reqRaw + data = append(data, respRaw...) + data = append(data, []byte("\n\n\n")...) + data = append(data, []byte(fullURL)...) _ = fileutil.CreateFolder(responseBaseDir) writeErr := os.WriteFile(responsePath, data, 0644) if writeErr != nil { @@ -1880,6 +1901,7 @@ retry: VHost: isvhost, WebServer: serverHeader, ResponseBody: serverResponseRaw, + BodyPreview: bodyPreview, WebSocket: isWebSocket, TLSData: resp.TLSData, CSPData: resp.CSPData, @@ -1905,7 +1927,7 @@ retry: ExtractRegex: extractRegex, StoredResponsePath: responsePath, ScreenshotBytes: screenshotBytes, - ScreenshotPath: screenshotPath, + ScreenshotPath: filepath.Join(hostFilename, screenshotResponseFile), HeadlessBody: headlessBody, KnowledgeBase: map[string]interface{}{ "PageType": r.errorPageClassifier.Classify(respData), @@ -1938,6 +1960,7 @@ func (r *Runner) handleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } if URL.IsAbs() { req.SetURL(URL) + req.Host = URL.Host faviconPath = "" } else { faviconPath = URL.String() diff --git a/runner/types.go b/runner/types.go index 2b24f37..d245d23 100644 --- a/runner/types.go +++ b/runner/types.go @@ -50,6 +50,7 @@ type Result struct { Error string `json:"error,omitempty" csv:"error"` WebServer string `json:"webserver,omitempty" csv:"webserver"` ResponseBody string `json:"body,omitempty" csv:"body"` + BodyPreview string `json:"body_preview,omitempty" csv:"body_preview"` ContentType string `json:"content_type,omitempty" csv:"content_type"` Method string `json:"method,omitempty" csv:"method"` Host string `json:"host,omitempty" csv:"host"` @@ -133,24 +134,7 @@ func resultToMap(resp Result) (map[string]any, error) { if err != nil { return nil, fmt.Errorf("error decoding: %v", err) } - return flatten(m), nil -} - -// mapsutil.Flatten w/o separator -func flatten(m map[string]any) map[string]any { - o := make(map[string]any) - for k, v := range m { - switch child := v.(type) { - case map[string]any: - nm := flatten(child) - for nk, nv := range nm { - o[nk] = nv - } - default: - o[k] = v - } - } - return o + return m, nil } var (