merge conflict

README.md

@@ -104,14 +104,15 @@ PROBES:
    -lc, -line-count      display response body line count
    -wc, -word-count      display response body word count
    -title                display page title
+   -bp, -body-preview    display first N characters of response body (default 100)
    -server, -web-server  display server name
    -td, -tech-detect     display technology in use based on wappalyzer dataset
    -method               display http request method
    -websocket            display server using websocket
    -ip                   display host ip
    -cname                display host cname
    -asn                  display host asn information
-   -cdn                  display cdn in use
+   -cdn                  display cdn/waf in use
    -probe                display probe status
 
 HEADLESS:
@@ -146,6 +147,7 @@ FILTERS:
    -fcdn, -filter-cdn string[]         filter host with specified cdn provider (google, leaseweb, stackpath, cloudfront, fastly)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
+   -strip                              strips all tags in response. supported formats: html,xml (default html)
 
 RATE-LIMIT:
    -t, -threads int              number of threads to use (default 50)
@@ -222,7 +224,7 @@ OPTIMIZATIONS:
    -nf, -no-fallback                  display both probed protocol (HTTPS and HTTP)
    -nfs, -no-fallback-scheme          probe with protocol scheme specified in input 
    -maxhr, -max-host-error int        max error count per host before skipping remaining path/s (default 30)
-   -ec, -exclude-cdn                  skip full port scans for CDNs (only checks for 80,443)
+   -ec, -exclude-cdn                  skip full port scans for CDN/WAF (only checks for 80,443)
    -retries int                       number of retries
    -timeout int                       timeout in seconds (default 5)
    -delay duration                    duration between each http request (eg: 200ms, 1s) (default -1ns)
@@ -556,6 +558,16 @@ Screenshots are stored in the output/screenshot directory by default. To specify
 httpx -screenshot -srd /path/to/custom/directory -u https://example.com
 ```
 
+### Body Preview & Strip HTML
+Body preview shows first N characters of response. And strip html tags in response
+```console
+httpx -u https://example.com -silent -bp -strip
+https://example.com [ Example Domain This domain is for use in illustrative examples in documents. You may use this domai]
+
+httpx -u https://example.com -silent -bp=200 -strip=html
+https://example.com [ Example Domain This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission. More information...]
+```
+
 #### ⏳ Performance Considerations
 
 Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option.

cmd/functional-test/testcases.txt

@@ -19,4 +19,5 @@ scanme.sh {{binary}} -silent -ztls
 scanme.sh {{binary}} -silent -jarm
 https://scanme.sh?a=1*1 {{binary}} -silent
 https://scanme.sh:443 {{binary}} -asn
-scanme.sh {{binary}} -silent -tls-impersonate
+scanme.sh {{binary}} -silent -tls-impersonate
+example.com {{binary}} -silent -bp -strip

common/httputilz/normalize.go

@@ -0,0 +1,11 @@
+package httputilz
+
+import "regexp"
+
+var (
+	normalizeSpacesRegex = regexp.MustCompile(`\s+`)
+)
+
+func NormalizeSpaces(data string) string {
+	return normalizeSpacesRegex.ReplaceAllString(data, " ")
+}

common/httpx/cdn.go

@@ -5,11 +5,17 @@ import (
 	"net"
 )
 
-// CdnCheck verifies if the given ip is part of Cdn ranges
+// CdnCheck verifies if the given ip is part of Cdn/WAF ranges
 func (h *HTTPX) CdnCheck(ip string) (bool, string, error) {
 	if h.cdn == nil {
 		return false, "", fmt.Errorf("cdn client not configured")
 	}
 
-	return h.cdn.CheckCDN(net.ParseIP((ip)))
+	// the goal is to check if ip is part of cdn/waf to decide if target should be scanned or not
+	// since 'cloud' itemtype does not fit logic here , we consider target is not part of cdn/waf
+	matched, value, itemType, err := h.cdn.Check(net.ParseIP((ip)))
+	if itemType == "cloud" {
+		return false, "", err
+	}
+	return matched, value, err
 }

common/httpx/httpx.go

@@ -16,6 +16,7 @@ import (
 	"github.com/projectdiscovery/cdncheck"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
+	"github.com/projectdiscovery/httpx/common/httputilz"
 	"github.com/projectdiscovery/rawhttp"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 	"github.com/projectdiscovery/utils/generic"
@@ -410,3 +411,14 @@ func (httpx *HTTPX) setCustomCookies(req *http.Request) {
 		}
 	}
 }
+
+func (httpx *HTTPX) Sanitize(respStr string, trimLine, normalizeSpaces bool) string {
+	respStr = httpx.htmlPolicy.Sanitize(respStr)
+	if trimLine {
+		respStr = strings.Replace(respStr, "\n", "", -1)
+	}
+	if normalizeSpaces {
+		respStr = httputilz.NormalizeSpaces(respStr)
+	}
+	return respStr
+}

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/projectdiscovery/clistats v0.0.19
 	github.com/projectdiscovery/fdmax v0.0.4
 	github.com/projectdiscovery/goconfig v0.0.1
-	github.com/projectdiscovery/goflags v0.1.11
+	github.com/projectdiscovery/goflags v0.1.14-0.20230809193030-a634ac4b5c5d
 	github.com/projectdiscovery/gologger v1.1.11
 	github.com/projectdiscovery/hmap v0.0.13
 	github.com/projectdiscovery/mapcidr v1.1.2
@@ -26,9 +26,9 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
 	go.etcd.io/bbolt v1.3.7 // indirect
-	golang.org/x/net v0.12.0
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/text v0.11.0
+	golang.org/x/net v0.14.0
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/text v0.12.0
 )
 
 require github.com/spaolacci/murmur3 v1.1.0
@@ -47,11 +47,11 @@ require (
 	github.com/projectdiscovery/fastdialer v0.0.35
 	github.com/projectdiscovery/ratelimit v0.0.9
 	github.com/projectdiscovery/tlsx v1.1.1
-	github.com/projectdiscovery/utils v0.0.44
+	github.com/projectdiscovery/utils v0.0.48
 	github.com/stretchr/testify v1.8.4
 	github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101
 	go.uber.org/multierr v1.11.0
-	golang.org/x/exp v0.0.0-20230420155640-133eef4313cb
+	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
 )
 
 require (
@@ -112,7 +112,7 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/sashabaranov/go-openai v1.14.1 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.6 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.7 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
@@ -140,10 +140,10 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/tools v0.8.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/oauth2 v0.11.0 // indirect
+	golang.org/x/tools v0.12.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect