Merge branch 'dev'

projectdiscovery · May 19, 2024 · 1b8a982 · 1b8a982
2 parents 76e61cc + 870a621
commit 1b8a982
Show file tree

Hide file tree

Showing 12 changed files with 196 additions and 104 deletions.
diff --git a/README.md b/README.md
@@ -114,7 +114,7 @@ PROBES:
    -ip                   display host ip
    -cname                display host cname
    -asn                  display host asn information
-   -cdn                  display cdn/waf in use
+   -cdn                  display cdn/waf in use (default true)
    -probe                display probe status
 
 HEADLESS:
@@ -139,7 +139,7 @@ MATCHERS:
 
 EXTRACTOR:
    -er, -extract-regex string[]   display response content with matched regex
-   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (url,ipv4,mail)
+   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (mail,url,ipv4)
 
 FILTERS:
    -fc, -filter-code string            filter response with specified status code (-fc 403,401)
@@ -190,9 +190,11 @@ OUTPUT:
    -include-chain                      include redirect http chain in JSON output (-json only)
    -store-chain                        include http redirect chain in responses (-sr only)
    -svrc, -store-vision-recon-cluster  include visual recon clusters (-ss and -sr only)
+   -pr, -protocol string               protocol to use (unknown, http11)
 
 CONFIGURATIONS:
    -config string                path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
+   -auth                         configure projectdiscovery cloud (pdcp) api key (default true)
    -r, -resolvers string[]       list of custom resolver (file or comma separated)
    -allow string[]               allowed list of IP/CIDR's to process (file or comma separated)
    -deny string[]                denied list of IP/CIDR's to process (file or comma separated)

diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go
@@ -74,7 +74,7 @@ func (h *httpxLibraryWithStream) Execute() error {
 		RateLimit:  150,
 		Retries:    2,
 		Timeout:    10,
-		TechDetect: "true",
+		TechDetect: true,
 		Stream:     true,
 		SkipDedupe: true,
 		OnResult: func(r runner.Result) {

diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -152,6 +153,12 @@ func New(options *Options) (*HTTPX, error) {
 		DisableKeepAlives: true,
 	}
 
+	if httpx.Options.Protocol == "http11" {
+		// disable http2
+		os.Setenv("GODEBUG", "http2client=0")
+		transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+	}
+
 	if httpx.Options.SniName != "" {
 		transport.TLSClientConfig.ServerName = httpx.Options.SniName
 	}
@@ -288,10 +295,12 @@ get_response:
 
 	// fill metrics
 	resp.StatusCode = httpresp.StatusCode
-	// number of words
-	resp.Words = len(strings.Split(respbodystr, " "))
-	// number of lines
-	resp.Lines = len(strings.Split(respbodystr, "\n"))
+	if respbodystr != "" {
+		// number of words
+		resp.Words = len(strings.Split(respbodystr, " "))
+		// number of lines
+		resp.Lines = len(strings.Split(strings.TrimSpace(respbodystr), "\n"))
+	}
 
 	if !h.Options.Unsafe && h.Options.TLSGrab {
 		if h.Options.ZTLS {

diff --git a/common/httpx/option.go b/common/httpx/option.go
@@ -45,6 +45,7 @@ type Options struct {
 	SniName                   string
 	TlsImpersonate            bool
 	NetworkPolicy             *networkpolicy.NetworkPolicy
+	Protocol                  Proto
 }
 
 // DefaultOptions contains the default options

diff --git a/common/httpx/proto.go b/common/httpx/proto.go
@@ -0,0 +1,10 @@
+package httpx
+
+type Proto string
+
+const (
+	UNKNOWN Proto = ""
+	HTTP11  Proto = "http11"
+	HTTP2   Proto = "http2"
+	HTTP3   Proto = "http3"
+)
diff --git a/common/httpx/title.go b/common/httpx/title.go
@@ -9,12 +9,22 @@ import (
 
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	"golang.org/x/net/html"
+	"slices"
 )
 
 var (
-	cutset        = "\n\t\v\f\r"
-	reTitle       = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
-	reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
+	cutset                  = "\n\t\v\f\r"
+	reTitle                 = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
+	reContentType           = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
+	supportedTitleMimeTypes = []string{
+		"text/html",
+		"application/xhtml+xml",
+		"application/xml",
+		"application/rss+xml",
+		"application/atom+xml",
+		"application/xhtml+xml",
+		"application/vnd.wap.xhtml+xml",
+	}
 )
 
 // ExtractTitle from a response
@@ -40,6 +50,10 @@ func ExtractTitle(r *Response) (title string) {
 	return title
 }
 
+func CanHaveTitleTag(mimeType string) bool {  
+    return slices.Contains(supportedTitleMimeTypes, mimeType)  
+}  
+
 func getTitleWithDom(r *Response) (*html.Node, error) {
 	var title *html.Node
 	var crawler func(*html.Node)

diff --git a/go.mod b/go.mod
@@ -23,22 +23,22 @@ require (
 	github.com/projectdiscovery/asnmap v1.1.0
 	github.com/projectdiscovery/cdncheck v1.0.9
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/dsl v0.0.46
-	github.com/projectdiscovery/fastdialer v0.0.61
+	github.com/projectdiscovery/dsl v0.0.56
+	github.com/projectdiscovery/fastdialer v0.0.71
 	github.com/projectdiscovery/fdmax v0.0.4
 	github.com/projectdiscovery/goconfig v0.0.1
-	github.com/projectdiscovery/goflags v0.1.41
+	github.com/projectdiscovery/goflags v0.1.52
 	github.com/projectdiscovery/gologger v1.1.12
-	github.com/projectdiscovery/hmap v0.0.41
-	github.com/projectdiscovery/mapcidr v1.1.16
+	github.com/projectdiscovery/hmap v0.0.42
+	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/networkpolicy v0.0.8
-	github.com/projectdiscovery/ratelimit v0.0.27
-	github.com/projectdiscovery/rawhttp v0.1.39
-	github.com/projectdiscovery/retryablehttp-go v1.0.49
+	github.com/projectdiscovery/ratelimit v0.0.40
+	github.com/projectdiscovery/rawhttp v0.1.49
+	github.com/projectdiscovery/retryablehttp-go v1.0.59
 	github.com/projectdiscovery/tlsx v1.1.6
-	github.com/projectdiscovery/useragent v0.0.38
-	github.com/projectdiscovery/utils v0.0.82
-	github.com/projectdiscovery/wappalyzergo v0.0.112
+	github.com/projectdiscovery/useragent v0.0.48
+	github.com/projectdiscovery/utils v0.0.92
+	github.com/projectdiscovery/wappalyzergo v0.1.0
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
 	github.com/spaolacci/murmur3 v1.1.0
@@ -47,9 +47,9 @@ require (
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/multierr v1.11.0
 	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
-	golang.org/x/net v0.23.0
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0
+	golang.org/x/net v0.25.0
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0
 )
 
 require (
@@ -81,6 +81,7 @@ require (
 	github.com/google/certificate-transparency-go v1.1.4 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.3.1 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -109,7 +110,7 @@ require (
 	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/gostruct v0.0.2 // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/projectdiscovery/retryabledns v1.0.57 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.59 // indirect
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
 	github.com/quic-go/quic-go v0.42.0 // indirect
 	github.com/refraction-networking/utls v1.5.4 // indirect
@@ -128,10 +129,9 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
 	github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
@@ -142,12 +142,14 @@ require (
 	github.com/ysmood/leakless v0.8.0 // indirect
 	github.com/yuin/goldmark v1.5.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
-	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	github.com/zcalusic/sysinfo v1.0.2 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect