diff --git a/README.md b/README.md index e8058ef3..2357b8c3 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,10 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi # Installation Instructions -httpx requires **go1.14+** to install successfully. Run the following command to get the repo - +httpx requires **go1.17** to install successfully. Run the following command to get the repo - ```sh -GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx +go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest ``` # Usage @@ -73,156 +73,103 @@ httpx -h This will display help for the tool. Here are all the switches it supports. -
- 👉 httpx help menu 👈 +```console +Usage: + ./httpx [flags] + +Flags: +INPUT: + -l, -list string Input file containing list of hosts to process + -request string File containing raw request + +PROBES: + -sc, -status-code Display Status Code + -tech, -tech-detect Display wappalyzer based technology detection + -cl, -content-length Display Content-Length + -server, -web-server Display Server header + -ct, -content-type Display Content-Type header + -rt, -response-time Display the response time + -title Display page title + -location Display Location header + -method Display Request method + -websocket Display server using websocket + -ip Display Host IP + -cname Display Host cname + -cdn Display if CDN in use + -probe Display probe status + -nf, -no-fallback Display both protocol (HTTPS and HTTP) + +MATCHERS: + -mc, -match-code string Match response with given status code (-mc 200,302) + -ml, -match-length string Match response with given content length (-ml 100,102) + -ms, -match-string string Match response with given string + -mr, -match-regex string Match response with specific regex + -er, -extract-regex string Display response content with matched regex + +FILTERS: + -fc, -filter-code string Filter response with given status code (-fc 403,401) + -fl, -filter-length string Filter response with given content length (-fl 23,33) + -fs, -filter-string string Filter response with specific string + -fe, -filter-regex string Filter response with specific regex + +RATE-LIMIT: + -t, -threads int Number of threads (default 50) + -rl, -rate-limit int Maximum requests to send per second (default 150) + +MISCELLANEOUS: + -tls-grab Perform TLS(SSL) data grabbing + -tls-probe Send HTTP probes on the extracted TLS domains + -csp-probe Send HTTP probes on the extracted CSP domains + -pipeline HTTP1.1 Pipeline probe + -http2 HTTP2 probe + -vhost VHOST Probe + -p, -ports string[] Port to scan (nmap syntax: eg 1,2-10,11) + -path string File or comma separated paths to request + -paths string File or comma separated paths to request (deprecated) + +OUTPUT: + -o, -output string File to write output + -sr, -store-response Store HTTP responses + -srd, -store-response-dir string Custom directory to store HTTP responses (default "output") + -json Output in JSONL(ines) format + -irr, -include-response Include HTTP request/response in JSON output (-json only) + -include-chain Include redirect HTTP Chain in JSON output (-json only) + -store-chain Include HTTP redirect chain in responses (-sr only) + -csv Output in CSV format + +CONFIGURATIONS: + -rsts, -response-size-to-save int Max response size to save in bytes (default 2147483647) + -rstr, -response-size-to-read int Max response size to read in bytes (default 2147483647) + -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) + -deny string[] Denied list of IP/CIDR's to process (file or comma separated) + -random-agent Enable Random User-Agent to use (default true) + -H, -header string[] Custom Header to send with request + -http-proxy, -proxy string HTTP Proxy, eg http://127.0.0.1:8080 + -unsafe Send raw requests skipping golang normalization + -resume Resume scan using resume.cfg + -nc, -no-color Disable color in output + -nfs, -no-fallback-scheme Probe with input protocol scheme + -fr, -follow-redirects Follow HTTP redirects + -fhr, -follow-host-redirects Follow redirects on the same host + -maxr, -max-redirects int Max number of redirects to follow per host (default 10) + -vhost-input Get a list of vhosts as input + -x string Request methods to use, use 'all' to probe all HTTP methods + -body string Post body to include in HTTP request + +OPTIMIZATIONS: + -retries int Number of retries + -timeout int Timeout in seconds (default 5) + -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) + -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) + +DEBUG: + -silent Silent mode + -verbose Verbose mode + -version Display version + -debug Debug mode + -stats Display scan statistic ``` - -H value - Custom Header to send with request - -allow value - Allow list of IP/CIDR's to process (file or comma separated) - -body string - Content to send in body with HTTP request - -cdn - Diplay CDN - -cname - Display Host cname - -content-length - Display HTTP response content length - -content-type - Display content-type header - -csp-probe - Send HTTP probes on the extracted CSP domains - -csv - Display output in CSV format - -debug - Debug mode - -deny value - Deny list of IP/CIDR's to process (file or comma separated) - -exclude-cdn - Skip full port scans for CDNs (only checks for 80,443) - -extract-regex string - Display response content with matched regex - -fc string - Filter response with specific status code (-fc 403,401) - -filter-regex string - Filter response with specific regex - -filter-string string - Filter response with specific string - -fl string - Filter response with specific content length (-fl 23) - -follow-host-redirects - Only Follow redirects on the same host - -follow-redirects - Follow HTTP Redirects - -http-proxy string - HTTP Proxy, eg http://127.0.0.1:8080 - -http2 - HTTP2 probe - -include-chain - Show Raw HTTP Chain In Output (-json only) - -include-response - Show Raw HTTP response In Output (-json only) - -ip - Display Host IP - -json - Display output in JSON format - -l string - Input file containing list of hosts to process - -location - Display location header - -match-regex string - Match response with specific regex - -match-string string - Match response with specific string - -max-host-error int - Max error count per host before skipping remaining path/s (default 30) - -max-redirects int - Max number of redirects to follow per host (default 10) - -mc string - Match response with specific status code (-mc 200,302) - -method - Display request method - -ml string - Match response with specific content length (-ml 102) - -no-color - Disable colored output - -no-fallback - Probe both protocol (HTTPS and HTTP) - -no-fallback-scheme - Probe with input protocol scheme - -o string - File to write output to (optional) - -path string - Request path/file (example '/api') - -paths string - Command separated paths or file containing one path per line (example '/api/v1,/apiv2') - -pipeline - HTTP1.1 Pipeline probe - -ports value - Port ranges to scan (nmap syntax: eg 1,2-10,11) - -probe - Display probe status - -random-agent - Use randomly selected HTTP User-Agent header value (default true) - -rate-limit int - Maximum requests to send per second (default 150) - -request string - File containing raw request - -response-in-json - Show Raw HTTP response In Output (-json only) (deprecated) - -response-size-to-read int - Max response size to read in bytes (default - unlimited) (default 2147483647) - -response-size-to-save int - Max response size to save in bytes (default - unlimited) (default 2147483647) - -response-time - Display the response time - -resume - Resume scan using resume.cfg - -retries int - Number of retries - -silent - Silent mode - -sr - Store HTTP response to directoy (default 'output') - -srd string - Custom directory to store HTTP responses (default "output") - -stats - Enable statistic on keypress (terminal may become unresponsive till the end) - -status-code - Display HTTP response status code - -store-chain - Save chain to file (default 'output') - -tech-detect - Perform wappalyzer based technology detection - -threads int - Number of threads (default 50) - -timeout int - Timeout in seconds (default 5) - -title - Display page title - -tls-grab - Perform TLS(SSL) data grabbing - -tls-probe - Send HTTP probes on the extracted TLS domains - -unsafe - Send raw requests skipping golang normalization - -verbose - Verbose Mode - -version - Show version of httpx - -vhost - Check for VHOSTs - -vhost-input - Get a list of vhosts as input - -web-server - Display server header - -websocket - Display server using websocket - -x string - Request Methods to use, use 'all' to probe all HTTP methods -``` -
# Running httpX @@ -230,14 +177,14 @@ This will display help for the tool. Here are all the switches it supports. This will run the tool against all the hosts and subdomains in `hosts.txt` and returns URLs running HTTP webserver. -```sh -▶ cat hosts.txt | httpx +```console +cat hosts.txt | httpx __ __ __ _ __ / /_ / /_/ /_____ | |/ / / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| v1.0 +/_/ /_/\__/\__/ .___/_/|_| v1.1.1 /_/ projectdiscovery.io @@ -259,8 +206,8 @@ https://support.hackerone.com This will run the tool with the `probe` flag against all of the hosts in **hosts.txt** and return URLs with probed status. -```sh -▶ httpx -l hosts.txt -silent -probe +```console +httpx -list hosts.txt -silent -probe http://ns.hackerone.com [FAILED] https://docs.hackerone.com [SUCCESS] @@ -286,8 +233,8 @@ http://b.ns.hackerone.com [FAILED] ### Running httpx with CIDR input -```sh -▶ echo 173.0.84.0/24 | httpx -silent +```console +echo 173.0.84.0/24 | httpx -silent https://173.0.84.29 https://173.0.84.43 @@ -313,15 +260,15 @@ https://173.0.84.34 ### Running httpx with subfinder -```sh -subfinder -d hackerone.com | httpx -title -tech-detect -status-code +```console +subfinder -d hackerone.com -silent| httpx -title -tech-detect -status-code __ __ __ _ __ / /_ / /_/ /_____ | |/ / / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.0.6 + /_/ v1.1.1 projectdiscovery.io @@ -345,7 +292,7 @@ https://resources.hackerone.com [301,301,404] [Sorry, no Folders found.] - When using `json` flag, all the information (default probes) included in the JSON output. -# Thanks +# Acknowledgement httpx is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow: diff --git a/go.mod b/go.mod index b757876a..9bdc2812 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -61,9 +62,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect + github.com/projectdiscovery/goflags v0.0.7 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 7d1bb015..6da79667 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -150,6 +152,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//A github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= +github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= +github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= diff --git a/runner/options.go b/runner/options.go index a532d1a3..7b132ec1 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,13 +1,13 @@ package runner import ( - "flag" "math" "os" "regexp" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -196,82 +196,112 @@ type Options struct { func ParseOptions() *Options { options := &Options{} - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing") - flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection") - flag.IntVar(&options.Threads, "threads", 50, "Number of threads") - flag.IntVar(&options.Retries, "retries", 0, "Number of retries") - flag.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds") - flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") - flag.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs") - flag.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input") - flag.BoolVar(&options.ExtractTitle, "title", false, "Display page title") - flag.BoolVar(&options.StatusCode, "status-code", false, "Display HTTP response status code") - flag.BoolVar(&options.Location, "location", false, "Display location header") - flag.Var(&options.CustomHeaders, "H", "Custom Header to send with request") - flag.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)") - flag.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length") - flag.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directory (default 'output')") - flag.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses") - flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects") - flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host") - flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") - flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") - flag.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format") - flag.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format") - flag.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process") - flag.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods") - flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") - flag.BoolVar(&options.Silent, "silent", false, "Silent mode") - flag.BoolVar(&options.Version, "version", false, "Show version of httpx") - flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode") - flag.BoolVar(&options.NoColor, "no-color", false, "Disable colored output") - flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Display server header") - flag.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket") - flag.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP response In Output (-json only) (deprecated)") - flag.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP response In Output (-json only)") - flag.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)") - flag.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains") - flag.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains") - flag.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')") - flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')") - flag.BoolVar(&options.OutputContentType, "content-type", false, "Display content-type header") - flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match response with specific status code (-mc 200,302)") - flag.StringVar(&options.OutputMatchContentLength, "ml", "", "Match response with specific content length (-ml 102)") - flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter response with specific status code (-fc 403,401)") - flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter response with specific content length (-fl 23)") - flag.StringVar(&options.InputRawRequest, "request", "", "File containing raw request") - flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization") - flag.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request") - flag.BoolVar(&options.Debug, "debug", false, "Debug mode") - flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe") - flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe") - flag.BoolVar(&options.OutputIP, "ip", false, "Display Host IP") - flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter response with specific string") - flag.StringVar(&options.OutputMatchString, "match-string", "", "Match response with specific string") - flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex") - flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex") - flag.BoolVar(&options.OutputCName, "cname", false, "Display Host cname") - flag.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN") - flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time") - flag.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)") - flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme") - flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)") - flag.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value") - flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')") - flag.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)") - flag.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)") - flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)") - flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)") - flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Display response content with matched regex") - flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second") - flag.BoolVar(&options.Probe, "probe", false, "Display probe status") - flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") - flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s") - - flag.Parse() + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) + createGroup(flagSet, "input", "Input", + flagSet.StringVarP(&options.InputFile,"list", "l", "", "Input file containing list of hosts to process"), + flagSet.StringVar(&options.InputRawRequest, "request", "", "File containing raw request"), + ) + + createGroup(flagSet, "Probes", "Probes", + flagSet.BoolVarP(&options.StatusCode,"status-code", "sc", false, "Display Status Code"), + flagSet.BoolVarP(&options.TechDetect,"tech-detect", "tech", false, "Display wappalyzer based technology detection"), + flagSet.BoolVarP(&options.ContentLength,"content-length", "cl", false, "Display Content-Length"), + flagSet.BoolVarP(&options.OutputServerHeader,"web-server","server", false, "Display Server header"), + flagSet.BoolVarP(&options.OutputContentType,"content-type", "ct", false, "Display Content-Type header"), + flagSet.BoolVarP(&options.OutputResponseTime,"response-time", "rt", false, "Display the response time"), + flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), + flagSet.BoolVar(&options.Location, "location", false, "Display Location header"), + flagSet.BoolVar(&options.OutputMethod, "method", false, "Display Request method"), + flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket"), + flagSet.BoolVar(&options.OutputIP, "ip", false, "Display Host IP"), + flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), + flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), + flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + flagSet.BoolVarP(&options.NoFallback,"no-fallback", "nf", false, "Display both protocol (HTTPS and HTTP)"), + ) + + createGroup(flagSet, "matchers", "Matchers", + flagSet.StringVarP(&options.OutputMatchStatusCode,"match-code", "mc", "", "Match response with given status code (-mc 200,302)"), + flagSet.StringVarP(&options.OutputMatchContentLength,"match-length", "ml", "", "Match response with given content length (-ml 100,102)"), + flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms","", "Match response with given string"), + flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr","", "Match response with specific regex"), + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er","", "Display response content with matched regex"), + ) + + createGroup(flagSet, "filters", "Filters", + flagSet.StringVarP(&options.OutputFilterStatusCode,"filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"), + flagSet.StringVarP(&options.OutputFilterContentLength,"filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"), + flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), + flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe","", "Filter response with specific regex"), + ) + + createGroup(flagSet, "rate-limit", "Rate-Limit", + flagSet.IntVarP(&options.Threads, "threads","t", 50, "Number of threads"), + flagSet.IntVarP(&options.RateLimit,"rate-limit","rl", 150, "Maximum requests to send per second"), + ) + + createGroup(flagSet, "Misc", "Miscellaneous", + flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), + flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), + flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), + flagSet.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe"), + flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe"), + flagSet.BoolVar(&options.VHost, "vhost", false, "VHOST Probe"), + flagSet.VarP(&options.CustomPorts,"ports","p", "Port to scan (nmap syntax: eg 1,2-10,11)"), + flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), + flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), + ) + + createGroup(flagSet, "output", "Output", + flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output"), + flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"), + flagSet.StringVarP(&options.StoreResponseDir,"store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), + flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"), + flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Include HTTP request/response in JSON output (-json only)"), + flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"), + flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"), + + ) + + createGroup(flagSet, "configs", "Configurations", + flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToRead,"response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), + flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"), + flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"), + flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), + flagSet.VarP(&options.CustomHeaders,"header", "H", "Custom Header to send with request"), + flagSet.StringVarP(&options.HTTPProxy,"proxy", "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), + flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), + flagSet.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg"), + flagSet.BoolVarP(&options.NoColor,"no-color", "nc", false, "Disable color in output"), + flagSet.BoolVarP(&options.NoFallbackScheme,"no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), + flagSet.BoolVarP(&options.FollowRedirects,"follow-redirects", "fr", false, "Follow HTTP redirects"), + flagSet.BoolVarP(&options.FollowHostRedirects,"follow-host-redirects","fhr", false, "Follow redirects on the same host"), + flagSet.IntVarP(&options.MaxRedirects,"max-redirects","maxr", 10, "Max number of redirects to follow per host"), + flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), + flagSet.StringVar(&options.Methods, "x", "", "Request methods to use, use 'all' to probe all HTTP methods"), + flagSet.StringVar(&options.RequestBody, "body", "", "Post body to include in HTTP request"), + ) + + createGroup(flagSet, "debug", "Debug", + flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), + flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose mode"), + flagSet.BoolVar(&options.Version, "version", false, "Display version"), + flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), + flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), + ) + + createGroup(flagSet, "Optimizations", "Optimizations", + flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), + flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), + flagSet.IntVarP(&options.HostMaxErrors,"max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), + flagSet.BoolVarP(&options.ExcludeCDN,"exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), + ) + + _ = flagSet.Parse() // Read the inputs and configure the logging options.configureOutput() @@ -366,3 +396,10 @@ func (options *Options) ShouldLoadResume() bool { func (options *Options) ShouldSaveResume() bool { return true } + +func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { + flagSet.SetGroup(groupName, description) + for _, currentFlag := range flags { + currentFlag.Group(groupName) + } +}