refactor hsts implementation

README.md

@@ -196,6 +196,7 @@ CONFIGURATIONS:
    -fr, -follow-redirects        follow http redirects
    -maxr, -max-redirects int     max number of redirects to follow per host (default 10)
    -fhr, -follow-host-redirects  follow redirects on the same host
+   -rhsts, -respect-hsts         respect HSTS response headers for redirect requests
    -vhost-input                  get a list of vhosts as input
    -x string                     request methods to probe, use 'all' to probe all HTTP methods
    -body string                  post body to include in http request

common/httpx/httpx.go

@@ -66,6 +66,14 @@ func New(options *Options) (*HTTPX, error) {
 	retryablehttpOptions.Timeout = httpx.Options.Timeout
 	retryablehttpOptions.RetryMax = httpx.Options.RetryMax
 
+	handleHSTS := func(req *http.Request) {
+		if req.Response.Header.Get("Strict-Transport-Security") == "" {
+			return
+		}
+
+		req.URL.Scheme = "https"
+	}
+
 	var redirectFunc = func(_ *http.Request, _ []*http.Request) error {
 		// Tell the http client to not follow redirect
 		return http.ErrUseLastResponse
@@ -77,30 +85,15 @@ func New(options *Options) (*HTTPX, error) {
 			// add custom cookies if necessary
 			httpx.setCustomCookies(redirectedRequest)
 
-			//Add redirect policy which takes HSTS into account.
-			//Since the net/http/client doesn't take it into account
-			//it is possible to modify it here.
-			//If during redirect the scheme switches from HTTPS to HTTP
-			//but the Strict-Transport-Security header is present the request
-			//would go to the specified location. This could mean that it is not
-			//followed the same way as a browser. There exist some cases in the wild.
-			if httpx.Options.RespectHSTS {
-				location := redirectedRequest.Response.Header.Get("Location")
-				hsts := redirectedRequest.Response.Header.Get("Strict-Transport-Security")
-				url, err := redirectedRequest.URL.Parse(location)
-				if err != nil {
-				} else {
-					if url.Scheme == "http" && hsts != "" {
-						url.Scheme = "https"
-					}
-				}
-				redirectedRequest.URL = url
+			if options.RespectHSTS {
+				handleHSTS(redirectedRequest)
 			}
 
 			if len(previousRequests) >= options.MaxRedirects {
 				// https://github.com/golang/go/issues/10069
 				return http.ErrUseLastResponse
 			}
+
 			return nil
 		}
 	}

common/httpx/response.go

@@ -69,13 +69,9 @@ func (r *Response) GetChainStatusCodes() []int {
 // GetChain dump the whole redirect chain as string
 func (r *Response) GetChain() string {
 	var respchain strings.Builder
-	for counter, chainItem := range r.Chain {
-		if counter != 0 {
-			respchain.Write(chainItem.Request)
-		}
-		if counter < len(r.Chain)-1 {
-			respchain.Write(chainItem.Response)
-		}
+	for _, chainItem := range r.Chain {
+		respchain.Write(chainItem.Request)
+		respchain.Write(chainItem.Response)
 	}
 	return respchain.String()
 }

runner/options.go

@@ -85,8 +85,6 @@ type ScanOptions struct {
 	Hashes                    string
 	Screenshot                bool
 	UseInstalledChrome        bool
-	NoScreenshotBytes         bool
-	NoHeadlessBody            bool
 	DisableStdini             bool
 }
 
@@ -135,8 +133,6 @@ func (s *ScanOptions) Clone() *ScanOptions {
 		OutputWordsCount:          s.OutputWordsCount,
 		Hashes:                    s.Hashes,
 		Screenshot:                s.Screenshot,
-		NoScreenshotBytes:         s.NoScreenshotBytes,
-		NoHeadlessBody:            s.NoHeadlessBody,
 		UseInstalledChrome:        s.UseInstalledChrome,
 	}
 }
@@ -280,8 +276,6 @@ type Options struct {
 	UseInstalledChrome bool
 	TlsImpersonate     bool
 	DisableStdin       bool
-	NoScreenshotBytes  bool
-	NoHeadlessBody     bool
 }
 
 // ParseOptions parses the command line options for application
@@ -325,8 +319,6 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("headless", "Headless",
 		flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"),
 		flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"),
-		flagSet.BoolVarP(&options.NoScreenshotBytes, "exclude-screenshot-bytes", "esb", false, "enable excluding screenshot bytes from json output"),
-		flagSet.BoolVarP(&options.NoHeadlessBody, "exclude-headless-body", "ehb", false, "enable excluding headless header from json output"),
 	)
 
 	flagSet.CreateGroup("matchers", "Matchers",

runner/runner.go

@@ -245,8 +245,6 @@ func New(options *Options) (*Runner, error) {
 		runner.browser = browser
 	}
 	scanopts.Screenshot = options.Screenshot
-	scanopts.NoScreenshotBytes = options.NoScreenshotBytes
-	scanopts.NoHeadlessBody = options.NoHeadlessBody
 	scanopts.UseInstalledChrome = options.UseInstalledChrome
 
 	if options.OutputExtractRegexs != nil {
@@ -1871,12 +1869,6 @@ retry:
 				gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err)
 			}
 		}
-		if scanopts.NoScreenshotBytes {
-			screenshotBytes = []byte{}
-		}
-		if scanopts.NoHeadlessBody {
-			headlessBody = ""
-		}
 	}
 
 	result := Result{