New feature: ignore same endpoints with different parameters

.gau.toml

@@ -2,6 +2,7 @@ threads = 2
 verbose = false
 retries = 15
 subdomains = false
+parameters = false
 providers = ["gau","commoncrawl","otx","urlscan"]
 blacklist = ["ttf","woff","svg","png","jpg"]
 json = false

README.md

@@ -31,6 +31,7 @@ $ gau -h
 |`--fc`| list of status codes to filter | gau --fc 404,302 |
 |`--from`| fetch urls from date (format: YYYYMM) | gau --from 202101 |
 |`--ft`| list of mime-types to filter | gau --ft text/plain|
+|`--fp`| remove different parameters of the same endpoint | gau --fp|
 |`--json`| output as json | gau --json |
 |`--mc`| list of status codes to match | gau --mc 200,500 |
 |`--mt`| list of mime-types to match |gau --mt text/html,application/json|

cmd/gau/main.go

@@ -56,12 +56,12 @@ func main() {
 	if config.JSON {
 		go func() {
 			defer writeWg.Done()
-			output.WriteURLsJSON(out, results, config.Blacklist)
+			output.WriteURLsJSON(out, results, config.Blacklist, config.RemoveParameters)
 		}()
 	} else {
 		go func() {
 			defer writeWg.Done()
-			if err = output.WriteURLs(out, results, config.Blacklist); err != nil {
+			if err = output.WriteURLs(out, results, config.Blacklist, config.RemoveParameters); err != nil {
 				log.Fatalf("error writing results: %v\n", err)
 			}
 		}()

pkg/output/output.go

@@ -13,7 +13,8 @@ type JSONResult struct {
 	Url string `json:"url"`
 }
 
-func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}) error {
+func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) error {
+	lastURL := make(map[string]struct{})
 	for result := range results {
 		buf := bytebufferpool.Get()
 		if len(blacklistMap) != 0 {
@@ -30,6 +31,19 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]
 				}
 			}
 		}
+		if RemoveParameters {
+			u, err := url.Parse(result)
+			if err != nil {
+				continue
+			}
+			if _, ok := lastURL[u.Host+u.Path]; ok {
+				continue
+			} else {
+				lastURL[u.Host+u.Path] = struct{}{} ;
+			}
+
+		}
+
 		buf.B = append(buf.B, []byte(result)...)
 		buf.B = append(buf.B, "\n"...)
 		_, err := writer.Write(buf.B)
@@ -41,7 +55,7 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]
 	return nil
 }
 
-func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}) {
+func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) {
 	var jr JSONResult
 	enc := jsoniter.NewEncoder(writer)
 	for result := range results {

pkg/providers/providers.go

@@ -5,7 +5,7 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-const Version = `2.0.8`
+const Version = `2.0.9`
 
 // Provider is a generic interface for all archive fetchers
 type Provider interface {
@@ -23,6 +23,7 @@ type Config struct {
 	Verbose           bool
 	MaxRetries        uint
 	IncludeSubdomains bool
+	RemoveParameters  bool
 	Client            *fasthttp.Client
 	Providers         []string
 	Blacklist         map[string]struct{}

runner/flags/flags.go

@@ -29,6 +29,7 @@ type Config struct {
 	Verbose           bool              `mapstructure:"verbose"`
 	MaxRetries        uint              `mapstructure:"retries"`
 	IncludeSubdomains bool              `mapstructure:"subdomains"`
+	RemoveParameters  bool              `mapstructure:"parameters"`
 	Providers         []string          `mapstructure:"providers"`
 	Blacklist         []string          `mapstructure:"blacklist"`
 	JSON              bool              `mapstructure:"json"`
@@ -60,6 +61,7 @@ func (c *Config) ProviderConfig() (*providers.Config, error) {
 		Verbose:           c.Verbose,
 		MaxRetries:        c.MaxRetries,
 		IncludeSubdomains: c.IncludeSubdomains,
+		RemoveParameters:  c.RemoveParameters,
 		Client: &fasthttp.Client{
 			TLSConfig: &tls.Config{
 				InsecureSkipVerify: true,
@@ -98,6 +100,7 @@ func New() *Options {
 	pflag.StringSlice("blacklist", []string{}, "list of extensions to skip")
 	pflag.StringSlice("providers", []string{}, "list of providers to use (wayback,commoncrawl,otx,urlscan)")
 	pflag.Bool("subs", false, "include subdomains of target domain")
+	pflag.Bool("fp", false, "remove different parameters of the same endpoint")
 	pflag.Bool("verbose", false, "show verbose output")
 	pflag.Bool("json", false, "output as json")
 
@@ -160,6 +163,7 @@ func (o *Options) DefaultConfig() *Config {
 		Verbose:           false,
 		MaxRetries:        5,
 		IncludeSubdomains: false,
+		RemoveParameters:  false,
 		Providers:         []string{"wayback", "commoncrawl", "otx", "urlscan"},
 		Blacklist:         []string{},
 		JSON:              false,
@@ -182,6 +186,7 @@ func (o *Options) getFlagValues(c *Config) {
 	threads := o.viper.GetUint("threads")
 	blacklist := o.viper.GetStringSlice("blacklist")
 	subs := o.viper.GetBool("subs")
+	fp := o.viper.GetBool("fp")
 
 	if version {
 		fmt.Printf("gau version: %s\n", providers.Version)
@@ -218,6 +223,10 @@ func (o *Options) getFlagValues(c *Config) {
 		c.IncludeSubdomains = subs
 	}
 
+	if fp {
+		c.RemoveParameters = fp
+	}
+
 	if json {
 		c.JSON = true
 	}