Skip to content

Commit

Permalink
Merge pull request #68 from caioluders/master
Browse files Browse the repository at this point in the history
New feature: ignore same endpoints with different parameters
  • Loading branch information
lc authored Mar 2, 2022
2 parents 9555c18 + 802222d commit 2789498
Show file tree
Hide file tree
Showing 6 changed files with 31 additions and 5 deletions.
1 change: 1 addition & 0 deletions .gau.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ threads = 2
verbose = false
retries = 15
subdomains = false
parameters = false
providers = ["gau","commoncrawl","otx","urlscan"]
blacklist = ["ttf","woff","svg","png","jpg"]
json = false
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ $ gau -h
|`--fc`| list of status codes to filter | gau --fc 404,302 |
|`--from`| fetch urls from date (format: YYYYMM) | gau --from 202101 |
|`--ft`| list of mime-types to filter | gau --ft text/plain|
|`--fp`| remove different parameters of the same endpoint | gau --fp|
|`--json`| output as json | gau --json |
|`--mc`| list of status codes to match | gau --mc 200,500 |
|`--mt`| list of mime-types to match |gau --mt text/html,application/json|
Expand Down
4 changes: 2 additions & 2 deletions cmd/gau/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ func main() {
if config.JSON {
go func() {
defer writeWg.Done()
output.WriteURLsJSON(out, results, config.Blacklist)
output.WriteURLsJSON(out, results, config.Blacklist, config.RemoveParameters)
}()
} else {
go func() {
defer writeWg.Done()
if err = output.WriteURLs(out, results, config.Blacklist); err != nil {
if err = output.WriteURLs(out, results, config.Blacklist, config.RemoveParameters); err != nil {
log.Fatalf("error writing results: %v\n", err)
}
}()
Expand Down
18 changes: 16 additions & 2 deletions pkg/output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ type JSONResult struct {
Url string `json:"url"`
}

func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}) error {
func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) error {
lastURL := make(map[string]struct{})
for result := range results {
buf := bytebufferpool.Get()
if len(blacklistMap) != 0 {
Expand All @@ -30,6 +31,19 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]
}
}
}
if RemoveParameters {
u, err := url.Parse(result)
if err != nil {
continue
}
if _, ok := lastURL[u.Host+u.Path]; ok {
continue
} else {
lastURL[u.Host+u.Path] = struct{}{} ;
}

}

buf.B = append(buf.B, []byte(result)...)
buf.B = append(buf.B, "\n"...)
_, err := writer.Write(buf.B)
Expand All @@ -41,7 +55,7 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap map[string]
return nil
}

func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}) {
func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap map[string]struct{}, RemoveParameters bool) {
var jr JSONResult
enc := jsoniter.NewEncoder(writer)
for result := range results {
Expand Down
3 changes: 2 additions & 1 deletion pkg/providers/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"github.com/valyala/fasthttp"
)

const Version = `2.0.8`
const Version = `2.0.9`

// Provider is a generic interface for all archive fetchers
type Provider interface {
Expand All @@ -23,6 +23,7 @@ type Config struct {
Verbose bool
MaxRetries uint
IncludeSubdomains bool
RemoveParameters bool
Client *fasthttp.Client
Providers []string
Blacklist map[string]struct{}
Expand Down
9 changes: 9 additions & 0 deletions runner/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type Config struct {
Verbose bool `mapstructure:"verbose"`
MaxRetries uint `mapstructure:"retries"`
IncludeSubdomains bool `mapstructure:"subdomains"`
RemoveParameters bool `mapstructure:"parameters"`
Providers []string `mapstructure:"providers"`
Blacklist []string `mapstructure:"blacklist"`
JSON bool `mapstructure:"json"`
Expand Down Expand Up @@ -60,6 +61,7 @@ func (c *Config) ProviderConfig() (*providers.Config, error) {
Verbose: c.Verbose,
MaxRetries: c.MaxRetries,
IncludeSubdomains: c.IncludeSubdomains,
RemoveParameters: c.RemoveParameters,
Client: &fasthttp.Client{
TLSConfig: &tls.Config{
InsecureSkipVerify: true,
Expand Down Expand Up @@ -98,6 +100,7 @@ func New() *Options {
pflag.StringSlice("blacklist", []string{}, "list of extensions to skip")
pflag.StringSlice("providers", []string{}, "list of providers to use (wayback,commoncrawl,otx,urlscan)")
pflag.Bool("subs", false, "include subdomains of target domain")
pflag.Bool("fp", false, "remove different parameters of the same endpoint")
pflag.Bool("verbose", false, "show verbose output")
pflag.Bool("json", false, "output as json")

Expand Down Expand Up @@ -160,6 +163,7 @@ func (o *Options) DefaultConfig() *Config {
Verbose: false,
MaxRetries: 5,
IncludeSubdomains: false,
RemoveParameters: false,
Providers: []string{"wayback", "commoncrawl", "otx", "urlscan"},
Blacklist: []string{},
JSON: false,
Expand All @@ -182,6 +186,7 @@ func (o *Options) getFlagValues(c *Config) {
threads := o.viper.GetUint("threads")
blacklist := o.viper.GetStringSlice("blacklist")
subs := o.viper.GetBool("subs")
fp := o.viper.GetBool("fp")

if version {
fmt.Printf("gau version: %s\n", providers.Version)
Expand Down Expand Up @@ -218,6 +223,10 @@ func (o *Options) getFlagValues(c *Config) {
c.IncludeSubdomains = subs
}

if fp {
c.RemoveParameters = fp
}

if json {
c.JSON = true
}
Expand Down

0 comments on commit 2789498

Please sign in to comment.