add blacklist flag

.gitignore

@@ -16,5 +16,6 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 .DS_Store
+.idea
 dist
 gau

go.mod

@@ -2,4 +2,7 @@ module github.com/lc/gau
 
 go 1.14
 
-require github.com/json-iterator/go v1.1.10
+require (
+	github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89
+	github.com/json-iterator/go v1.1.10
+)

go.sum

@@ -1,3 +1,5 @@
+github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ=
+github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -10,3 +12,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/net v0.0.0-20180811021610-c39426892332 h1:efGso+ep0DjyCBJPjvoz0HI6UldX4Md2F1rZFe1ir0E=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

main.go

@@ -57,12 +57,12 @@ func run(config *providers.Config, domains []string) {
 	writewg.Add(1)
 	if config.JSON {
 		go func() {
-			output.WriteURLsJSON(results, out)
+			output.WriteURLsJSON(results, out, config.Blacklist)
 			writewg.Done()
 		}()
 	} else {
 		go func() {
-			output.WriteURLs(results, out)
+			output.WriteURLs(results, out, config.Blacklist)
 			writewg.Done()
 		}()
 	}
@@ -102,6 +102,7 @@ func main() {
 	output := flag.String("o", "", "filename to write results to")
 	jsonOut := flag.Bool("json", false, "write output as json")
 	randomAgent := flag.Bool("random-agent", false, "use random user-agent")
+	blacklist := flag.String("b","","extensions to skip, ex: ttf,woff,svg,png,jpg")
 	flag.Parse()
 
 	if *version {
@@ -132,13 +133,19 @@ func main() {
 		}
 	}
 
+	extensions := strings.Split(*blacklist,",")
+	extMap := make(map[string]struct{})
+	for _, ext := range extensions {
+		extMap[ext] = struct{}{}
+	}
 	config := providers.Config{
 		Verbose:           *verbose,
 		RandomAgent:       *randomAgent,
 		MaxRetries:        *maxRetries,
 		IncludeSubdomains: *includeSubs,
 		Output:            *output,
 		JSON:              *jsonOut,
+		Blacklist: 			extMap,
 		Client: &http.Client{
 			Timeout:   time.Second * 15,
 			Transport: tr,

output/output.go

@@ -3,6 +3,8 @@ package output
 import (
 	"bufio"
 	"io"
+	"net/url"
+	"path"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
@@ -11,11 +13,24 @@ import (
 type JSONResult struct {
 	Url string `json:"url"`
 }
-
-func WriteURLs(results <-chan string, writer io.Writer) error {
+func WriteURLs(results <-chan string, writer io.Writer, blacklistMap map[string]struct{}) error {
 	wr := bufio.NewWriter(writer)
 	str := &strings.Builder{}
 	for result := range results {
+		if len(blacklistMap) != 0 {
+			u, err := url.Parse(result)
+			if err != nil {
+				continue
+			}
+			base := strings.Split(path.Base(u.Path),".")
+			ext := base[len(base)-1]
+			if ext != "" {
+				_, ok := blacklistMap[ext]
+				if ok {
+					continue
+				}
+			}
+		}
 		str.WriteString(result)
 		str.WriteRune('\n')
 		_, err := wr.WriteString(str.String())
@@ -27,10 +42,24 @@ func WriteURLs(results <-chan string, writer io.Writer) error {
 	}
 	return wr.Flush()
 }
-func WriteURLsJSON(results <-chan string, writer io.Writer) {
+func WriteURLsJSON(results <-chan string, writer io.Writer, blacklistMap map[string]struct{}) {
 	var jr JSONResult
 	enc := jsoniter.NewEncoder(writer)
 	for result := range results {
+		if len(blacklistMap) != 0 {
+			u, err := url.Parse(result)
+			if err != nil {
+				continue
+			}
+			base := strings.Split(path.Base(u.Path),".")
+			ext := base[len(base)-1]
+			if ext != "" {
+				_, ok := blacklistMap[ext]
+				if ok {
+					continue
+				}
+			}
+		}
 		jr.Url = result
 		enc.Encode(jr)
 	}

providers/provider.go

@@ -24,6 +24,7 @@ type Config struct {
 	IncludeSubdomains bool
 	Client            *http.Client
 	Providers         []string
+	Blacklist 		  map[string]struct{}
 	Output            string
 	JSON              bool
 }