Merge branch 'DamianZaremba-url-rewrite2'

Makefile

@@ -1,4 +1,4 @@
-SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go
+SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go
 
 all: build-x86 build-x64
 

README.md

@@ -132,6 +132,13 @@ gor --input-raw :80 --output-http "http://user:pass@staging .com"
 
 Note: This will overwrite any Authorization headers in the original request.
 
+#### Rewrite the target urls based on a mapping
+```
+# rewrite url to match the following
+gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do
+```
+
+
 ## Stats 
 ### ElasticSearch 
 For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details.
@@ -140,7 +147,6 @@ For deep response analyze based on url, cookie, user-agent and etc. you can expo
 gor --input-tcp :80 --output-http "http://staging.com" --output-http-elasticsearch "es_host:api_port/index_name"
 ```
 
-
 ## Additional help
 
 Feel free to ask question directly by email or by creating github issue.
@@ -203,6 +209,9 @@ https://github.com/buger/gor/releases
   -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second.
     
   -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.
+
+  -output-http-rewrite-url=[]: Rewrites the url in the request based on a mapping
+    gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do
 ```
 
 ## Building from source

output_http.go

@@ -42,9 +42,10 @@ type HTTPOutput struct {
 	buf     chan []byte
 	needWorker chan int
 
-	urlRegexp         HTTPUrlRegexp
-	headerFilters     HTTPHeaderFilters
-	headerHashFilters HTTPHeaderHashFilters
+	urlRegexp             HTTPUrlRegexp
+	headerFilters         HTTPHeaderFilters
+	headerHashFilters     HTTPHeaderHashFilters
+    outputHTTPUrlRewrite  UrlRewriteMap
 
 	headers HTTPHeaders
 	methods HTTPMethods
@@ -54,7 +55,8 @@ type HTTPOutput struct {
 	bufStats *GorStat
 }
 
-func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string) io.Writer {
+func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap) io.Writer {
+
 	o := new(HTTPOutput)
 
 	optionsArr := strings.Split(options, "|")
@@ -71,6 +73,7 @@ func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, url
 	o.urlRegexp = urlRegexp
 	o.headerFilters = headerFilters
 	o.headerHashFilters = headerHashFilters
+	o.outputHTTPUrlRewrite = outputHTTPUrlRewrite
 
 	o.buf = make(chan []byte, 100)
 	if Settings.outputHTTPStats {
@@ -174,6 +177,9 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) {
 		return
 	}
 
+    // Rewrite the path as necessary
+    request.URL.Path = o.outputHTTPUrlRewrite.Rewrite(request.URL.Path)
+
 	// Change HOST of original request
 	URL := o.address + request.URL.Path + "?" + request.URL.RawQuery
 

output_http_test.go

@@ -63,7 +63,7 @@ func TestHTTPOutput(t *testing.T) {
 		wg.Done()
 	})
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "")
+	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}
@@ -96,7 +96,7 @@ func BenchmarkHTTPOutput(b *testing.B) {
 		wg.Done()
 	})
 
-	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "")
+	output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "",UrlRewriteMap{})
 
 	Plugins.Inputs = []io.Reader{input}
 	Plugins.Outputs = []io.Writer{output}

plugins.go

@@ -41,6 +41,6 @@ func InitPlugins() {
 	}
 
 	for _, options := range Settings.outputHTTP {
-		Plugins.Outputs = append(Plugins.Outputs, NewHTTPOutput(options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch))
+		Plugins.Outputs = append(Plugins.Outputs, NewHTTPOutput(options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite))
 	}
 }

settings.go

@@ -33,6 +33,7 @@ type AppSettings struct {
 	outputHTTPHeaders           HTTPHeaders
 	outputHTTPMethods           HTTPMethods
 	outputHTTPUrlRegexp         HTTPUrlRegexp
+	outputHTTPUrlRewrite        UrlRewriteMap
 	outputHTTPHeaderFilters     HTTPHeaderFilters
 	outputHTTPHeaderHashFilters HTTPHeaderHashFilters
 	outputHTTPElasticSearch     string
@@ -78,6 +79,7 @@ func init() {
 	flag.BoolVar(&Settings.outputHTTPStats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.")
 
 	flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'")
+	flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do")
 }
 
 func Debug(args ...interface{}) {

settings_url_map.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+type urlRewrite struct {
+	src    string
+	target string
+}
+
+type UrlRewriteMap []urlRewrite
+
+func (r *UrlRewriteMap) String() string {
+	return fmt.Sprint(*r)
+}
+
+func (r *UrlRewriteMap) Set(value string) error {
+	valArr := strings.SplitN(value, ":", 2)
+	if len(valArr) < 2 {
+		return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
+	}
+	*r = append(*r, urlRewrite{src: valArr[0], target: valArr[1]})
+	return nil
+}
+
+func (r *UrlRewriteMap) Rewrite(path string) string {
+        for _, f := range *r {
+                if f.src == path {
+                        return f.target
+                }
+        }
+        return path
+}

settings_url_map_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestUrlRewriteMap(t *testing.T) {
+    var url string;
+
+	rewrites := UrlRewriteMap{}
+
+	err := rewrites.Set("/abc:/123")
+	if err != nil {
+		t.Error("Should not error on /abc:/123")
+	}
+
+	url = "/abc"
+
+    if(rewrites.Rewrite(url) == url) {
+        t.Error("Request url should have been rewritten, wasn't")
+    }
+
+    url = "/wibble"
+    if(rewrites.Rewrite(url) != url) {
+        t.Error("Request url should not have been rewritten, was")
+    }
+}