Skip to content

Commit

Permalink
Merge pull request #160 from buger/raw-http-client
Browse files Browse the repository at this point in the history
Send unmodified http request
  • Loading branch information
buger committed Jul 6, 2015
2 parents 43fd14b + 4f114f0 commit 9ddb718
Show file tree
Hide file tree
Showing 29 changed files with 1,037 additions and 456 deletions.
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ FROM google/golang

RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean

WORKDIR /gopath/src/gor
WORKDIR /gopath/src/github.com/buger/gor/

ADD . /gopath/src/gor
ADD . /gopath/src/github.com/buger/gor/

RUN go get
20 changes: 11 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,31 +1,33 @@
SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go
SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go http_modifier.go

SOURCE_PATH = /gopath/src/github.com/buger/gor/

release: release-x86 release-x64

release-x64:
docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor

release-x86:
docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor

dbuild:
docker build -t gor .

dtest:
docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s

dfmt:
docker run -v `pwd`:/gopath/src/gor -t -i gor go fmt
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt

dvet:
docker run -v `pwd`:/gopath/src/gor -t -i gor go vet
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet

dbench:
docker run -v `pwd`:/gopath/src/gor -t -i gor go test -v -run NOT_EXISTING -bench HTTP
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test -v -run NOT_EXISTING -bench HTTP

# Used mainly for debugging, because docker container do not have access to parent machine ports
drun:
docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --verbose

dbash:
docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash
37 changes: 37 additions & 0 deletions byteutils/byteutils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package byteutils

func Cut(a []byte, from, to int) []byte {
copy(a[from:], a[to:])
a = a[:len(a)-to+from]

return a
}

func Insert(a []byte, i int, b []byte) []byte {
a = append(a, make([]byte, len(b))...)
copy(a[i+len(b):], a[i:])
copy(a[i:i+len(b)], b)

return a
}

// Unlike bytes.Replace it allows you to specify range
func Replace(a []byte, from, to int, new []byte) []byte {
lenDiff := len(new) - (to - from)

if lenDiff > 0 {
// Extend if new segment bigger
a = append(a, make([]byte, lenDiff)...)
copy(a[to+lenDiff:], a[to:])
copy(a[from:from+len(new)], new)

return a
} else if lenDiff < 0 {
copy(a[from:], new)
copy(a[from+len(new):],a[to:])
return a[:len(a) + lenDiff]
} else { // same size
copy(a[from:], new)
return a
}
}
32 changes: 32 additions & 0 deletions byteutils/byteutils_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package byteutils

import (
"testing"
"bytes"
)

func TestCut(t *testing.T) {
if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
t.Error("Should properly cut")
}
}

func TestInsert(t *testing.T) {
if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
t.Error("Should insert into middle of slice")
}
}

func TestReplace(t *testing.T) {
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
t.Error("Should replace when same length")
}

if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
t.Error("Should replace when replacement length bigger")
}

if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
t.Error("Should replace when replacement length bigger")
}
}
88 changes: 44 additions & 44 deletions elasticsearch.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"encoding/json"
"github.com/buger/elastigo/api"
"github.com/buger/elastigo/core"
"github.com/buger/gor/proto"
"log"
"net/http"
"regexp"
"time"
)
Expand All @@ -26,27 +26,27 @@ type ESPlugin struct {
}

type ESRequestResponse struct {
ReqUrl string `json:"Req_URL"`
ReqMethod string `json:"Req_Method"`
ReqUserAgent string `json:"Req_User-Agent"`
ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"`
ReqAccept string `json:"Req_Accept,omitempty"`
ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"`
ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"`
ReqConnection string `json:"Req_Connection,omitempty"`
ReqCookies []*http.Cookie `json:"Req_Cookies,omitempty"`
RespStatus string `json:"Resp_Status"`
RespStatusCode int `json:"Resp_Status-Code"`
RespProto string `json:"Resp_Proto,omitempty"`
RespContentLength int64 `json:"Resp_Content-Length,omitempty"`
RespContentType string `json:"Resp_Content-Type,omitempty"`
RespTransferEncoding []string `json:"Resp_Transfer-Encoding,omitempty"`
RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"`
RespExpires string `json:"Resp_Expires,omitempty"`
RespCacheControl string `json:"Resp_Cache-Control,omitempty"`
RespVary string `json:"Resp_Vary,omitempty"`
RespSetCookie string `json:"Resp_Set-Cookie,omitempty"`
Rtt int64 `json:"RTT"`
ReqUrl []byte `json:"Req_URL"`
ReqMethod []byte `json:"Req_Method"`
ReqUserAgent []byte `json:"Req_User-Agent"`
ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"`
ReqAccept []byte `json:"Req_Accept,omitempty"`
ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"`
ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"`
ReqConnection []byte `json:"Req_Connection,omitempty"`
ReqCookies []byte `json:"Req_Cookies,omitempty"`
RespStatus []byte `json:"Resp_Status"`
RespStatusCode []byte `json:"Resp_Status-Code"`
RespProto []byte `json:"Resp_Proto,omitempty"`
RespContentLength []byte `json:"Resp_Content-Length,omitempty"`
RespContentType []byte `json:"Resp_Content-Type,omitempty"`
RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"`
RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"`
RespExpires []byte `json:"Resp_Expires,omitempty"`
RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"`
RespVary []byte `json:"Resp_Vary,omitempty"`
RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"`
Rtt int64 `json:"RTT"`
Timestamp time.Time
}

Expand Down Expand Up @@ -111,35 +111,35 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {
return int64(fl)
}

func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start, stop time.Time) {
if resp == nil {
func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
if len(resp) == 0 {
// nil http response - skipped elasticsearch export for this request
return
}
t := time.Now()
rtt := p.RttDurationToMs(stop.Sub(start))

esResp := ESRequestResponse{
ReqUrl: req.URL.String(),
ReqMethod: req.Method,
ReqUserAgent: req.UserAgent(),
ReqAcceptLanguage: req.Header.Get("Accept-Language"),
ReqAccept: req.Header.Get("Accept"),
ReqAcceptEncoding: req.Header.Get("Accept-Encoding"),
ReqIfModifiedSince: req.Header.Get("If-Modified-Since"),
ReqConnection: req.Header.Get("Connection"),
ReqCookies: req.Cookies(),
RespStatus: resp.Status,
RespStatusCode: resp.StatusCode,
RespProto: resp.Proto,
RespContentLength: resp.ContentLength,
RespContentType: resp.Header.Get("Content-Type"),
RespTransferEncoding: resp.TransferEncoding,
RespContentEncoding: resp.Header.Get("Content-Encoding"),
RespExpires: resp.Header.Get("Expires"),
RespCacheControl: resp.Header.Get("Cache-Control"),
RespVary: resp.Header.Get("Vary"),
RespSetCookie: resp.Header.Get("Set-Cookie"),
ReqUrl: proto.Path(req),
ReqMethod: proto.Method(req),
ReqUserAgent: proto.GetHeader(req, "User-Agent"),
ReqAcceptLanguage: proto.GetHeader(req, "Accept-Language"),
ReqAccept: proto.GetHeader(req, "Accept"),
ReqAcceptEncoding: proto.GetHeader(req, "Accept-Encoding"),
ReqIfModifiedSince: proto.GetHeader(req, "If-Modified-Since"),
ReqConnection: proto.GetHeader(req, "Connection"),
ReqCookies: proto.GetHeader(req, "Cookie"),
RespStatus: proto.Status(resp),
RespStatusCode: proto.Status(resp),
RespProto: proto.Method(resp),
RespContentLength: proto.GetHeader(resp, "Content-Length"),
RespContentType: proto.GetHeader(resp, "Content-Type"),
RespTransferEncoding: proto.GetHeader(resp, "Transfer-Encoding"),
RespContentEncoding: proto.GetHeader(resp, "Content-Encoding"),
RespExpires: proto.GetHeader(resp, "Expires"),
RespCacheControl: proto.GetHeader(resp, "Cache-Control"),
RespVary: proto.GetHeader(resp, "Vary"),
RespSetCookie: proto.GetHeader(resp, "Set-Cookie"),
Rtt: rtt,
Timestamp: t,
}
Expand Down
16 changes: 13 additions & 3 deletions emitter.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,25 @@ func Start(stop chan int) {
func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
buf := make([]byte, 5*1024*1024)
wIndex := 0
modifier := NewHTTPModifier(&Settings.modifierConfig)

for {
nr, er := src.Read(buf)
if nr > 0 && len(buf) > nr {
Debug("Sending", src, ": ", string(buf[0:nr]))
payload := buf[0:nr]

if modifier != nil {
payload = modifier.Rewrite(payload)

// If modifier tells to skip request
if len(payload) == 0 {
continue
}
}

if Settings.splitOutput {
// Simple round robin
writers[wIndex].Write(buf[0:nr])
writers[wIndex].Write(payload)

wIndex++

Expand All @@ -40,7 +50,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
}
} else {
for _, dst := range writers {
dst.Write(buf[0:nr])
dst.Write(payload)
}
}

Expand Down
Loading

0 comments on commit 9ddb718

Please sign in to comment.