From c80198527fa6383ac51e32f499314c8841e8e3ec Mon Sep 17 00:00:00 2001 From: DashJay Date: Tue, 30 Jun 2020 10:08:48 +0800 Subject: [PATCH] add blend mode to --- CHANGELOG.CN.txt | 2 +- CHANGELOG.txt | 2 +- README.CN.md | 5 +-- README.md | 7 +++-- cmd/main.go | 22 +++++++++++-- core/blend.go | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ core/common.go | 66 +++++++++++++++++++++++++++++++++++++++ core/copy.go | 1 + core/proxy.go | 75 +++++++++++++------------------------------- 9 files changed, 199 insertions(+), 62 deletions(-) create mode 100644 core/blend.go create mode 100644 core/common.go diff --git a/CHANGELOG.CN.txt b/CHANGELOG.CN.txt index 870759a..0d2bf9d 100644 --- a/CHANGELOG.CN.txt +++ b/CHANGELOG.CN.txt @@ -1 +1 @@ -1. 添加中间件模式 \ No newline at end of file +1. 添加混合模式 \ No newline at end of file diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 56b1999..3057357 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1 +1 @@ -1. add middleware mode for superlcx \ No newline at end of file +1. add blendmode. \ No newline at end of file diff --git a/README.CN.md b/README.CN.md index c63a8c1..420d3a3 100644 --- a/README.CN.md +++ b/README.CN.md @@ -27,8 +27,9 @@ Usage of ./superlcx: - 优点: 拷贝模式直接工作在TCP层, 他并不关心转发的是什么. 在 `io.Copy`的帮助下，运行只需要非常少的内存。 - 缺点: 拷贝模式并不知道应用层的内容. 所有他可以做的仅仅是全部dump出来，做其他操作。 -- 融合模式（开发中） - - 期待优点：占用非常小的内存，提供丰富的接口例如`proxy_pass`等 +- 融合模式 + - 优点：混合模式虽然工作在TCP层，但是以牺牲解析HTTP为代价，为请求和返回值的分析提供了中间间模式运行的可能性。 + - 缺点：因为IO量可能比较大，可能会引起内存抖动，大量gc，在高并发下不适合小内存的机器。 ### -M 中间件 当工作在代理模式下，可以调用中间件来对过程中的流量进行分析。例如，系统内置stdout中间件（示例中间件），可以通过`-M stdout`来使用。 diff --git a/README.md b/README.md index 26b1bdd..72ea63e 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,15 @@ Usage of ./superlcx: ### mode - proxy - advantages: the proxy mode work with http proxy package which can offer(expose) more api like modifyResponse,Transport,director, etc. - - disadvantages: the proxy mode will cause memory mem jitter, not suitable for limited memory machine if you need high performance + - disadvantages: the proxy mode will cause memory jitter, not suitable for limited memory machine if you need high performance - copy - advantages: the copy mode directly work on tcp layer, it doesn't care about what would be transfer. With the help of `io.Copy`, it needs less RAM. - disadvantages: the copy mode know nothing about application layer. all things it can do is dumping them all out. -- hybrid(ing) - - (expect)advantages: allocate low memory and expose more API like proxy_pass...etc +- blend + - advantages: allocate low memory comparing with the proxy mode. it can run with the middleware interface. + - disadvantages: the blend mode still need more memory(less than proxy), and could lead to memory jitter. ### -M middleware When working in the proxy mode, middleware can be invoked to analyze the traffic in the process. For example, the built-in stdout middleware (sample middleware) can be used via '-M stdout'. diff --git a/cmd/main.go b/cmd/main.go index cd2ff58..f553322 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -25,6 +25,7 @@ var ( hostPort string mode string middleware string + logCfg string ) func init() { @@ -37,9 +38,10 @@ func init() { }() flag.BoolVar(&showVersion, "v", false, "show version and about then exit.") flag.IntVar(&listenPort, "l", 8080, "listen port") - flag.StringVar(&hostPort, "host", "0.0.0.0:8081", "target host:port") - flag.StringVar(&mode, "m", "proxy", "run mode") + flag.StringVar(&hostPort, "host", "0.0.0.0:8081", "target host:port.") + flag.StringVar(&mode, "m", "proxy", "run mode .") flag.StringVar(&middleware, "M", "", "middleware, comma separated if more than one, eg: --M stdout,dumps") + flag.StringVar(&logCfg, "log", "t", "l -> line of code, d -> date, t -> time, order doesn't matter") flag.Parse() if listenPort < 1 || listenPort > 65535 { panic("[x] Listen Port Invalid") @@ -61,6 +63,19 @@ Superlcx [%s], a tool kit for port transfer with middlewares! `, version) os.Exit(0) } + logC := strings.Split(logCfg, "") + logFlag := log.Ltime + for _, c := range logC { + switch c { + case "d": + logFlag |= log.Ldate + case "l": + logFlag |= log.Lshortfile + default: + continue + } + } + log.SetFlags(logFlag) // Buried point for debug go func() { http.ListenAndServe(":8999", nil) @@ -80,6 +95,9 @@ Superlcx [%s], a tool kit for port transfer with middlewares! case "copy": c := core.NewSapCopy(lis, hostPort) c.Serve() + case "blend": + c := core.NewSapBlend(lis, hostPort, middleware) + c.Serve() default: flag.PrintDefaults() os.Exit(-1) diff --git a/core/blend.go b/core/blend.go new file mode 100644 index 0000000..a85655c --- /dev/null +++ b/core/blend.go @@ -0,0 +1,81 @@ +package core + +import ( + "bufio" + "context" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "time" +) + +const maxReadTime = 10 * time.Second + +type SapBlend struct { + lis net.Listener + defaultUrl *url.URL + *middleware +} + +func NewSapBlend(lis net.Listener, target string, middleware string) *SapBlend { + defaultUrl, err := url.Parse(fmt.Sprintf("http://%s/", target)) + if err != nil { + panic(fmt.Sprintf("default url [%s] parse error, detail:[%s]", defaultUrl, err)) + } + log.Printf("parse default url as %s", defaultUrl) + b := &SapBlend{lis: lis, defaultUrl: defaultUrl, middleware: newMiddleware(middleware)} + + return b +} + +func (s *SapBlend) Serve() { + log.Printf("superlcx work in blend mode!") + tr := http.DefaultTransport + for { + conn, err := s.lis.Accept() + if err != nil { + log.Printf("[x] listener accept error, detail:[%s]", err) + } + go func() { + buf := bufio.NewReader(conn) + conn.SetDeadline(time.Now().Add(maxReadTime)) + defer conn.Close() + wait := 100 * time.Microsecond + for { + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.ReadRequest(buf) + if err != nil { + if err == io.EOF { + time.Sleep(wait) + wait *= 2 + continue + } else { + // 这里能识别已经关闭的链接 + log.Printf("[-] connection over, detail:[%s]", err) + return + } + } + newReq := req.Clone(ctx) + organizeUrl(newReq, s.defaultUrl) + + for _, reqH := range s.reqHandlers { + reqH(newReq) + } + resp, err := tr.RoundTrip(newReq) + if err != nil { + log.Printf("[x] default transport req error, detail:[%s]", err) + continue + } + for _, respH := range s.respHandlers { + respH(resp) + } + + resp.Write(conn) + cancel() + } + }() + } +} diff --git a/core/common.go b/core/common.go new file mode 100644 index 0000000..9c3b009 --- /dev/null +++ b/core/common.go @@ -0,0 +1,66 @@ +package core + +import ( + "net/http" + "net/url" + "strings" + + "superlcx/middlewares/stdout" +) + +func organizeUrl(req *http.Request, target *url.URL) { + if _, ok := req.Header["User-Agent"]; !ok { + // explicitly disable User-Agent so it's not set to default value + req.Header.Set("User-Agent", "") + } + singleJoiningSlash := func(a, b string) string { + aslash := strings.HasSuffix(a, "/") + bslash := strings.HasPrefix(b, "/") + switch { + case aslash && bslash: + return a + b[1:] + case !aslash && !bslash: + return a + "/" + b + } + return a + b + } + targetQuery := target.RawQuery + req.URL.Scheme = target.Scheme + req.URL.Host = target.Host + req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) + if targetQuery == "" || req.URL.RawQuery == "" { + req.URL.RawQuery = targetQuery + req.URL.RawQuery + } else { + req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery + } +} + +type middleware struct { + reqHandlers []func(req *http.Request) + respHandlers []func(resp *http.Response) +} + +func newMiddleware(mid string) *middleware { + middle := &middleware{ + reqHandlers: []func(req *http.Request){}, + respHandlers: []func(resp *http.Response){}, + } + if mid != "" { + ms := strings.Split(mid, ",") + for _, m := range ms { + switch m { + case "stdout": + middle.RegisterMiddleware(stdout.HandleRequest, stdout.HandleResponse) + default: + reqH, respH := find(m) + middle.RegisterMiddleware(reqH, respH) + } + } + } + return middle +} + +func (m *middleware) RegisterMiddleware(reqH func(req *http.Request), respH func(resp *http.Response)) { + m.reqHandlers = append(m.reqHandlers, reqH) + m.respHandlers = append(m.respHandlers, respH) +} diff --git a/core/copy.go b/core/copy.go index 7b1bd07..f09390e 100644 --- a/core/copy.go +++ b/core/copy.go @@ -13,6 +13,7 @@ type SapCopy struct { } func (s *SapCopy) Serve() { + log.Printf("superlcx work in copy mode!") for { conn, err := s.lis.Accept() if err != nil { diff --git a/core/proxy.go b/core/proxy.go index bb84b0f..1191d18 100644 --- a/core/proxy.go +++ b/core/proxy.go @@ -13,69 +13,37 @@ import ( "superlcx/middlewares/stdout" ) -type Proxy struct { - defaultUrl *url.URL - lis net.Listener - reqHandlers []func(req *http.Request) - respHandlers []func(resp *http.Response) -} - -func (p *Proxy) RegisterMiddleware(reqH func(req *http.Request), respH func(resp *http.Response)) { - p.reqHandlers = append(p.reqHandlers, reqH) - p.respHandlers = append(p.respHandlers, respH) +type SapProxy struct { + defaultUrl *url.URL + lis net.Listener + middleware } // NewSapProxy 构建一个SapProxy -func NewSapProxy(lis net.Listener, defaultHost string, middleware string) *Proxy { +func NewSapProxy(lis net.Listener, defaultHost string, middleware string) *SapProxy { u, err := url.Parse(fmt.Sprintf("http://%s", defaultHost)) if err != nil { panic(err) } log.Printf("parse default url as %s", u) - p := &Proxy{ - defaultUrl: u, - lis: lis, - reqHandlers: make([]func(req *http.Request), 0), - respHandlers: make([]func(resp *http.Response), 0), + p := &SapProxy{ + defaultUrl: u, + lis: lis, } p.Register(middleware) return p } -func (p *Proxy) director(req *http.Request) { - for _, fn := range p.reqHandlers { +func (s *SapProxy) director(req *http.Request) { + organizeUrl(req, s.defaultUrl) + for _, fn := range s.reqHandlers { fn(req) } - if _, ok := req.Header["User-Agent"]; !ok { - // explicitly disable User-Agent so it's not set to default value - req.Header.Set("User-Agent", "") - } - singleJoiningSlash := func(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b - } - return a + b - } - target := p.defaultUrl - targetQuery := target.RawQuery - req.URL.Scheme = target.Scheme - req.URL.Host = target.Host - req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) - if targetQuery == "" || req.URL.RawQuery == "" { - req.URL.RawQuery = targetQuery + req.URL.RawQuery - } else { - req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery - } } type myTripper struct { http.RoundTripper - p *Proxy + p *SapProxy } func (t *myTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { @@ -92,29 +60,30 @@ func (t *myTripper) RoundTrip(req *http.Request) (resp *http.Response, err error return resp, nil } -func (p *Proxy) modifyResponse(r *http.Response) error { +func (s *SapProxy) modifyResponse(r *http.Response) error { return nil } -func (p *Proxy) Serve() { +func (s *SapProxy) Serve() { + log.Printf("superlcx work in proxy mode!") proxy := &httputil.ReverseProxy{ - Director: p.director, - Transport: &myTripper{RoundTripper: http.DefaultTransport, p: p}, - ModifyResponse: p.modifyResponse, + Director: s.director, + Transport: &myTripper{RoundTripper: http.DefaultTransport, p: s}, + ModifyResponse: s.modifyResponse, } - panic(http.Serve(p.lis, proxy)) + panic(http.Serve(s.lis, proxy)) } -func (p *Proxy) Register(middleware string) { +func (s *SapProxy) Register(middleware string) { if middleware != "" { ms := strings.Split(middleware, ",") for _, m := range ms { switch m { case "stdout": - p.RegisterMiddleware(stdout.HandleRequest, stdout.HandleResponse) + s.RegisterMiddleware(stdout.HandleRequest, stdout.HandleResponse) default: reqH, respH := find(m) - p.RegisterMiddleware(reqH, respH) + s.RegisterMiddleware(reqH, respH) } } }