Skip to content

Commit

Permalink
add blend mode to
Browse files Browse the repository at this point in the history
  • Loading branch information
DashJay committed Jun 30, 2020
1 parent 492cc28 commit c801985
Show file tree
Hide file tree
Showing 9 changed files with 199 additions and 62 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.CN.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1. 添加中间件模式
1. 添加混合模式
2 changes: 1 addition & 1 deletion CHANGELOG.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1. add middleware mode for superlcx
1. add blendmode.
5 changes: 3 additions & 2 deletions README.CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ Usage of ./superlcx:
- 优点: 拷贝模式直接工作在TCP层, 他并不关心转发的是什么. 在 `io.Copy`的帮助下,运行只需要非常少的内存。
- 缺点: 拷贝模式并不知道应用层的内容. 所有他可以做的仅仅是全部dump出来,做其他操作。
- 融合模式(开发中)
- 期待优点:占用非常小的内存,提供丰富的接口例如`proxy_pass`
- 融合模式
- 优点:混合模式虽然工作在TCP层,但是以牺牲解析HTTP为代价,为请求和返回值的分析提供了中间间模式运行的可能性。
- 缺点:因为IO量可能比较大,可能会引起内存抖动,大量gc,在高并发下不适合小内存的机器。
### -M 中间件
当工作在代理模式下,可以调用中间件来对过程中的流量进行分析。例如,系统内置stdout中间件(示例中间件),可以通过`-M stdout`来使用。
Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@ Usage of ./superlcx:
### mode
- proxy
- advantages: the proxy mode work with http proxy package which can offer(expose) more api like modifyResponse,Transport,director, etc.
- disadvantages: the proxy mode will cause memory mem jitter, not suitable for limited memory machine if you need high performance
- disadvantages: the proxy mode will cause memory jitter, not suitable for limited memory machine if you need high performance
- copy
- advantages: the copy mode directly work on tcp layer, it doesn't care about what would be transfer. With the help of `io.Copy`, it needs less RAM.
- disadvantages: the copy mode know nothing about application layer. all things it can do is dumping them all out.
- hybrid(ing)
- (expect)advantages: allocate low memory and expose more API like proxy_pass...etc
- blend
- advantages: allocate low memory comparing with the proxy mode. it can run with the middleware interface.
- disadvantages: the blend mode still need more memory(less than proxy), and could lead to memory jitter.
### -M middleware
When working in the proxy mode, middleware can be invoked to analyze the traffic in the process. For example, the built-in stdout middleware (sample middleware) can be used via '-M stdout'.
Expand Down
22 changes: 20 additions & 2 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ var (
hostPort string
mode string
middleware string
logCfg string
)

func init() {
Expand All @@ -37,9 +38,10 @@ func init() {
}()
flag.BoolVar(&showVersion, "v", false, "show version and about then exit.")
flag.IntVar(&listenPort, "l", 8080, "listen port")
flag.StringVar(&hostPort, "host", "0.0.0.0:8081", "target host:port")
flag.StringVar(&mode, "m", "proxy", "run mode")
flag.StringVar(&hostPort, "host", "0.0.0.0:8081", "target host:port.")
flag.StringVar(&mode, "m", "proxy", "run mode <proxy|copy|blend>.")
flag.StringVar(&middleware, "M", "", "middleware, comma separated if more than one, eg: --M stdout,dumps")
flag.StringVar(&logCfg, "log", "t", "l -> line of code, d -> date, t -> time, order doesn't matter")
flag.Parse()
if listenPort < 1 || listenPort > 65535 {
panic("[x] Listen Port Invalid")
Expand All @@ -61,6 +63,19 @@ Superlcx [%s], a tool kit for port transfer with middlewares!
`, version)
os.Exit(0)
}
logC := strings.Split(logCfg, "")
logFlag := log.Ltime
for _, c := range logC {
switch c {
case "d":
logFlag |= log.Ldate
case "l":
logFlag |= log.Lshortfile
default:
continue
}
}
log.SetFlags(logFlag)
// Buried point for debug
go func() {
http.ListenAndServe(":8999", nil)
Expand All @@ -80,6 +95,9 @@ Superlcx [%s], a tool kit for port transfer with middlewares!
case "copy":
c := core.NewSapCopy(lis, hostPort)
c.Serve()
case "blend":
c := core.NewSapBlend(lis, hostPort, middleware)
c.Serve()
default:
flag.PrintDefaults()
os.Exit(-1)
Expand Down
81 changes: 81 additions & 0 deletions core/blend.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package core

import (
"bufio"
"context"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"time"
)

const maxReadTime = 10 * time.Second

type SapBlend struct {
lis net.Listener
defaultUrl *url.URL
*middleware
}

func NewSapBlend(lis net.Listener, target string, middleware string) *SapBlend {
defaultUrl, err := url.Parse(fmt.Sprintf("http://%s/", target))
if err != nil {
panic(fmt.Sprintf("default url [%s] parse error, detail:[%s]", defaultUrl, err))
}
log.Printf("parse default url as %s", defaultUrl)
b := &SapBlend{lis: lis, defaultUrl: defaultUrl, middleware: newMiddleware(middleware)}

return b
}

func (s *SapBlend) Serve() {
log.Printf("superlcx work in blend mode!")
tr := http.DefaultTransport
for {
conn, err := s.lis.Accept()
if err != nil {
log.Printf("[x] listener accept error, detail:[%s]", err)
}
go func() {
buf := bufio.NewReader(conn)
conn.SetDeadline(time.Now().Add(maxReadTime))
defer conn.Close()
wait := 100 * time.Microsecond
for {
ctx, cancel := context.WithCancel(context.Background())
req, err := http.ReadRequest(buf)
if err != nil {
if err == io.EOF {
time.Sleep(wait)
wait *= 2
continue
} else {
// 这里能识别已经关闭的链接
log.Printf("[-] connection over, detail:[%s]", err)
return
}
}
newReq := req.Clone(ctx)
organizeUrl(newReq, s.defaultUrl)

for _, reqH := range s.reqHandlers {
reqH(newReq)
}
resp, err := tr.RoundTrip(newReq)
if err != nil {
log.Printf("[x] default transport req error, detail:[%s]", err)
continue
}
for _, respH := range s.respHandlers {
respH(resp)
}

resp.Write(conn)
cancel()
}
}()
}
}
66 changes: 66 additions & 0 deletions core/common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package core

import (
"net/http"
"net/url"
"strings"

"superlcx/middlewares/stdout"
)

func organizeUrl(req *http.Request, target *url.URL) {
if _, ok := req.Header["User-Agent"]; !ok {
// explicitly disable User-Agent so it's not set to default value
req.Header.Set("User-Agent", "")
}
singleJoiningSlash := func(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
targetQuery := target.RawQuery
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
}

type middleware struct {
reqHandlers []func(req *http.Request)
respHandlers []func(resp *http.Response)
}

func newMiddleware(mid string) *middleware {
middle := &middleware{
reqHandlers: []func(req *http.Request){},
respHandlers: []func(resp *http.Response){},
}
if mid != "" {
ms := strings.Split(mid, ",")
for _, m := range ms {
switch m {
case "stdout":
middle.RegisterMiddleware(stdout.HandleRequest, stdout.HandleResponse)
default:
reqH, respH := find(m)
middle.RegisterMiddleware(reqH, respH)
}
}
}
return middle
}

func (m *middleware) RegisterMiddleware(reqH func(req *http.Request), respH func(resp *http.Response)) {
m.reqHandlers = append(m.reqHandlers, reqH)
m.respHandlers = append(m.respHandlers, respH)
}
1 change: 1 addition & 0 deletions core/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type SapCopy struct {
}

func (s *SapCopy) Serve() {
log.Printf("superlcx work in copy mode!")
for {
conn, err := s.lis.Accept()
if err != nil {
Expand Down
75 changes: 22 additions & 53 deletions core/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,69 +13,37 @@ import (
"superlcx/middlewares/stdout"
)

type Proxy struct {
defaultUrl *url.URL
lis net.Listener
reqHandlers []func(req *http.Request)
respHandlers []func(resp *http.Response)
}

func (p *Proxy) RegisterMiddleware(reqH func(req *http.Request), respH func(resp *http.Response)) {
p.reqHandlers = append(p.reqHandlers, reqH)
p.respHandlers = append(p.respHandlers, respH)
type SapProxy struct {
defaultUrl *url.URL
lis net.Listener
middleware
}

// NewSapProxy 构建一个SapProxy
func NewSapProxy(lis net.Listener, defaultHost string, middleware string) *Proxy {
func NewSapProxy(lis net.Listener, defaultHost string, middleware string) *SapProxy {
u, err := url.Parse(fmt.Sprintf("http://%s", defaultHost))
if err != nil {
panic(err)
}
log.Printf("parse default url as %s", u)
p := &Proxy{
defaultUrl: u,
lis: lis,
reqHandlers: make([]func(req *http.Request), 0),
respHandlers: make([]func(resp *http.Response), 0),
p := &SapProxy{
defaultUrl: u,
lis: lis,
}
p.Register(middleware)
return p
}

func (p *Proxy) director(req *http.Request) {
for _, fn := range p.reqHandlers {
func (s *SapProxy) director(req *http.Request) {
organizeUrl(req, s.defaultUrl)
for _, fn := range s.reqHandlers {
fn(req)
}
if _, ok := req.Header["User-Agent"]; !ok {
// explicitly disable User-Agent so it's not set to default value
req.Header.Set("User-Agent", "")
}
singleJoiningSlash := func(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
target := p.defaultUrl
targetQuery := target.RawQuery
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
}

type myTripper struct {
http.RoundTripper
p *Proxy
p *SapProxy
}

func (t *myTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
Expand All @@ -92,29 +60,30 @@ func (t *myTripper) RoundTrip(req *http.Request) (resp *http.Response, err error

return resp, nil
}
func (p *Proxy) modifyResponse(r *http.Response) error {
func (s *SapProxy) modifyResponse(r *http.Response) error {
return nil
}

func (p *Proxy) Serve() {
func (s *SapProxy) Serve() {
log.Printf("superlcx work in proxy mode!")
proxy := &httputil.ReverseProxy{
Director: p.director,
Transport: &myTripper{RoundTripper: http.DefaultTransport, p: p},
ModifyResponse: p.modifyResponse,
Director: s.director,
Transport: &myTripper{RoundTripper: http.DefaultTransport, p: s},
ModifyResponse: s.modifyResponse,
}
panic(http.Serve(p.lis, proxy))
panic(http.Serve(s.lis, proxy))
}

func (p *Proxy) Register(middleware string) {
func (s *SapProxy) Register(middleware string) {
if middleware != "" {
ms := strings.Split(middleware, ",")
for _, m := range ms {
switch m {
case "stdout":
p.RegisterMiddleware(stdout.HandleRequest, stdout.HandleResponse)
s.RegisterMiddleware(stdout.HandleRequest, stdout.HandleResponse)
default:
reqH, respH := find(m)
p.RegisterMiddleware(reqH, respH)
s.RegisterMiddleware(reqH, respH)
}
}
}
Expand Down

0 comments on commit c801985

Please sign in to comment.