From 3baf04e57af02561d2c9ba3f9d73a4dccccf160c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 4 Jan 2022 16:27:45 +0100 Subject: [PATCH 1/3] doc: show how to use the experimental tls factory This commit adds a new example showing how one can use the experimental TLSClientFactory function to create an uTLS connection when using a SOCKS5 (or other) proxy. Users have requested this functionality. The example seems to work for me. Compare: ``` sbs@localhost:~$ go run . -utls | jq -r .ja3_hash 0ffee3ba8e615ad22535e7f771690a28 ``` to: ``` sbs@localhost:~$ go run . | jq -r .ja3_hash 3fed133de60c35724739b913924b6c24 ``` Once users have confirmed this code works as intended, I'll update the README to advert this functionality. --- example/example-proxy/README.md | 10 +++++ example/example-proxy/go.mod | 16 ++++++++ example/example-proxy/go.sum | 22 ++++++++++ example/example-proxy/main.go | 70 ++++++++++++++++++++++++++++++++ example/example-proxy/tls.go | 72 +++++++++++++++++++++++++++++++++ example/example-utls/go.mod | 4 +- example/example-utls/go.sum | 7 ++-- 7 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 example/example-proxy/README.md create mode 100644 example/example-proxy/go.mod create mode 100644 example/example-proxy/go.sum create mode 100644 example/example-proxy/main.go create mode 100644 example/example-proxy/tls.go diff --git a/example/example-proxy/README.md b/example/example-proxy/README.md new file mode 100644 index 00000000..48122213 --- /dev/null +++ b/example/example-proxy/README.md @@ -0,0 +1,10 @@ +# Example of using TLSClientFactory + +This example shows how you can use `TLSClientFactory` to force this +library to use `refraction-networking/utls` for proxied conns. + +* [main.go](main.go) sets up an `oohttp.Transport` instance and uses +the `Proxy` field to configure a SOCKS5 proxy; + +* [utls.go](utls.go) contains the code to replace `tls` with `utls` +when using a proxy through the `TLSClientFactory`. diff --git a/example/example-proxy/go.mod b/example/example-proxy/go.mod new file mode 100644 index 00000000..0c338326 --- /dev/null +++ b/example/example-proxy/go.mod @@ -0,0 +1,16 @@ +module github.com/ooni/oohttp/example/example-proxy + +go 1.17 + +require ( + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + github.com/ooni/oohttp v0.0.0-20220104151715-47436feafc4c + github.com/refraction-networking/utls v1.0.0 +) + +require ( + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/text v0.3.7 // indirect +) diff --git a/example/example-proxy/go.sum b/example/example-proxy/go.sum new file mode 100644 index 00000000..c4b033a8 --- /dev/null +++ b/example/example-proxy/go.sum @@ -0,0 +1,22 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/ooni/oohttp v0.0.0-20220104151715-47436feafc4c h1:9iEGMyxk72orD7hPxU8dzQL4frjOUEX1YTqMvK2/idM= +github.com/ooni/oohttp v0.0.0-20220104151715-47436feafc4c/go.mod h1:IhQrrBqPX6/Pb8mGKdw3wskNOk5r8ngqmqAXrH3j7vw= +github.com/refraction-networking/utls v1.0.0 h1:6XQHSjDmeBCF9sPq8p2zMVGq7Ud3rTD2q88Fw8Tz1tA= +github.com/refraction-networking/utls v1.0.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211205041911-012df41ee64c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/example/example-proxy/main.go b/example/example-proxy/main.go new file mode 100644 index 00000000..9c0cb7b0 --- /dev/null +++ b/example/example-proxy/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net/http" + "net/url" + "time" + + "github.com/armon/go-socks5" + oohttp "github.com/ooni/oohttp" +) + +// startProxyServer starts a proxy server at the given endpoint. +func startProxyServer(endpoint string) { + conf := &socks5.Config{} + server, err := socks5.New(conf) + if err != nil { + log.Fatal(err) + } + if err := server.ListenAndServe("tcp", endpoint); err != nil { + log.Fatal(err) + } +} + +// useProxy uses the oohttp library to possibly use refraction-networking/utls +// when communicating with a remote TLS endpoint through the given proxy. +func useProxy(URL, proxy string) { + w := &oohttp.StdlibTransport{ + Transport: &oohttp.Transport{ + Proxy: func(*oohttp.Request) (*url.URL, error) { + return &url.URL{Scheme: "socks5", Host: proxy}, nil + }, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } + clnt := &http.Client{Transport: w} + resp, err := clnt.Get(URL) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + r := io.LimitReader(resp.Body, 1<<22) + data, err := io.ReadAll(r) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s\n", string(data)) +} + +func main() { + proxy := flag.String("proxy", "127.0.0.1:54321", "where the proxy shold listen") + // The default URL we use should return us the JA3 fingerprint + // we're using to communicate with the server. We expect such a + // fingerprint to change when we use the `-utls` flag. + url := flag.String("url", "https://ja3er.com/json", "the URL to get") + utls := flag.Bool("utls", false, "try using uTLS") + flag.Parse() + if *utls { + useUTLS() + } + go startProxyServer(*proxy) + useProxy(*url, *proxy) +} diff --git a/example/example-proxy/tls.go b/example/example-proxy/tls.go new file mode 100644 index 00000000..51c9e2fd --- /dev/null +++ b/example/example-proxy/tls.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "crypto/tls" + "net" + + oohttp "github.com/ooni/oohttp" + utls "github.com/refraction-networking/utls" +) + +// utlsConnAdapter adapts utls.UConn to the oohttp.TLSConn interface. +type utlsConnAdapter struct { + *utls.UConn +} + +var _ oohttp.TLSConn = &utlsConnAdapter{} + +// ConnectionState implements TLSConn's ConnectionState. +func (c *utlsConnAdapter) ConnectionState() tls.ConnectionState { + ustate := c.UConn.ConnectionState() + return tls.ConnectionState{ + Version: ustate.Version, + HandshakeComplete: ustate.HandshakeComplete, + DidResume: ustate.DidResume, + CipherSuite: ustate.CipherSuite, + NegotiatedProtocol: ustate.NegotiatedProtocol, + NegotiatedProtocolIsMutual: ustate.NegotiatedProtocolIsMutual, + ServerName: ustate.ServerName, + PeerCertificates: ustate.PeerCertificates, + VerifiedChains: ustate.VerifiedChains, + SignedCertificateTimestamps: ustate.SignedCertificateTimestamps, + OCSPResponse: ustate.OCSPResponse, + TLSUnique: ustate.TLSUnique, + } +} + +// HandshakeContext implements TLSConn's HandshakeContext. +func (c *utlsConnAdapter) HandshakeContext(ctx context.Context) error { + errch := make(chan error, 1) + go func() { + errch <- c.UConn.Handshake() + }() + select { + case err := <-errch: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// useUTLS configures oohttp to use uTLS +func useUTLS() { + // Warning: this code modifies TLSClientFactory without locking it, which is + // a race-safe pattern only when you do that before you use HTTP code. + // + // What we basically do here is the following: + // + // 1. we create an utls.UConn + // + // 2. we use an adapter so the oohttp library sees it as a TLSConn + oohttp.TLSClientFactory = func(conn net.Conn, config *tls.Config) oohttp.TLSConn { + uConfig := &utls.Config{ + RootCAs: config.RootCAs, + NextProtos: config.NextProtos, + ServerName: config.ServerName, + InsecureSkipVerify: config.InsecureSkipVerify, + DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled, + } + return &utlsConnAdapter{utls.UClient(conn, uConfig, utls.HelloFirefox_55)} + } +} diff --git a/example/example-utls/go.mod b/example/example-utls/go.mod index ffca0584..9fe834d7 100644 --- a/example/example-utls/go.mod +++ b/example/example-utls/go.mod @@ -2,11 +2,11 @@ module github.com/ooni/oohttp/example/example-utls go 1.17 -require github.com/ooni/oohttp v0.0.0-20211206145729-209dd31a30a4 +require github.com/ooni/oohttp v0.0.0-20220104151715-47436feafc4c require ( github.com/refraction-networking/utls v1.0.0 - golang.org/x/net v0.0.0-20211205041911-012df41ee64c // indirect + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/text v0.3.7 // indirect ) diff --git a/example/example-utls/go.sum b/example/example-utls/go.sum index b31559c5..727c4995 100644 --- a/example/example-utls/go.sum +++ b/example/example-utls/go.sum @@ -1,12 +1,13 @@ -github.com/ooni/oohttp v0.0.0-20211206145729-209dd31a30a4 h1:epdBKp+qNzXw4xb18H8DWz8/GzwF4kL6AG84w5FdO2s= -github.com/ooni/oohttp v0.0.0-20211206145729-209dd31a30a4/go.mod h1:IhQrrBqPX6/Pb8mGKdw3wskNOk5r8ngqmqAXrH3j7vw= +github.com/ooni/oohttp v0.0.0-20220104151715-47436feafc4c h1:9iEGMyxk72orD7hPxU8dzQL4frjOUEX1YTqMvK2/idM= +github.com/ooni/oohttp v0.0.0-20220104151715-47436feafc4c/go.mod h1:IhQrrBqPX6/Pb8mGKdw3wskNOk5r8ngqmqAXrH3j7vw= github.com/refraction-networking/utls v1.0.0 h1:6XQHSjDmeBCF9sPq8p2zMVGq7Ud3rTD2q88Fw8Tz1tA= github.com/refraction-networking/utls v1.0.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211205041911-012df41ee64c h1:7SfqwP5fxEtl/P02w5IhKc86ziJ+A25yFrkVgoy2FT8= golang.org/x/net v0.0.0-20211205041911-012df41ee64c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 984314b787646a2654270cdbe5f9678e64eb73de Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 4 Jan 2022 16:45:16 +0100 Subject: [PATCH 2/3] fix: avoid race between listen and connect Interestingly, I didn't see this on macOS but now I see it on my Linux system (where I just tested with `-race`). --- example/example-proxy/main.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/example/example-proxy/main.go b/example/example-proxy/main.go index 9c0cb7b0..03632c78 100644 --- a/example/example-proxy/main.go +++ b/example/example-proxy/main.go @@ -7,20 +7,26 @@ import ( "log" "net/http" "net/url" + "net" "time" "github.com/armon/go-socks5" oohttp "github.com/ooni/oohttp" ) -// startProxyServer starts a proxy server at the given endpoint. -func startProxyServer(endpoint string) { +// startProxyServer starts a SOCKS5 proxy server at the given endpoint. +func startProxyServer(endpoint string, ch chan<- interface{}) { conf := &socks5.Config{} server, err := socks5.New(conf) if err != nil { log.Fatal(err) } - if err := server.ListenAndServe("tcp", endpoint); err != nil { + listener, err := net.Listen("tcp", endpoint) + if err != nil { + log.Fatal(err) + } + close(ch) // signal the main goroutine it can now continue + if err := server.Serve(listener); err != nil { log.Fatal(err) } } @@ -65,6 +71,8 @@ func main() { if *utls { useUTLS() } - go startProxyServer(*proxy) + ch := make(chan interface {}) + go startProxyServer(*proxy, ch) + <-ch // wait for the listener to be listening useProxy(*url, *proxy) } From de9a54abd733cbe709e2565118d56e06232c255e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 4 Jan 2022 16:47:05 +0100 Subject: [PATCH 3/3] Update example/example-proxy/main.go --- example/example-proxy/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/example-proxy/main.go b/example/example-proxy/main.go index 03632c78..f1745b6d 100644 --- a/example/example-proxy/main.go +++ b/example/example-proxy/main.go @@ -61,7 +61,7 @@ func useProxy(URL, proxy string) { } func main() { - proxy := flag.String("proxy", "127.0.0.1:54321", "where the proxy shold listen") + proxy := flag.String("proxy", "127.0.0.1:54321", "where the proxy should listen") // The default URL we use should return us the JA3 fingerprint // we're using to communicate with the server. We expect such a // fingerprint to change when we use the `-utls` flag.