From d188b28308ace555d20025017e6ea0392c49bbcc Mon Sep 17 00:00:00 2001 From: Chinmay Kousik Date: Fri, 8 Jul 2022 12:28:25 +0530 Subject: [PATCH 01/86] WebRTC transport implementation --- go.mod | 17 +- go.sum | 85 ++ p2p/transport/webrtc/connection.go | 368 +++++++ p2p/transport/webrtc/datachannel.go | 28 + p2p/transport/webrtc/fetch_ip_linux_test.go | 10 + p2p/transport/webrtc/fetch_ip_test.go | 40 + p2p/transport/webrtc/internal/encoding/hex.go | 161 +++ .../webrtc/internal/encoding/hex_test.go | 126 +++ p2p/transport/webrtc/internal/fingerprint.go | 26 + p2p/transport/webrtc/internal/sdp.go | 144 +++ p2p/transport/webrtc/internal/util.go | 34 + p2p/transport/webrtc/internal/util_test.go | 101 ++ p2p/transport/webrtc/listener.go | 332 +++++++ p2p/transport/webrtc/message.go | 3 + p2p/transport/webrtc/pb/generate.go | 3 + p2p/transport/webrtc/pb/message.pb.go | 224 +++++ p2p/transport/webrtc/pb/message.proto | 20 + p2p/transport/webrtc/stream.go | 198 ++++ p2p/transport/webrtc/stream_error.go | 25 + p2p/transport/webrtc/stream_read.go | 120 +++ p2p/transport/webrtc/stream_state.go | 136 +++ p2p/transport/webrtc/stream_write.go | 182 ++++ p2p/transport/webrtc/transport.go | 505 ++++++++++ p2p/transport/webrtc/transport_test.go | 914 ++++++++++++++++++ p2p/transport/webrtc/udpmux/mux.go | 293 ++++++ p2p/transport/webrtc/udpmux/mux_test.go | 87 ++ .../webrtc/udpmux/muxed_connection.go | 98 ++ p2p/transport/webrtc/udpmux/packetqueue.go | 92 ++ .../webrtc/udpmux/packetqueue_bench_test.go | 41 + .../webrtc/udpmux/packetqueue_test.go | 61 ++ p2p/transport/webrtc/webrtc.go | 15 + test-plans/go.mod | 21 +- 32 files changed, 4500 insertions(+), 10 deletions(-) create mode 100644 p2p/transport/webrtc/connection.go create mode 100644 p2p/transport/webrtc/datachannel.go create mode 100644 p2p/transport/webrtc/fetch_ip_linux_test.go create mode 100644 p2p/transport/webrtc/fetch_ip_test.go create mode 100644 p2p/transport/webrtc/internal/encoding/hex.go create mode 100644 p2p/transport/webrtc/internal/encoding/hex_test.go create mode 100644 p2p/transport/webrtc/internal/fingerprint.go create mode 100644 p2p/transport/webrtc/internal/sdp.go create mode 100644 p2p/transport/webrtc/internal/util.go create mode 100644 p2p/transport/webrtc/internal/util_test.go create mode 100644 p2p/transport/webrtc/listener.go create mode 100644 p2p/transport/webrtc/message.go create mode 100644 p2p/transport/webrtc/pb/generate.go create mode 100644 p2p/transport/webrtc/pb/message.pb.go create mode 100644 p2p/transport/webrtc/pb/message.proto create mode 100644 p2p/transport/webrtc/stream.go create mode 100644 p2p/transport/webrtc/stream_error.go create mode 100644 p2p/transport/webrtc/stream_read.go create mode 100644 p2p/transport/webrtc/stream_state.go create mode 100644 p2p/transport/webrtc/stream_write.go create mode 100644 p2p/transport/webrtc/transport.go create mode 100644 p2p/transport/webrtc/transport_test.go create mode 100644 p2p/transport/webrtc/udpmux/mux.go create mode 100644 p2p/transport/webrtc/udpmux/mux_test.go create mode 100644 p2p/transport/webrtc/udpmux/muxed_connection.go create mode 100644 p2p/transport/webrtc/udpmux/packetqueue.go create mode 100644 p2p/transport/webrtc/udpmux/packetqueue_bench_test.go create mode 100644 p2p/transport/webrtc/udpmux/packetqueue_test.go create mode 100644 p2p/transport/webrtc/webrtc.go diff --git a/go.mod b/go.mod index da172c0581..e42b65f804 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,11 @@ require ( github.com/multiformats/go-multistream v0.4.1 github.com/multiformats/go-varint v0.0.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/pion/datachannel v1.5.5 + github.com/pion/ice/v2 v2.3.6 + github.com/pion/logging v0.2.2 + github.com/pion/stun v0.6.0 + github.com/pion/webrtc/v3 v3.2.9 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.4.0 github.com/quic-go/quic-go v0.38.1 @@ -94,9 +99,19 @@ require ( github.com/miekg/dns v1.1.55 // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect + github.com/pion/interceptor v0.1.17 // indirect + github.com/pion/mdns v0.0.7 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.10 // indirect + github.com/pion/rtp v1.7.13 // indirect + github.com/pion/sctp v1.8.7 // indirect + github.com/pion/sdp/v3 v3.0.6 // indirect + github.com/pion/srtp/v2 v2.0.15 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/turn/v2 v2.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/go.sum b/go.sum index 8e84048fc9..c9f6e8e93a 100644 --- a/go.sum +++ b/go.sum @@ -381,6 +381,7 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= @@ -388,6 +389,7 @@ github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= @@ -396,6 +398,45 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= +github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= +github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= +github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= +github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= +github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= +github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= +github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= +github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= +github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= +github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= +github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= +github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= +github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= +github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= +github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= +github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= +github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= +github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= +github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= +github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= +github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= +github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= +github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= +github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -444,6 +485,7 @@ github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtD github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -485,12 +527,19 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -506,6 +555,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -546,6 +596,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -583,6 +636,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -623,9 +678,17 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -649,6 +712,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -709,12 +774,25 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -722,6 +800,11 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -778,6 +861,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go new file mode 100644 index 0000000000..6b24b7c744 --- /dev/null +++ b/p2p/transport/webrtc/connection.go @@ -0,0 +1,368 @@ +package libp2pwebrtc + +import ( + "context" + "errors" + "fmt" + "math" + "math/rand" + "os" + "sync" + "sync/atomic" + + ic "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + tpt "github.com/libp2p/go-libp2p/core/transport" + pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-msgio" + ma "github.com/multiformats/go-multiaddr" + "github.com/pion/datachannel" + "github.com/pion/webrtc/v3" + "google.golang.org/protobuf/proto" +) + +var _ tpt.CapableConn = &connection{} + +const ( + maxAcceptQueueLen = 10 +) + +type acceptStream struct { + stream datachannel.ReadWriteCloser + channel *webrtc.DataChannel +} + +type connection struct { + // debug identifier for the connection + dbgId int + pc *webrtc.PeerConnection + transport *WebRTCTransport + scope network.ConnManagementScope + + localPeer peer.ID + localMultiaddr ma.Multiaddr + + remotePeer peer.ID + remoteKey ic.PubKey + remoteMultiaddr ma.Multiaddr + + m sync.Mutex + streams map[uint16]*webRTCStream + + acceptQueue chan acceptStream + idAllocator *sidAllocator + + ctx context.Context + cancel context.CancelFunc +} + +func newConnection( + direction network.Direction, + pc *webrtc.PeerConnection, + transport *WebRTCTransport, + scope network.ConnManagementScope, + + localPeer peer.ID, + localMultiaddr ma.Multiaddr, + + remotePeer peer.ID, + remoteKey ic.PubKey, + remoteMultiaddr ma.Multiaddr, +) (*connection, error) { + idAllocator := newSidAllocator(direction) + + ctx, cancel := context.WithCancel(context.Background()) + conn := &connection{ + dbgId: rand.Intn(65536), + pc: pc, + transport: transport, + scope: scope, + + localPeer: localPeer, + localMultiaddr: localMultiaddr, + + remotePeer: remotePeer, + remoteKey: remoteKey, + remoteMultiaddr: remoteMultiaddr, + ctx: ctx, + cancel: cancel, + streams: make(map[uint16]*webRTCStream), + idAllocator: idAllocator, + + acceptQueue: make(chan acceptStream, maxAcceptQueueLen), + } + + pc.OnConnectionStateChange(conn.onConnectionStateChange) + pc.OnDataChannel(func(dc *webrtc.DataChannel) { + if conn.IsClosed() { + return + } + dc.OnOpen(func() { + rwc, err := dc.Detach() + if err != nil { + log.Warnf("could not detach datachannel: id: %d", *dc.ID()) + return + } + select { + case conn.acceptQueue <- acceptStream{rwc, dc}: + default: + log.Warnf("connection busy, rejecting stream") + b, _ := proto.Marshal(&pb.Message{Flag: pb.Message_RESET.Enum()}) + w := msgio.NewWriter(rwc) + w.WriteMsg(b) + rwc.Close() + } + }) + }) + + return conn, nil +} + +func (c *connection) resetStreams() { + if c.IsClosed() { + return + } + c.m.Lock() + defer c.m.Unlock() + for k, stream := range c.streams { + // reset the streams, but we do not need to be notified + // of stream closure + stream.close(true, false) + delete(c.streams, k) + } + +} + +// ConnState implements transport.CapableConn +func (c *connection) ConnState() network.ConnectionState { + return network.ConnectionState{ + Transport: "p2p-webrtc-direct", + } +} + +// Close closes the underlying peerconnection. +func (c *connection) Close() error { + if c.IsClosed() { + return nil + } + + c.scope.Done() + c.cancel() + return c.pc.Close() +} + +func (c *connection) IsClosed() bool { + select { + case <-c.ctx.Done(): + return true + default: + return false + } +} + +func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error) { + if c.IsClosed() { + return nil, os.ErrClosed + } + + streamID, err := c.idAllocator.nextID() + if err != nil { + return nil, err + } + dc, err := c.pc.CreateDataChannel("", &webrtc.DataChannelInit{ID: streamID}) + if err != nil { + return nil, err + } + rwc, err := c.detachChannel(ctx, dc) + if err != nil { + return nil, fmt.Errorf("open stream: %w", err) + } + stream := newStream( + c, + dc, + rwc, + nil, + nil, + ) + err = c.addStream(stream) + if err != nil { + stream.Close() + return nil, err + } + return stream, nil +} + +func (c *connection) AcceptStream() (network.MuxedStream, error) { + select { + case <-c.ctx.Done(): + return nil, os.ErrClosed + case accStream := <-c.acceptQueue: + stream := newStream( + c, + accStream.channel, + accStream.stream, + nil, + nil, + ) + if err := c.addStream(stream); err != nil { + stream.Close() + return nil, err + } + return stream, nil + } +} + +// implement network.ConnSecurity +func (c *connection) LocalPeer() peer.ID { + return c.localPeer +} + +func (c *connection) RemotePeer() peer.ID { + return c.remotePeer +} + +func (c *connection) RemotePublicKey() ic.PubKey { + return c.remoteKey +} + +// implement network.ConnMultiaddrs +func (c *connection) LocalMultiaddr() ma.Multiaddr { + return c.localMultiaddr +} + +func (c *connection) RemoteMultiaddr() ma.Multiaddr { + return c.remoteMultiaddr +} + +// implement network.ConnScoper +func (c *connection) Scope() network.ConnScope { + return c.scope +} + +func (c *connection) Transport() tpt.Transport { + return c.transport +} + +func (c *connection) addStream(stream *webRTCStream) error { + c.m.Lock() + defer c.m.Unlock() + if _, ok := c.streams[stream.id]; ok { + return errors.New("stream ID already exists") + } + c.streams[stream.id] = stream + return nil +} + +func (c *connection) removeStream(id uint16) { + c.m.Lock() + defer c.m.Unlock() + delete(c.streams, id) +} + +func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { + log.Debugf("[%s][%d] handling peerconnection state: %s", c.localPeer, c.dbgId, state) + if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed { + // reset any streams + c.resetStreams() + c.cancel() + c.scope.Done() + c.pc.Close() + } +} + +// detachChannel detaches an outgoing channel by taking into account the context +// passed to `OpenStream` as well the closure of the underlying peerconnection +// +// The underlying SCTP stream for a datachannel implements a net.Conn interface. +// However, the datachannel creates a goroutine which continuously reads from +// the SCTP stream and surfaces the data using an OnMessage callback. +// +// The actual abstractions are as follows: webrtc.DataChannel +// wraps pion.DataChannel, which wraps sctp.Stream. +// +// The goroutine for reading, Detach method, +// and the OnMessage callback are present at the webrtc.DataChannel level. +// Detach provides us abstracted access to the underlying pion.DataChannel, +// which allows us to issue Read calls to the datachannel. +// This was desired because it was not feasible to introduce backpressure +// with the OnMessage callbacks. The tradeoff is a change in the semantics of +// the OnOpen callback, and having to force close Read locally. +func (c *connection) detachChannel(ctx context.Context, dc *webrtc.DataChannel) (rwc datachannel.ReadWriteCloser, err error) { + done := make(chan struct{}) + // OnOpen will return immediately for detached datachannels + // refer: https://github.com/pion/webrtc/blob/7ab3174640b3ce15abebc2516a2ca3939b5f105f/datachannel.go#L278-L282 + dc.OnOpen(func() { + rwc, err = dc.Detach() + // this is safe since the function should return instantly if the peerconnection is closed + close(done) + }) + select { + case <-c.ctx.Done(): + return nil, errors.New("connection closed") + case <-ctx.Done(): + return nil, ctx.Err() + case <-done: + } + return +} + +// A note on these setters and why they are needed: +// +// The connection object sets up receiving datachannels (streams) from the remote peer. +// Please consider the XX noise handshake pattern from a peer A to peer B as described at: +// https://noiseexplorer.com/patterns/XX/ +// +// The initiator A completes the noise handshake before B. +// This would allow A to create new datachannels before B has set up the callbacks to process incoming datachannels. +// This would create a situation where A has successfully created a stream but B is not aware of it. +// Moving the construction of the connection object before the noise handshake eliminates this issue, +// as callbacks have been set up for both peers. +// +// This could lead to a case where streams are created during the noise handshake, +// and the handshake fails. In this case, we would close the underlying peerconnection. + +// only used during connection setup +func (c *connection) setRemotePeer(id peer.ID) { + c.remotePeer = id +} + +// only used during connection setup +func (c *connection) setRemotePublicKey(key ic.PubKey) { + c.remoteKey = key +} + +// sidAllocator is a helper struct to allocate stream IDs for datachannels. ID +// reuse is not currently implemented. This prevents streams in pion from hanging +// with `invalid DCEP message` errors. +// The id is picked using the scheme described in: +// https://datatracker.ietf.org/doc/html/draft-ietf-rtcweb-data-channel-08#section-6.5 +// By definition, the DTLS role for inbound connections is set to DTLS Server, +// and outbound connections are DTLS Client. +type sidAllocator struct { + n atomic.Uint32 +} + +func newSidAllocator(direction network.Direction) *sidAllocator { + switch direction { + case network.DirInbound: + // server will use odd values + a := new(sidAllocator) + a.n.Store(1) + return a + case network.DirOutbound: + // client will use even values + return new(sidAllocator) + default: + panic(fmt.Sprintf("create SID allocator for direction: %s", direction)) + } +} + +func (a *sidAllocator) nextID() (*uint16, error) { + nxt := a.n.Add(2) + if nxt > math.MaxUint16 { + return nil, fmt.Errorf("sid exhausted") + } + result := uint16(nxt) + return &result, nil +} diff --git a/p2p/transport/webrtc/datachannel.go b/p2p/transport/webrtc/datachannel.go new file mode 100644 index 0000000000..aedca6d5b0 --- /dev/null +++ b/p2p/transport/webrtc/datachannel.go @@ -0,0 +1,28 @@ +package libp2pwebrtc + +import ( + "context" + + "github.com/pion/datachannel" + "github.com/pion/webrtc/v3" +) + +// only use this if the datachannels are detached, since the OnOpen callback +// will be called immediately. Only use after the peerconnection is open. +// The context should close if the peerconnection underlying the datachannel +// is closed. +func getDetachedChannel(ctx context.Context, dc *webrtc.DataChannel) (rwc datachannel.ReadWriteCloser, err error) { + done := make(chan struct{}) + dc.OnOpen(func() { + defer close(done) + rwc, err = dc.Detach() + }) + // this is safe since for detached datachannels, the peerconnection runs the onOpen + // callback immediately if the SCTP transport is also connected. + select { + case <-done: + case <-ctx.Done(): + return nil, ctx.Err() + } + return +} diff --git a/p2p/transport/webrtc/fetch_ip_linux_test.go b/p2p/transport/webrtc/fetch_ip_linux_test.go new file mode 100644 index 0000000000..8bfe033e13 --- /dev/null +++ b/p2p/transport/webrtc/fetch_ip_linux_test.go @@ -0,0 +1,10 @@ +//go:build linux +// +build linux + +package libp2pwebrtc + +import "net" + +func getListenerAndDialerIP() (net.IP, net.IP) { + return net.IPv4(0, 0, 0, 0), net.IPv4(127, 0, 0, 1) +} diff --git a/p2p/transport/webrtc/fetch_ip_test.go b/p2p/transport/webrtc/fetch_ip_test.go new file mode 100644 index 0000000000..62dde5f434 --- /dev/null +++ b/p2p/transport/webrtc/fetch_ip_test.go @@ -0,0 +1,40 @@ +//go:build !linux +// +build !linux + +package libp2pwebrtc + +import "net" + +// non-linux builds need to bind to a non-loopback interface +// to accept incoming connections. 0.0.0.0 does not work since +// Pion will bind to a local interface which is not loopback +// and there may not be a route from, say 192.168.0.0/16 to 0.0.0.0. + +func getListenerAndDialerIP() (listenerIp net.IP, dialerIp net.IP) { + listenerIp = net.IPv4(0, 0, 0, 0) + dialerIp = net.IPv4(0, 0, 0, 0) + ifaces, err := net.Interfaces() + if err != nil { + return + } + for _, iface := range ifaces { + log.Debugf("checking interface: %s", iface.Name) + if iface.Flags&net.FlagUp == 0 { + continue + } + addrs, err := iface.Addrs() + if err != nil { + return + } + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.IsPrivate() { + if ipnet.IP.To4() != nil { + listenerIp = ipnet.IP.To4() + dialerIp = listenerIp + return + } + } + } + } + return +} diff --git a/p2p/transport/webrtc/internal/encoding/hex.go b/p2p/transport/webrtc/internal/encoding/hex.go new file mode 100644 index 0000000000..a08c5cf20c --- /dev/null +++ b/p2p/transport/webrtc/internal/encoding/hex.go @@ -0,0 +1,161 @@ +package encoding + +// The code in this file is adapted from the Go standard library's hex package. +// As found in https://cs.opensource.google/go/go/+/refs/tags/go1.20.2:src/encoding/hex/hex.go +// +// The reason we adapted the original code is to allow us to deal with interspersed requirements +// while at the same time hex encoding/decoding, without having to do so in two passes. + +import ( + "encoding/hex" + "errors" + "strings" +) + +// EncodeInterspersedHex encodes a byte slice into a string of hex characters, +// separating each encoded byte with a colon (':'). +// +// Example: { 0x01, 0x02, 0x03 } -> "01:02:03" +func EncodeInterspersedHex(src []byte) string { + var builder strings.Builder + EncodeInterspersedHexToBuilder(src, &builder) + return builder.String() +} + +// EncodeInterspersedHexToBuilder encodes a byte slice into a of hex characters, +// separating each encoded byte with a colon (':'). String is written to the builder. +// +// Example: { 0x01, 0x02, 0x03 } -> "01:02:03" +func EncodeInterspersedHexToBuilder(src []byte, builder *strings.Builder) { + if len(src) == 0 { + return + } + builder.Grow(len(src)*3 - 1) + v := src[0] + builder.WriteByte(hextable[v>>4]) + builder.WriteByte(hextable[v&0x0f]) + for _, v = range src[1:] { + builder.WriteByte(':') + builder.WriteByte(hextable[v>>4]) + builder.WriteByte(hextable[v&0x0f]) + } +} + +// DecodeInterspersedHex decodes a byte slice string of hex characters into a byte slice, +// where the hex characters are expected to be separated by a colon (':'). +// +// Example: {'0', '1', ':', '0', '2', ':', '0', '3'} -> { 0x01, 0x02, 0x03 } +func DecodeInterspersedHex(src []byte) ([]byte, error) { + if len(src) == 0 { + return []byte{}, nil + } + if len(src) < 2 { + return nil, hex.ErrLength + } + + dst := make([]byte, (len(src)+1)/3) + i, j := 0, 1 + for ; j < len(src); j += 3 { // jump one extra byte for the separator (:) + p := src[j-1] + q := src[j] + if j+1 < len(src) && src[j+1] != ':' { + return nil, errUnexpectedIntersperseHexChar + } + + a := reverseHexTable[p] + b := reverseHexTable[q] + if a > 0x0f { + return nil, hex.InvalidByteError(p) + } + if b > 0x0f { + return nil, hex.InvalidByteError(q) + } + dst[i] = (a << 4) | b + i++ + } + if (len(src)+1)%3 != 0 { + if len(src)%3 == 0 { + j -= 1 + } + // Check for invalid char before reporting bad length, + // since the invalid char (if present) is an earlier problem. + if reverseHexTable[src[j-1]] > 0x0f { + return nil, hex.InvalidByteError(src[j-1]) + } + return nil, hex.ErrLength + } + return dst[:i], nil +} + +// DecodeInterpersedHexFromASCIIString decodes an ASCII string of hex characters into a byte slice, +// where the hex characters are expected to be separated by a colon (':'). +// +// NOTE that this function returns an error in case the input string contains non-ASCII characters. +// +// Example: "01:02:03" -> { 0x01, 0x02, 0x03 } +func DecodeInterpersedHexFromASCIIString(src string) ([]byte, error) { + if len(src) == 0 { + return []byte{}, nil + } + if len(src) < 2 { + return nil, hex.ErrLength + } + + dst := make([]byte, (len(src)+1)/3) + i, j := 0, 1 + for ; j < len(src); j += 3 { // jump one extra byte for the separator (:) + p := src[j-1] + q := src[j] + if j+1 < len(src) && src[j+1] != ':' { + return nil, errUnexpectedIntersperseHexChar + } + + a := reverseHexTable[p] + b := reverseHexTable[q] + if a > 0x0f { + return nil, hex.InvalidByteError(p) + } + if b > 0x0f { + return nil, hex.InvalidByteError(q) + } + dst[i] = (a << 4) | b + i++ + } + if (len(src)+1)%3 != 0 { + if len(src)%3 == 0 { + j -= 1 + } + // Check for invalid char before reporting bad length, + // since the invalid char (if present) is an earlier problem. + if reverseHexTable[src[j-1]] > 0x0f { + return nil, hex.InvalidByteError(src[j-1]) + } + return nil, hex.ErrLength + } + return dst[:i], nil +} + +var ( + errUnexpectedIntersperseHexChar = errors.New("unexpected character in interspersed hex string") +) + +const ( + hextable = "0123456789abcdef" + reverseHexTable = "" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" + + "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +) diff --git a/p2p/transport/webrtc/internal/encoding/hex_test.go b/p2p/transport/webrtc/internal/encoding/hex_test.go new file mode 100644 index 0000000000..71f83647c1 --- /dev/null +++ b/p2p/transport/webrtc/internal/encoding/hex_test.go @@ -0,0 +1,126 @@ +package encoding + +import ( + "encoding/hex" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncodeInterspersedHex(t *testing.T) { + b, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + require.NoError(t, err) + require.Equal(t, "ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad", EncodeInterspersedHex(b)) +} + +func TestEncodeInterspersedHexToBuilder(t *testing.T) { + b, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + require.NoError(t, err) + var builder strings.Builder + EncodeInterspersedHexToBuilder(b, &builder) + require.Equal(t, "ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad", builder.String()) +} + +func TestDecodeInterpersedHexStringLowerCase(t *testing.T) { + b, err := DecodeInterpersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") + require.NoError(t, err) + require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) +} + +func TestDecodeInterpersedHexStringMixedCase(t *testing.T) { + b, err := DecodeInterpersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") + require.NoError(t, err) + require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) +} + +func TestDecodeInterpersedHexStringOneByte(t *testing.T) { + b, err := DecodeInterpersedHexFromASCIIString("ba") + require.NoError(t, err) + require.Equal(t, "ba", hex.EncodeToString(b)) +} + +func TestDecodeInterpersedHexBytesLowerCase(t *testing.T) { + b, err := DecodeInterspersedHex([]byte("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad")) + require.NoError(t, err) + require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) +} + +func TestDecodeInterpersedHexBytesMixedCase(t *testing.T) { + b, err := DecodeInterspersedHex([]byte("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad")) + require.NoError(t, err) + require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) +} + +func TestDecodeInterpersedHexBytesOneByte(t *testing.T) { + b, err := DecodeInterspersedHex([]byte("ba")) + require.NoError(t, err) + require.Equal(t, "ba", hex.EncodeToString(b)) +} + +func TestEncodeInterperseHexNilSlice(t *testing.T) { + require.Equal(t, "", EncodeInterspersedHex(nil)) + require.Equal(t, "", EncodeInterspersedHex([]byte{})) +} + +func TestDecodeInterspersedHexNilSlice(t *testing.T) { + for _, v := range [][]byte{nil, {}} { + b, err := DecodeInterspersedHex(v) + require.NoError(t, err) + require.Equal(t, []byte{}, b) + } +} + +func TestDecodeInterpersedHexFromASCIIStringEmpty(t *testing.T) { + b, err := DecodeInterpersedHexFromASCIIString("") + require.NoError(t, err) + require.Equal(t, []byte{}, b) +} + +func TestDecodeInterpersedHexInvalid(t *testing.T) { + for _, v := range []string{"0", "0000", "000"} { + _, err := DecodeInterspersedHex([]byte(v)) + require.Error(t, err) + } +} + +func TestDecodeInterpersedHexValid(t *testing.T) { + b, err := DecodeInterspersedHex([]byte("00")) + require.NoError(t, err) + require.Equal(t, []byte{0}, b) +} + +func TestDecodeInterpersedHexFromASCIIStringInvalid(t *testing.T) { + for _, v := range []string{"0", "0000", "000"} { + _, err := DecodeInterpersedHexFromASCIIString(v) + require.Error(t, err) + } +} + +func TestDecodeInterpersedHexFromASCIIStringValid(t *testing.T) { + b, err := DecodeInterpersedHexFromASCIIString("00") + require.NoError(t, err) + require.Equal(t, []byte{0}, b) +} + +func FuzzInterpersedHex(f *testing.F) { + f.Fuzz(func(t *testing.T, b []byte) { + decoded, err := DecodeInterspersedHex(b) + if err != nil { + return + } + encoded := EncodeInterspersedHex(decoded) + require.Equal(t, strings.ToLower(string(b)), encoded) + }) +} + +func FuzzInterspersedHexASCII(f *testing.F) { + f.Fuzz(func(t *testing.T, s string) { + decoded, err := DecodeInterpersedHexFromASCIIString(s) + if err != nil { + return + } + encoded := EncodeInterspersedHex(decoded) + require.Equal(t, strings.ToLower(s), encoded) + }) +} diff --git a/p2p/transport/webrtc/internal/fingerprint.go b/p2p/transport/webrtc/internal/fingerprint.go new file mode 100644 index 0000000000..bd8305ef14 --- /dev/null +++ b/p2p/transport/webrtc/internal/fingerprint.go @@ -0,0 +1,26 @@ +package internal + +import ( + "crypto" + "crypto/x509" + "errors" +) + +// Fingerprint is forked from pion to avoid bytes to string alloc, +// and to avoid the entire hex interspersing when we do not need it anyway + +var ( + errHashUnavailable = errors.New("fingerprint: hash algorithm is not linked into the binary") +) + +// Fingerprint creates a fingerprint for a certificate using the specified hash algorithm +func Fingerprint(cert *x509.Certificate, algo crypto.Hash) ([]byte, error) { + if !algo.Available() { + return nil, errHashUnavailable + } + h := algo.New() + // Hash.Writer is specified to be never returning an error. + // https://golang.org/pkg/hash/#Hash + h.Write(cert.Raw) + return h.Sum(nil), nil +} diff --git a/p2p/transport/webrtc/internal/sdp.go b/p2p/transport/webrtc/internal/sdp.go new file mode 100644 index 0000000000..9372ba97e4 --- /dev/null +++ b/p2p/transport/webrtc/internal/sdp.go @@ -0,0 +1,144 @@ +package internal + +import ( + "crypto" + "fmt" + "net" + "strings" + + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" + multihash "github.com/multiformats/go-multihash" +) + +// clientSDP describes an SDP format string which can be used +// to infer a client's SDP offer from the incoming STUN message. +// The fingerprint used to render a client SDP is arbitrary since +// it fingerprint verification is disabled in favour of a noise +// handshake. The max message size is fixed to 16384 bytes. +const clientSDP = `v=0 +o=- 0 0 IN %[1]s %[2]s +s=- +c=IN %[1]s %[2]s +t=0 0 + +m=application %[3]d UDP/DTLS/SCTP webrtc-datachannel +a=mid:0 +a=ice-options:ice2 +a=ice-ufrag:%[4]s +a=ice-pwd:%[4]s +a=fingerprint:sha-256 ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad +a=setup:actpass +a=sctp-port:5000 +a=max-message-size:16384 +` + +func RenderClientSDP(addr *net.UDPAddr, ufrag string) string { + ipVersion := "IP4" + if addr.IP.To4() == nil { + ipVersion = "IP6" + } + return fmt.Sprintf( + clientSDP, + ipVersion, + addr.IP, + addr.Port, + ufrag, + ) +} + +// serverSDP defines an SDP format string used by a dialer +// to infer the SDP answer of a server based on the provided +// multiaddr, and the locally set ICE credentials. The max +// message size is fixed to 16384 bytes. +const serverSDP = `v=0 +o=- 0 0 IN %[1]s %[2]s +s=- +t=0 0 +a=ice-lite +m=application %[3]d UDP/DTLS/SCTP webrtc-datachannel +c=IN %[1]s %[2]s +a=mid:0 +a=ice-options:ice2 +a=ice-ufrag:%[4]s +a=ice-pwd:%[4]s +a=fingerprint:%[5]s + +a=setup:passive +a=sctp-port:5000 +a=max-message-size:16384 +a=candidate:1 1 UDP 1 %[2]s %[3]d typ host +a=end-of-candidates +` + +func RenderServerSDP(addr *net.UDPAddr, ufrag string, fingerprint multihash.DecodedMultihash) (string, error) { + ipVersion := "IP4" + if addr.IP.To4() == nil { + ipVersion = "IP6" + } + + sdpString, err := getSupportedSDPString(fingerprint.Code) + if err != nil { + return "", err + } + + var builder strings.Builder + builder.Grow(len(fingerprint.Digest)*3 + 8) + builder.WriteString(sdpString) + builder.WriteByte(' ') + encoding.EncodeInterspersedHexToBuilder(fingerprint.Digest, &builder) + fp := builder.String() + + return fmt.Sprintf( + serverSDP, + ipVersion, + addr.IP, + addr.Port, + ufrag, + fp, + ), nil +} + +// GetSupportedSDPHash converts a multihash code to the +// corresponding crypto.Hash for supported protocols. If a +// crypto.Hash cannot be found, it returns `(0, false)` +func GetSupportedSDPHash(code uint64) (crypto.Hash, bool) { + switch code { + case multihash.MD5: + return crypto.MD5, true + case multihash.SHA1: + return crypto.SHA1, true + case multihash.SHA3_224: + return crypto.SHA3_224, true + case multihash.SHA2_256: + return crypto.SHA256, true + case multihash.SHA3_384: + return crypto.SHA3_384, true + case multihash.SHA2_512: + return crypto.SHA512, true + default: + return 0, false + } +} + +// getSupportedSDPString converts a multihash code +// to a string format recognised by pion for fingerprint +// algorithms +func getSupportedSDPString(code uint64) (string, error) { + // values based on (cryto.Hash).String() + switch code { + case multihash.MD5: + return "md5", nil + case multihash.SHA1: + return "sha-1", nil + case multihash.SHA3_224: + return "sha3-224", nil + case multihash.SHA2_256: + return "sha-256", nil + case multihash.SHA3_384: + return "sha3-384", nil + case multihash.SHA2_512: + return "sha-512", nil + default: + return "", fmt.Errorf("unsupported hash code (%d)", code) + } +} diff --git a/p2p/transport/webrtc/internal/util.go b/p2p/transport/webrtc/internal/util.go new file mode 100644 index 0000000000..2bde1988e8 --- /dev/null +++ b/p2p/transport/webrtc/internal/util.go @@ -0,0 +1,34 @@ +package internal + +import ( + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" + ma "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multibase" + mh "github.com/multiformats/go-multihash" + + "github.com/pion/webrtc/v3" +) + +func DecodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { + remoteFingerprintMultibase, err := maddr.ValueForProtocol(ma.P_CERTHASH) + if err != nil { + return nil, err + } + _, data, err := multibase.Decode(remoteFingerprintMultibase) + if err != nil { + return nil, err + } + return mh.Decode(data) +} + +func EncodeDTLSFingerprint(fp webrtc.DTLSFingerprint) (string, error) { + digest, err := encoding.DecodeInterpersedHexFromASCIIString(fp.Value) + if err != nil { + return "", err + } + encoded, err := mh.Encode(digest, mh.SHA2_256) + if err != nil { + return "", err + } + return multibase.Encode(multibase.Base64url, encoded) +} diff --git a/p2p/transport/webrtc/internal/util_test.go b/p2p/transport/webrtc/internal/util_test.go new file mode 100644 index 0000000000..20fc9c7874 --- /dev/null +++ b/p2p/transport/webrtc/internal/util_test.go @@ -0,0 +1,101 @@ +package internal + +import ( + "encoding/hex" + "net" + "testing" + + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +const expectedServerSDP = `v=0 +o=- 0 0 IN IP4 0.0.0.0 +s=- +t=0 0 +a=ice-lite +m=application 37826 UDP/DTLS/SCTP webrtc-datachannel +c=IN IP4 0.0.0.0 +a=mid:0 +a=ice-options:ice2 +a=ice-ufrag:d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581 +a=ice-pwd:d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581 +a=fingerprint:sha-256 ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad + +a=setup:passive +a=sctp-port:5000 +a=max-message-size:16384 +a=candidate:1 1 UDP 1 0.0.0.0 37826 typ host +a=end-of-candidates +` + +func TestRenderServerSDP(t *testing.T) { + encoded, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + require.NoError(t, err) + + testMultihash := multihash.DecodedMultihash{ + Code: multihash.SHA2_256, + Name: multihash.Codes[multihash.SHA2_256], + Digest: encoded, + Length: len(encoded), + } + addr := &net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: 37826} + ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" + fingerprint := testMultihash + + sdp, err := RenderServerSDP(addr, ufrag, fingerprint) + require.NoError(t, err) + require.Equal(t, expectedServerSDP, sdp) +} + +const expectedClientSDP = `v=0 +o=- 0 0 IN IP4 0.0.0.0 +s=- +c=IN IP4 0.0.0.0 +t=0 0 + +m=application 37826 UDP/DTLS/SCTP webrtc-datachannel +a=mid:0 +a=ice-options:ice2 +a=ice-ufrag:d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581 +a=ice-pwd:d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581 +a=fingerprint:sha-256 ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad +a=setup:actpass +a=sctp-port:5000 +a=max-message-size:16384 +` + +func TestRenderClientSDP(t *testing.T) { + addr := &net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: 37826} + ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" + sdp := RenderClientSDP(addr, ufrag) + require.Equal(t, expectedClientSDP, sdp) +} + +func BenchmarkRenderClientSDP(b *testing.B) { + addr := &net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: 37826} + ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" + + for i := 0; i < b.N; i++ { + RenderClientSDP(addr, ufrag) + } +} + +func BenchmarkRenderServerSDP(b *testing.B) { + encoded, _ := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + + testMultihash := multihash.DecodedMultihash{ + Code: multihash.SHA2_256, + Name: multihash.Codes[multihash.SHA2_256], + Digest: encoded, + Length: len(encoded), + } + addr := &net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: 37826} + ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" + fingerprint := testMultihash + + for i := 0; i < b.N; i++ { + RenderServerSDP(addr, ufrag, fingerprint) + } + +} diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go new file mode 100644 index 0000000000..be42f820fa --- /dev/null +++ b/p2p/transport/webrtc/listener.go @@ -0,0 +1,332 @@ +package libp2pwebrtc + +import ( + "context" + "crypto" + "encoding/hex" + "fmt" + "net" + "os" + "strings" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/udpmux" + + tpt "github.com/libp2p/go-libp2p/core/transport" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/multiformats/go-multibase" + "github.com/multiformats/go-multihash" + + "github.com/pion/ice/v2" + pionlogger "github.com/pion/logging" + "github.com/pion/webrtc/v3" +) + +var _ tpt.Listener = &listener{} + +const ( + candidateSetupTimeout = 20 * time.Second + DefaultMaxInFlightConnections = 10 +) + +type candidateAddr struct { + ufrag string + raddr *net.UDPAddr +} + +type listener struct { + transport *WebRTCTransport + + mux ice.UDPMux + + config webrtc.Configuration + localFingerprint webrtc.DTLSFingerprint + localFingerprintMultibase string + + localAddr net.Addr + localMultiaddr ma.Multiaddr + + // buffered incoming connections + acceptQueue chan tpt.CapableConn + + // Accepting a connection requires instantiating a peerconnection + // and a noise connection which is expensive. We therefore limit + // the number of in-flight connection requests. A connection + // is considered to be in flight from the instant it is handled + // until it is dequeued by a call to Accept, or errors out in some + // way. + inFlightInputQueue chan struct{} + + // used to control the lifecycle of the listener + ctx context.Context + cancel context.CancelFunc +} + +func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.PacketConn, config webrtc.Configuration) (*listener, error) { + localFingerprints, err := config.Certificates[0].GetFingerprints() + if err != nil { + return nil, err + } + + localMh, err := hex.DecodeString(strings.ReplaceAll(localFingerprints[0].Value, ":", "")) + if err != nil { + return nil, err + } + localMhBuf, err := multihash.Encode(localMh, multihash.SHA2_256) + if err != nil { + return nil, err + } + localFpMultibase, err := multibase.Encode(multibase.Base64url, localMhBuf) + if err != nil { + return nil, err + } + + inFlightQueueCh := make(chan struct{}, transport.maxInFlightConnections) + for i := uint32(0); i < transport.maxInFlightConnections; i++ { + inFlightQueueCh <- struct{}{} + } + + l := &listener{ + transport: transport, + config: config, + localFingerprint: localFingerprints[0], + localFingerprintMultibase: localFpMultibase, + localMultiaddr: laddr, + localAddr: socket.LocalAddr(), + acceptQueue: make(chan tpt.CapableConn), + inFlightInputQueue: inFlightQueueCh, + } + + l.ctx, l.cancel = context.WithCancel(context.Background()) + l.mux = udpmux.NewUDPMux(socket, func(ufrag string, addr net.Addr) bool { + select { + case <-inFlightQueueCh: + // we have space to accept, Yihaa + default: + log.Debug("candidate chan full, dropping incoming candidate") + return false + } + + go func() { + defer func() { + // free this spot once again + inFlightQueueCh <- struct{}{} + }() + + ctx, cancel := context.WithTimeout(l.ctx, candidateSetupTimeout) + defer cancel() + + candidateAddr := candidateAddr{ufrag: ufrag, raddr: addr.(*net.UDPAddr)} + conn, err := l.handleCandidate(ctx, &candidateAddr) + if err != nil { + log.Debugf("could not accept connection: %s: %v", ufrag, err) + return + } + + select { + case <-ctx.Done(): + log.Warn("could not push connection: ctx done") + conn.Close() + + case l.acceptQueue <- conn: + // block until the connection is accepted, + // or until we are done, this effectively blocks our in flight from continuing to progress + } + }() + + return true + }) + + return l, err +} + +func (l *listener) handleCandidate(ctx context.Context, addr *candidateAddr) (tpt.CapableConn, error) { + remoteMultiaddr, err := manet.FromNetAddr(addr.raddr) + if err != nil { + return nil, err + } + scope, err := l.transport.rcmgr.OpenConnection(network.DirInbound, false, remoteMultiaddr) + if err != nil { + return nil, err + } + conn, err := l.setupConnection(ctx, scope, remoteMultiaddr, addr) + if err != nil { + scope.Done() + return nil, err + } + return conn, nil +} + +func (l *listener) setupConnection( + ctx context.Context, scope network.ConnManagementScope, + remoteMultiaddr ma.Multiaddr, addr *candidateAddr, +) (tConn tpt.CapableConn, err error) { + var pc *webrtc.PeerConnection + defer func() { + if err != nil { + if pc != nil { + _ = pc.Close() + } + if tConn != nil { + _ = tConn.Close() + } + } + }() + + settingEngine := webrtc.SettingEngine{} + + // suppress pion logs + loggerFactory := pionlogger.NewDefaultLoggerFactory() + loggerFactory.DefaultLogLevel = pionlogger.LogLevelWarn + settingEngine.LoggerFactory = loggerFactory + + settingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleServer) + settingEngine.SetICECredentials(addr.ufrag, addr.ufrag) + settingEngine.SetLite(true) + settingEngine.SetICEUDPMux(l.mux) + settingEngine.SetIncludeLoopbackCandidate(true) + settingEngine.DisableCertificateFingerprintVerification(true) + settingEngine.SetICETimeouts( + l.transport.peerConnectionTimeouts.Disconnect, + l.transport.peerConnectionTimeouts.Failed, + l.transport.peerConnectionTimeouts.Keepalive, + ) + settingEngine.DetachDataChannels() + + api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) + + pc, err = api.NewPeerConnection(l.config) + if err != nil { + return nil, err + } + + negotiated, id := handshakeChannelNegotiated, handshakeChannelID + rawDatachannel, err := pc.CreateDataChannel("", &webrtc.DataChannelInit{ + Negotiated: &negotiated, + ID: &id, + }) + if err != nil { + return nil, err + } + + errC := awaitPeerConnectionOpen(addr.ufrag, pc) + // we infer the client sdp from the incoming STUN connectivity check + // by setting the ice-ufrag equal to the incoming check. + clientSdpString := internal.RenderClientSDP(addr.raddr, addr.ufrag) + clientSdp := webrtc.SessionDescription{SDP: clientSdpString, Type: webrtc.SDPTypeOffer} + pc.SetRemoteDescription(clientSdp) + + answer, err := pc.CreateAnswer(nil) + if err != nil { + return nil, err + } + + err = pc.SetLocalDescription(answer) + if err != nil { + return nil, err + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case err := <-errC: + if err != nil { + return nil, fmt.Errorf("peer connection error: %w", err) + } + + } + + rwc, err := getDetachedChannel(ctx, rawDatachannel) + if err != nil { + return nil, err + } + + handshakeChannel := newStream(nil, rawDatachannel, rwc, l.localAddr, addr.raddr) + // The connection is instantiated before performing the Noise handshake. This is + // to handle the case where the remote is faster and attempts to initiate a stream + // before the ondatachannel callback can be set. + conn, err := newConnection( + network.DirInbound, + pc, + l.transport, + scope, + l.transport.localPeerId, + l.localMultiaddr, + "", // remotePeer + nil, // remoteKey + remoteMultiaddr, + ) + if err != nil { + return nil, err + } + + // we do not yet know A's peer ID so accept any inbound + secureConn, err := l.transport.noiseHandshake(ctx, pc, handshakeChannel, "", crypto.SHA256, true) + if err != nil { + return nil, err + } + + // earliest point where we know the remote's peerID + err = scope.SetPeer(secureConn.RemotePeer()) + if err != nil { + return nil, err + } + + conn.setRemotePeer(secureConn.RemotePeer()) + conn.setRemotePublicKey(secureConn.RemotePublicKey()) + + return conn, err +} + +func (l *listener) Accept() (tpt.CapableConn, error) { + select { + case <-l.ctx.Done(): + return nil, os.ErrClosed + case conn := <-l.acceptQueue: + return conn, nil + } +} + +func (l *listener) Close() error { + select { + case <-l.ctx.Done(): + default: + l.cancel() + } + return nil +} + +func (l *listener) Addr() net.Addr { + return l.localAddr +} + +func (l *listener) Multiaddr() ma.Multiaddr { + return l.localMultiaddr +} + +func awaitPeerConnectionOpen(ufrag string, pc *webrtc.PeerConnection) <-chan error { + errC := make(chan error, 1) + var once sync.Once + pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + switch state { + case webrtc.PeerConnectionStateConnected: + once.Do(func() { close(errC) }) + case webrtc.PeerConnectionStateFailed: + once.Do(func() { + errC <- fmt.Errorf("peerconnection failed: %s", ufrag) + close(errC) + }) + case webrtc.PeerConnectionStateDisconnected: + // the connection can move to a disconnected state and back to a connected state without ICE renegotiation. + // This could happen when underlying UDP packets are lost, and therefore the connection moves to the disconnected state. + // If the connection then receives packets on the connection, it can move back to the connected state. + // If no packets are received until the failed timeout is triggered, the connection moves to the failed state. + log.Warn("peerconnection disconnected") + } + }) + return errC +} diff --git a/p2p/transport/webrtc/message.go b/p2p/transport/webrtc/message.go new file mode 100644 index 0000000000..8b57fc8078 --- /dev/null +++ b/p2p/transport/webrtc/message.go @@ -0,0 +1,3 @@ +package libp2pwebrtc + +//go:generate protoc --go_out=. --go_opt=Mpb/message.proto=./pb pb/message.proto diff --git a/p2p/transport/webrtc/pb/generate.go b/p2p/transport/webrtc/pb/generate.go new file mode 100644 index 0000000000..5785a95251 --- /dev/null +++ b/p2p/transport/webrtc/pb/generate.go @@ -0,0 +1,3 @@ +package pb + +//go:generate protoc --go_out=. --go_opt=paths=source_relative -I . message.proto diff --git a/p2p/transport/webrtc/pb/message.pb.go b/p2p/transport/webrtc/pb/message.pb.go new file mode 100644 index 0000000000..fffc025f7f --- /dev/null +++ b/p2p/transport/webrtc/pb/message.pb.go @@ -0,0 +1,224 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc v3.21.12 +// source: message.proto + +package pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Message_Flag int32 + +const ( + // The sender will no longer send messages on the stream. + Message_FIN Message_Flag = 0 + // The sender will no longer read messages on the stream. Incoming data is + // being discarded on receipt. + Message_STOP_SENDING Message_Flag = 1 + // The sender abruptly terminates the sending part of the stream. The + // receiver can discard any data that it already received on that stream. + Message_RESET Message_Flag = 2 +) + +// Enum value maps for Message_Flag. +var ( + Message_Flag_name = map[int32]string{ + 0: "FIN", + 1: "STOP_SENDING", + 2: "RESET", + } + Message_Flag_value = map[string]int32{ + "FIN": 0, + "STOP_SENDING": 1, + "RESET": 2, + } +) + +func (x Message_Flag) Enum() *Message_Flag { + p := new(Message_Flag) + *p = x + return p +} + +func (x Message_Flag) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Message_Flag) Descriptor() protoreflect.EnumDescriptor { + return file_message_proto_enumTypes[0].Descriptor() +} + +func (Message_Flag) Type() protoreflect.EnumType { + return &file_message_proto_enumTypes[0] +} + +func (x Message_Flag) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *Message_Flag) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = Message_Flag(num) + return nil +} + +// Deprecated: Use Message_Flag.Descriptor instead. +func (Message_Flag) EnumDescriptor() ([]byte, []int) { + return file_message_proto_rawDescGZIP(), []int{0, 0} +} + +type Message struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Flag *Message_Flag `protobuf:"varint,1,opt,name=flag,enum=Message_Flag" json:"flag,omitempty"` + Message []byte `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` +} + +func (x *Message) Reset() { + *x = Message{} + if protoimpl.UnsafeEnabled { + mi := &file_message_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_message_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_message_proto_rawDescGZIP(), []int{0} +} + +func (x *Message) GetFlag() Message_Flag { + if x != nil && x.Flag != nil { + return *x.Flag + } + return Message_FIN +} + +func (x *Message) GetMessage() []byte { + if x != nil { + return x.Message + } + return nil +} + +var File_message_proto protoreflect.FileDescriptor + +var file_message_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x74, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x04, 0x66, 0x6c, + 0x61, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x2e, 0x46, 0x6c, 0x61, 0x67, 0x52, 0x04, 0x66, 0x6c, 0x61, 0x67, 0x12, 0x18, 0x0a, + 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x0a, 0x04, 0x46, 0x6c, 0x61, 0x67, 0x12, + 0x07, 0x0a, 0x03, 0x46, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x4f, 0x50, + 0x5f, 0x53, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, + 0x53, 0x45, 0x54, 0x10, 0x02, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x62, 0x70, 0x32, 0x70, 0x2f, 0x67, 0x6f, 0x2d, 0x6c, 0x69, + 0x62, 0x70, 0x32, 0x70, 0x2f, 0x70, 0x32, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, + 0x72, 0x74, 0x2f, 0x77, 0x65, 0x62, 0x72, 0x74, 0x63, 0x2f, 0x70, 0x62, +} + +var ( + file_message_proto_rawDescOnce sync.Once + file_message_proto_rawDescData = file_message_proto_rawDesc +) + +func file_message_proto_rawDescGZIP() []byte { + file_message_proto_rawDescOnce.Do(func() { + file_message_proto_rawDescData = protoimpl.X.CompressGZIP(file_message_proto_rawDescData) + }) + return file_message_proto_rawDescData +} + +var file_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_message_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_message_proto_goTypes = []interface{}{ + (Message_Flag)(0), // 0: Message.Flag + (*Message)(nil), // 1: Message +} +var file_message_proto_depIdxs = []int32{ + 0, // 0: Message.flag:type_name -> Message.Flag + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_message_proto_init() } +func file_message_proto_init() { + if File_message_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Message); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_message_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_message_proto_goTypes, + DependencyIndexes: file_message_proto_depIdxs, + EnumInfos: file_message_proto_enumTypes, + MessageInfos: file_message_proto_msgTypes, + }.Build() + File_message_proto = out.File + file_message_proto_rawDesc = nil + file_message_proto_goTypes = nil + file_message_proto_depIdxs = nil +} diff --git a/p2p/transport/webrtc/pb/message.proto b/p2p/transport/webrtc/pb/message.proto new file mode 100644 index 0000000000..d6b1957beb --- /dev/null +++ b/p2p/transport/webrtc/pb/message.proto @@ -0,0 +1,20 @@ +syntax = "proto2"; + +option go_package = "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb"; + +message Message { + enum Flag { + // The sender will no longer send messages on the stream. + FIN = 0; + // The sender will no longer read messages on the stream. Incoming data is + // being discarded on receipt. + STOP_SENDING = 1; + // The sender abruptly terminates the sending part of the stream. The + // receiver can discard any data that it already received on that stream. + RESET = 2; + } + + optional Flag flag=1; + + optional bytes message = 2; +} diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go new file mode 100644 index 0000000000..829a042a89 --- /dev/null +++ b/p2p/transport/webrtc/stream.go @@ -0,0 +1,198 @@ +package libp2pwebrtc + +import ( + "context" + "net" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/network" + pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-msgio/pbio" + "github.com/pion/datachannel" + "github.com/pion/webrtc/v3" +) + +var _ network.MuxedStream = &webRTCStream{} + +const ( + // maxMessageSize is limited to 16384 bytes in the SDP. + maxMessageSize = 16384 + // Pion SCTP association has an internal receive buffer of 1MB (roughly, 1MB per connection). + // We can change this value in the SettingEngine before creating the peerconnection. + // https://github.com/pion/webrtc/blob/v3.1.49/sctptransport.go#L341 + maxBufferedAmount = 2 * maxMessageSize + // bufferedAmountLowThreshold and maxBufferedAmount are bound + // to a stream but congestion control is done on the whole + // SCTP association. This means that a single stream can monopolize + // the complete congestion control window (cwnd) if it does not + // read stream data and it's remote continues to send. We can + // add messages to the send buffer once there is space for 1 full + // sized message. + bufferedAmountLowThreshold = maxBufferedAmount / 2 + + // Proto overhead assumption is 5 bytes + protoOverhead = 5 + // Varint overhead is assumed to be 2 bytes. This is safe since + // 1. This is only used and when writing message, and + // 2. We only send messages in chunks of `maxMessageSize - varintOverhead` + // which includes the data and the protobuf header. Since `maxMessageSize` + // is less than or equal to 2 ^ 14, the varint will not be more than + // 2 bytes in length. + varintOverhead = 2 +) + +// Package pion detached data channel into a net.Conn +// and then a network.MuxedStream +type webRTCStream struct { + reader pbio.Reader + // pbio.Reader is not thread safe, + // and while our Read is not promised to be thread safe, + // we ourselves internally read from multiple routines... + readerMux sync.Mutex + // this buffer is limited up to a single message. Reason we need it + // is because a reader might read a message midway, and so we need a + // wait to buffer that for as long as the remaining part is not (yet) read + readBuffer []byte + + writer pbio.Writer + // public write API is not promised to be thread safe, however we also write from + // read functionality due to our spec limiations, so (e.g. 1 channel for state and data) + // and thus we do need to protect the writer + writerMux sync.Mutex + + writerDeadline time.Time + writerDeadlineMux sync.Mutex + + writerDeadlineUpdated chan struct{} + writeAvailable chan struct{} + + readLoopOnce sync.Once + + stateHandler webRTCStreamState + + conn *connection + id uint16 + dataChannel *datachannel.DataChannel + + laddr net.Addr + raddr net.Addr + + ctx context.Context + cancel context.CancelFunc + + closeOnce sync.Once +} + +func newStream( + connection *connection, + channel *webrtc.DataChannel, + rwc datachannel.ReadWriteCloser, + laddr, raddr net.Addr, +) *webRTCStream { + ctx, cancel := context.WithCancel(context.Background()) + + result := &webRTCStream{ + reader: pbio.NewDelimitedReader(rwc, maxMessageSize), + writer: pbio.NewDelimitedWriter(rwc), + + writerDeadlineUpdated: make(chan struct{}, 1), + writeAvailable: make(chan struct{}, 1), + + conn: connection, + id: *channel.ID(), + dataChannel: rwc.(*datachannel.DataChannel), + + laddr: laddr, + raddr: raddr, + + ctx: ctx, + cancel: cancel, + } + + channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) + channel.OnBufferedAmountLow(func() { + select { + case result.writeAvailable <- struct{}{}: + default: + } + }) + + return result +} + +func (s *webRTCStream) Close() error { + return s.close(false, true) +} + +func (s *webRTCStream) Reset() error { + return s.close(true, true) +} + +func (s *webRTCStream) LocalAddr() net.Addr { + return s.laddr +} + +func (s *webRTCStream) RemoteAddr() net.Addr { + return s.raddr +} + +func (s *webRTCStream) SetDeadline(t time.Time) error { + return s.SetWriteDeadline(t) +} + +func (s *webRTCStream) processIncomingFlag(flag pb.Message_Flag) { + if s.isClosed() { + return + } + state, reset := s.stateHandler.HandleInboundFlag(flag) + if state == stateClosed { + log.Debug("closing: after handle inbound flag") + s.close(reset, true) + } +} + +// this is used to force reset a stream +func (s *webRTCStream) close(isReset bool, notifyConnection bool) error { + if s.isClosed() { + return nil + } + + var err error + s.closeOnce.Do(func() { + log.Debugf("closing stream %d: reset: %t, notify: %t", s.id, isReset, notifyConnection) + if isReset { + s.stateHandler.Reset() + // write the RESET message. The error is explicitly ignored + // because we do not know if the remote is still connected + s.writeMessageToWriter(&pb.Message{Flag: pb.Message_RESET.Enum()}) + } else { + s.stateHandler.Close() + // write a FIN message for standard stream closure + // we write directly to the underlying writer since we do not care about + // cancelled contexts or deadlines for this. + s.writeMessageToWriter(&pb.Message{Flag: pb.Message_FIN.Enum()}) + } + // close the context + s.cancel() + // force close reads + s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) // pion ignores zero times + // close the channel. We do not care about the error message in + // this case + err = s.dataChannel.Close() + if notifyConnection && s.conn != nil { + s.conn.removeStream(s.id) + } + }) + + return err +} + +func (s *webRTCStream) isClosed() bool { + select { + case <-s.ctx.Done(): + return true + default: + return false + } +} diff --git a/p2p/transport/webrtc/stream_error.go b/p2p/transport/webrtc/stream_error.go new file mode 100644 index 0000000000..b2546b8dec --- /dev/null +++ b/p2p/transport/webrtc/stream_error.go @@ -0,0 +1,25 @@ +package libp2pwebrtc + +import "net" + +type Error struct { + message string + temporary, timeout bool +} + +var _ net.Error = &Error{} + +func (e *Error) Error() string { + return e.message +} + +func (e *Error) Temporary() bool { + return e.temporary +} + +func (e *Error) Timeout() bool { + return e.timeout +} + +// ErrTimeout is used when an i/o deadline is reached +var ErrTimeout = &Error{message: "i/o deadline reached", timeout: true, temporary: true} diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go new file mode 100644 index 0000000000..852af3a22d --- /dev/null +++ b/p2p/transport/webrtc/stream_read.go @@ -0,0 +1,120 @@ +package libp2pwebrtc + +import ( + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/libp2p/go-libp2p/core/network" + pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" +) + +// Read from the underlying datachannel. This also +// process sctp control messages such as DCEP, which is +// handled internally by pion, and stream closure which +// is signaled by `Read` on the datachannel returning +// io.EOF. +func (s *webRTCStream) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + + var ( + readErr error + read int + ) + for read == 0 && readErr == nil { + if s.isClosed() { + if s.stateHandler.IsReset() { + return 0, network.ErrReset + } + return 0, io.EOF + } + read, readErr = s.readMessage(b) + } + if errors.Is(readErr, os.ErrDeadlineExceeded) { + return read, ErrTimeout + } + return read, readErr +} + +func (s *webRTCStream) readMessage(b []byte) (int, error) { + read := copy(b, s.readBuffer) + s.readBuffer = s.readBuffer[read:] + remaining := len(s.readBuffer) + + if remaining == 0 && !s.stateHandler.AllowRead() { + log.Debug("[2] stream has no more data to read") + if read != 0 { + return read, nil + } + return read, io.EOF + } + + if read > 0 { + return read, nil + } + + // read from datachannel + var msg pb.Message + err := s.readMessageFromDataChannel(&msg) + if err != nil { + // This case occurs when the remote node goes away + // without writing a FIN message + if errors.Is(err, io.EOF) { + // if the channel was properly closed, return EOF + if !s.stateHandler.AllowRead() && !s.stateHandler.IsReset() { + return 0, io.EOF + } + return 0, network.ErrReset + } + + if errors.Is(err, os.ErrDeadlineExceeded) && s.stateHandler.IsReset() { + return 0, network.ErrReset + } + return read, err + } + + // append incoming data to read readBuffer + if s.stateHandler.AllowRead() && msg.Message != nil { + s.readBuffer = append(s.readBuffer, msg.GetMessage()...) + } + + // process any flags on the message + if msg.Flag != nil { + s.processIncomingFlag(msg.GetFlag()) + } + + return read, nil +} + +func (s *webRTCStream) readMessageFromDataChannel(msg *pb.Message) error { + s.readerMux.Lock() + defer s.readerMux.Unlock() + return s.reader.ReadMsg(msg) +} + +func (s *webRTCStream) SetReadDeadline(t time.Time) error { + return s.dataChannel.SetReadDeadline(t) +} + +func (s *webRTCStream) CloseRead() error { + if s.isClosed() { + return nil + } + var err error + s.closeOnce.Do(func() { + err = s.writeMessageToWriter(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) + if err != nil { + log.Debug("could not write STOP_SENDING message") + err = fmt.Errorf("could not close stream for reading: %w", err) + return + } + if s.stateHandler.CloseRead() == stateClosed { + s.close(false, true) + } + }) + return err +} diff --git a/p2p/transport/webrtc/stream_state.go b/p2p/transport/webrtc/stream_state.go new file mode 100644 index 0000000000..a7b05ec937 --- /dev/null +++ b/p2p/transport/webrtc/stream_state.go @@ -0,0 +1,136 @@ +package libp2pwebrtc + +import ( + "sync" + + pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" +) + +type channelState uint8 + +const ( + stateOpen channelState = iota + stateReadClosed + stateWriteClosed + stateClosed +) + +type webRTCStreamState struct { + mu sync.RWMutex + state channelState + reset bool +} + +func (ss *webRTCStreamState) HandleInboundFlag(flag pb.Message_Flag) (channelState, bool) { + ss.mu.Lock() + defer ss.mu.Unlock() + + if ss.state == stateClosed { + return ss.state, ss.reset + } + + switch flag { + case pb.Message_FIN: + ss.closeReadInner() + + case pb.Message_STOP_SENDING: + ss.closeWriteInner() + + case pb.Message_RESET: + ss.closeInner(true) + default: + // ignore values that are invalid for flags + } + + return ss.state, ss.reset +} + +func (ss *webRTCStreamState) State() channelState { + ss.mu.RLock() + defer ss.mu.RUnlock() + return ss.state +} + +func (ss *webRTCStreamState) AllowRead() bool { + ss.mu.RLock() + defer ss.mu.RUnlock() + return ss.state == stateOpen || ss.state == stateWriteClosed +} + +func (ss *webRTCStreamState) CloseRead() channelState { + ss.mu.Lock() + defer ss.mu.Unlock() + + if ss.state == stateClosed { + return ss.state + } + + ss.closeReadInner() + return ss.state +} + +func (ss *webRTCStreamState) closeReadInner() { + if ss.state == stateOpen { + ss.state = stateReadClosed + } else if ss.state == stateWriteClosed { + ss.closeInner(false) + } +} + +func (ss *webRTCStreamState) AllowWrite() bool { + ss.mu.RLock() + defer ss.mu.RUnlock() + return ss.state == stateOpen || ss.state == stateReadClosed +} + +func (ss *webRTCStreamState) CloseWrite() channelState { + ss.mu.Lock() + defer ss.mu.Unlock() + + if ss.state == stateClosed { + return ss.state + } + + ss.closeWriteInner() + return ss.state +} + +func (ss *webRTCStreamState) closeWriteInner() { + if ss.state == stateOpen { + ss.state = stateWriteClosed + } else if ss.state == stateReadClosed { + ss.closeInner(false) + } +} + +func (ss *webRTCStreamState) Closed() bool { + ss.mu.RLock() + defer ss.mu.RUnlock() + return ss.state == stateClosed +} +func (ss *webRTCStreamState) Close() { + ss.mu.Lock() + defer ss.mu.Unlock() + ss.state = stateClosed + ss.reset = false +} + +func (ss *webRTCStreamState) Reset() { + ss.mu.Lock() + defer ss.mu.Unlock() + ss.state = stateClosed + ss.reset = true +} + +func (ss *webRTCStreamState) IsReset() bool { + ss.mu.Lock() + defer ss.mu.Unlock() + return ss.reset +} + +func (ss *webRTCStreamState) closeInner(reset bool) { + if ss.state != stateClosed { + ss.state = stateClosed + ss.reset = reset + } +} diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go new file mode 100644 index 0000000000..ebc5791fe3 --- /dev/null +++ b/p2p/transport/webrtc/stream_write.go @@ -0,0 +1,182 @@ +package libp2pwebrtc + +import ( + "errors" + "fmt" + "io" + "math" + "os" + "time" + + "github.com/libp2p/go-libp2p/core/network" + pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "google.golang.org/protobuf/proto" +) + +func (s *webRTCStream) Write(b []byte) (int, error) { + if !s.stateHandler.AllowWrite() { + return 0, io.ErrClosedPipe + } + + // Check if there is any message on the wire. This is used for control + // messages only when the read side of the stream is closed + if s.stateHandler.State() == stateReadClosed { + s.readLoopOnce.Do(s.spawnControlMessageReader) + } + + const chunkSize = maxMessageSize - protoOverhead - varintOverhead + + var n int + + for len(b) > 0 { + end := len(b) + if chunkSize < end { + end = chunkSize + } + + err := s.writeMessage(&pb.Message{Message: b[:end]}) + n += end + b = b[end:] + if err != nil { + if errors.Is(err, os.ErrDeadlineExceeded) { + err = ErrTimeout + } + return n, err + } + } + + return n, nil +} + +// used for reading control messages while writing, in case the reader is closed, +// as to ensure we do still get control messages. This is important as according to the spec +// our data and control channels are intermixed on the same conn. +func (s *webRTCStream) spawnControlMessageReader() { + go func() { + // zero the read deadline, so read call only returns + // when the underlying datachannel closes or there is + // a message on the channel + s.dataChannel.SetReadDeadline(time.Time{}) + var msg pb.Message + for { + if s.stateHandler.Closed() { + return + } + err := s.readMessageFromDataChannel(&msg) + if err != nil { + if errors.Is(err, io.EOF) { + s.close(true, true) + } + return + } + if msg.Flag != nil { + state, reset := s.stateHandler.HandleInboundFlag(msg.GetFlag()) + if state == stateClosed { + log.Debug("closing: after handle inbound flag") + s.close(reset, true) + } + } + } + }() +} + +func (s *webRTCStream) writeMessage(msg *pb.Message) error { + var writeDeadlineTimer *time.Timer + defer func() { + if writeDeadlineTimer != nil { + writeDeadlineTimer.Stop() + } + }() + + for { + if !s.stateHandler.AllowWrite() { + return io.ErrClosedPipe + } + + writeDeadline, hasWriteDeadline := s.getWriteDeadline() + if !hasWriteDeadline { + // writeDeadline = time.Unix(999999999999999999, 0) + // Does this cause the timer to overflow ? https://cs.opensource.google/go/go/+/master:src/time/sleep.go;l=32?q=runtimeTimer&ss=go%2Fgo + // this could be causing an overflow above https://cs.opensource.google/go/go/+/master:src/time/time.go;l=1123?q=unixTim&ss=go%2Fgo + // Go adds 62135596800 to the seconds parameter of `time.Unix` + writeDeadline = time.Unix(math.MaxInt64-62135596801, 0) + } + if writeDeadlineTimer == nil { + writeDeadlineTimer = time.NewTimer(time.Until(writeDeadline)) + } else { + if !writeDeadlineTimer.Stop() { + <-writeDeadlineTimer.C + } + writeDeadlineTimer.Reset(time.Until(writeDeadline)) + } + + bufferedAmount := int(s.dataChannel.BufferedAmount()) + addedBuffer := bufferedAmount + varintOverhead + proto.Size(msg) + if addedBuffer > maxBufferedAmount { + select { + case <-writeDeadlineTimer.C: + return os.ErrDeadlineExceeded + case <-s.writeAvailable: + return s.writeMessageToWriter(msg) + case <-s.ctx.Done(): + if s.stateHandler.IsReset() { + return network.ErrReset + } + return io.ErrClosedPipe + case <-s.writerDeadlineUpdated: + } + } else { + return s.writeMessageToWriter(msg) + } + } +} + +func (s *webRTCStream) writeMessageToWriter(msg *pb.Message) error { + s.writerMux.Lock() + defer s.writerMux.Unlock() + return s.writer.WriteMsg(msg) +} + +func (s *webRTCStream) SetWriteDeadline(t time.Time) error { + s.writerDeadlineMux.Lock() + defer s.writerDeadlineMux.Unlock() + s.writerDeadline = t + select { + case s.writerDeadlineUpdated <- struct{}{}: + default: + } + return nil +} + +func (s *webRTCStream) getWriteDeadline() (time.Time, bool) { + s.writerDeadlineMux.Lock() + defer s.writerDeadlineMux.Unlock() + return s.writerDeadline, !s.writerDeadline.IsZero() +} + +func (s *webRTCStream) CloseWrite() error { + if s.isClosed() { + return nil + } + var err error + s.closeOnce.Do(func() { + err = s.writeMessage(&pb.Message{Flag: pb.Message_FIN.Enum()}) + if err != nil { + log.Debug("could not write FIN message") + err = fmt.Errorf("close stream for writing: %w", err) + return + } + // if successfully written, process the outgoing flag + state := s.stateHandler.CloseWrite() + // unblock and fail any ongoing writes + select { + case s.writeAvailable <- struct{}{}: + default: + } + // check if closure required + if state == stateClosed { + s.close(false, true) + } + }) + return err +} diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go new file mode 100644 index 0000000000..cbe8ba7548 --- /dev/null +++ b/p2p/transport/webrtc/transport.go @@ -0,0 +1,505 @@ +package libp2pwebrtc + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "errors" + "fmt" + "net" + "time" + + "github.com/libp2p/go-libp2p/core/connmgr" + ic "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/pnet" + "github.com/libp2p/go-libp2p/core/sec" + tpt "github.com/libp2p/go-libp2p/core/transport" + "github.com/libp2p/go-libp2p/p2p/security/noise" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" + + logging "github.com/ipfs/go-log/v2" + ma "github.com/multiformats/go-multiaddr" + mafmt "github.com/multiformats/go-multiaddr-fmt" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/multiformats/go-multihash" + pionlogger "github.com/pion/logging" + + "github.com/pion/webrtc/v3" +) + +var log = logging.Logger("webrtc-transport") + +var dialMatcher = mafmt.And(mafmt.UDP, mafmt.Base(ma.P_P2P_WEBRTC_DIRECT), mafmt.Base(ma.P_CERTHASH)) + +const ( + // handshakeChannelNegotiated is used to specify that the + // handshake data channel does not need negotiation via DCEP. + // A constant is used since the `DataChannelInit` struct takes + // references instead of values. + handshakeChannelNegotiated = true + // handshakeChannelID is the agreed ID for the handshake data + // channel. A constant is used since the `DataChannelInit` struct takes + // references instead of values. We specify the type here as this + // value is only ever copied and passed by reference + handshakeChannelID = uint16(0) +) + +// timeout values for the peerconnection +// https://github.com/pion/webrtc/blob/v3.1.50/settingengine.go#L102-L109 +const ( + DefaultDisconnectedTimeout = 20 * time.Second + DefaultFailedTimeout = 30 * time.Second + DefaultKeepaliveTimeout = 15 * time.Second +) + +type WebRTCTransport struct { + webrtcConfig webrtc.Configuration + rcmgr network.ResourceManager + gater connmgr.ConnectionGater + privKey ic.PrivKey + noiseTpt *noise.Transport + localPeerId peer.ID + + // timeouts + peerConnectionTimeouts iceTimeouts + + // in-flight connections + maxInFlightConnections uint32 +} + +var _ tpt.Transport = &WebRTCTransport{} + +type Option func(*WebRTCTransport) error + +type iceTimeouts struct { + Disconnect time.Duration + Failed time.Duration + Keepalive time.Duration +} + +func New(privKey ic.PrivKey, psk pnet.PSK, gater connmgr.ConnectionGater, rcmgr network.ResourceManager, opts ...Option) (*WebRTCTransport, error) { + if psk != nil { + log.Error("WebRTC doesn't support private networks yet.") + return nil, fmt.Errorf("WebRTC doesn't support private networks yet") + } + localPeerID, err := peer.IDFromPrivateKey(privKey) + if err != nil { + return nil, fmt.Errorf("get local peer ID: %w", err) + } + // We use elliptic P-256 since it is widely supported by browsers. + // + // Implementation note: Testing with the browser, + // it seems like Chromium only supports ECDSA P-256 or RSA key signatures in the webrtc TLS certificate. + // We tried using P-228 and P-384 which caused the DTLS handshake to fail with Illegal Parameter + // + // Please refer to this is a list of suggested algorithms for the WebCrypto API. + // The algorithm for generating a certificate for an RTCPeerConnection + // must adhere to the WebCrpyto API. From my observation, + // RSA and ECDSA P-256 is supported on almost all browsers. + // Ed25519 is not present on the list. + pk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate key for cert: %w", err) + } + cert, err := webrtc.GenerateCertificate(pk) + if err != nil { + return nil, fmt.Errorf("generate certificate: %w", err) + } + config := webrtc.Configuration{ + Certificates: []webrtc.Certificate{*cert}, + } + noiseTpt, err := noise.New(noise.ID, privKey, nil) + if err != nil { + return nil, fmt.Errorf("unable to create noise transport: %w", err) + } + transport := &WebRTCTransport{ + rcmgr: rcmgr, + gater: gater, + webrtcConfig: config, + privKey: privKey, + noiseTpt: noiseTpt, + localPeerId: localPeerID, + + peerConnectionTimeouts: iceTimeouts{ + Disconnect: DefaultDisconnectedTimeout, + Failed: DefaultFailedTimeout, + Keepalive: DefaultKeepaliveTimeout, + }, + + maxInFlightConnections: DefaultMaxInFlightConnections, + } + for _, opt := range opts { + if err := opt(transport); err != nil { + return nil, err + } + } + return transport, nil +} + +func (t *WebRTCTransport) Protocols() []int { + return []int{ma.P_P2P_WEBRTC_DIRECT} +} + +func (t *WebRTCTransport) Proxy() bool { + return false +} + +func (t *WebRTCTransport) CanDial(addr ma.Multiaddr) bool { + return dialMatcher.Matches(addr) +} + +var webRTCMultiAddr = ma.StringCast("/p2p-webrtc-direct") + +func (t *WebRTCTransport) Listen(addr ma.Multiaddr) (tpt.Listener, error) { + addr, wrtcComponent := ma.SplitLast(addr) + isWebrtc := wrtcComponent.Equal(webRTCMultiAddr) + if !isWebrtc { + return nil, fmt.Errorf("must listen on webrtc multiaddr") + } + nw, host, err := manet.DialArgs(addr) + if err != nil { + return nil, fmt.Errorf("listener could not fetch dialargs: %w", err) + } + udpAddr, err := net.ResolveUDPAddr(nw, host) + if err != nil { + return nil, fmt.Errorf("listener could not resolve udp address: %w", err) + } + + socket, err := net.ListenUDP(nw, udpAddr) + if err != nil { + return nil, fmt.Errorf("listen on udp: %w", err) + } + + listener, err := t.listenSocket(socket) + if err != nil { + socket.Close() + return nil, err + } + return listener, nil +} + +func (t *WebRTCTransport) listenSocket(socket *net.UDPConn) (tpt.Listener, error) { + listenerMultiaddr, err := manet.FromNetAddr(socket.LocalAddr()) + if err != nil { + return nil, err + } + + listenerFingerprint, err := t.getCertificateFingerprint() + if err != nil { + return nil, err + } + + encodedLocalFingerprint, err := internal.EncodeDTLSFingerprint(listenerFingerprint) + if err != nil { + return nil, err + } + + certMultiaddress, err := ma.NewMultiaddr(fmt.Sprintf("/p2p-webrtc-direct/certhash/%s", encodedLocalFingerprint)) + if err != nil { + return nil, err + } + + listenerMultiaddr = listenerMultiaddr.Encapsulate(certMultiaddress) + + return newListener( + t, + listenerMultiaddr, + socket, + t.webrtcConfig, + ) +} + +func (t *WebRTCTransport) Dial(ctx context.Context, remoteMultiaddr ma.Multiaddr, p peer.ID) (tpt.CapableConn, error) { + scope, err := t.rcmgr.OpenConnection(network.DirOutbound, true, remoteMultiaddr) + if err != nil { + return nil, err + } + err = scope.SetPeer(p) + if err != nil { + scope.Done() + return nil, err + } + conn, err := t.dial(ctx, scope, remoteMultiaddr, p) + if err != nil { + scope.Done() + return nil, err + } + return conn, nil +} + +func (t *WebRTCTransport) dial( + ctx context.Context, + scope network.ConnManagementScope, + remoteMultiaddr ma.Multiaddr, + p peer.ID, +) (tConn tpt.CapableConn, err error) { + var pc *webrtc.PeerConnection + defer func() { + if err != nil { + if pc != nil { + _ = pc.Close() + } + if tConn != nil { + _ = tConn.Close() + } + } + }() + + remoteMultihash, err := internal.DecodeRemoteFingerprint(remoteMultiaddr) + if err != nil { + return nil, fmt.Errorf("decode fingerprint: %w", err) + } + remoteHashFunction, ok := internal.GetSupportedSDPHash(remoteMultihash.Code) + if !ok { + return nil, fmt.Errorf("unsupported hash function: %w", nil) + } + + rnw, rhost, err := manet.DialArgs(remoteMultiaddr) + if err != nil { + return nil, fmt.Errorf("generate dial args: %w", err) + } + + raddr, err := net.ResolveUDPAddr(rnw, rhost) + if err != nil { + return nil, fmt.Errorf("resolve udp address: %w", err) + } + + // Instead of encoding the local fingerprint we + // generate a random UUID as the connection ufrag. + // The only requirement here is that the ufrag and password + // must be equal, which will allow the server to determine + // the password using the STUN message. + ufrag := genUfrag() + + settingEngine := webrtc.SettingEngine{} + // suppress pion logs + loggerFactory := pionlogger.NewDefaultLoggerFactory() + loggerFactory.DefaultLogLevel = pionlogger.LogLevelDisabled + settingEngine.LoggerFactory = loggerFactory + + settingEngine.SetICECredentials(ufrag, ufrag) + settingEngine.DetachDataChannels() + settingEngine.SetICETimeouts( + t.peerConnectionTimeouts.Disconnect, + t.peerConnectionTimeouts.Failed, + t.peerConnectionTimeouts.Keepalive, + ) + // By default, webrtc will not collect candidates on the loopback address. + // This is disallowed in the ICE specification. However, implementations + // do not strictly follow this, for eg. Chrome gathers TCP loopback candidates. + // If you run pion on a system with only the loopback interface UP, + // it will not connect to anything. However, if it has any other interface + // (even a private one, eg. 192.168.0.0/16), it will gather candidates on it and + // will be able to connect to other pion instances on the same interface. + settingEngine.SetIncludeLoopbackCandidate(true) + + api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) + + pc, err = api.NewPeerConnection(t.webrtcConfig) + if err != nil { + return nil, fmt.Errorf("instantiate peerconnection: %w", err) + } + + errC := awaitPeerConnectionOpen(ufrag, pc) + // We need to set negotiated = true for this channel on both + // the client and server to avoid DCEP errors. + negotiated, id := handshakeChannelNegotiated, handshakeChannelID + rawHandshakeChannel, err := pc.CreateDataChannel("", &webrtc.DataChannelInit{ + Negotiated: &negotiated, + ID: &id, + }) + if err != nil { + return nil, fmt.Errorf("create datachannel: %w", err) + } + + // do offer-answer exchange + offer, err := pc.CreateOffer(nil) + if err != nil { + return nil, fmt.Errorf("create offer: %w", err) + } + + err = pc.SetLocalDescription(offer) + if err != nil { + return nil, fmt.Errorf("set local description: %w", err) + } + + answerSDPString, err := internal.RenderServerSDP(raddr, ufrag, *remoteMultihash) + if err != nil { + return nil, fmt.Errorf("render server SDP: %w", err) + } + + answer := webrtc.SessionDescription{SDP: answerSDPString, Type: webrtc.SDPTypeAnswer} + err = pc.SetRemoteDescription(answer) + if err != nil { + return nil, fmt.Errorf("set remote description: %w", err) + } + + // await peerconnection opening + select { + case err := <-errC: + if err != nil { + return nil, err + } + case <-ctx.Done(): + return nil, errors.New("peerconnection opening timed out") + } + + detached, err := getDetachedChannel(ctx, rawHandshakeChannel) + if err != nil { + return nil, err + } + // set the local address from the candidate pair + cp, err := rawHandshakeChannel.Transport().Transport().ICETransport().GetSelectedCandidatePair() + if cp == nil { + return nil, errors.New("ice connection did not have selected candidate pair: nil result") + } + if err != nil { + return nil, fmt.Errorf("ice connection did not have selected candidate pair: error: %w", err) + } + laddr := &net.UDPAddr{IP: net.ParseIP(cp.Local.Address), Port: int(cp.Local.Port)} + + channel := newStream(nil, rawHandshakeChannel, detached, laddr, raddr) + // the local address of the selected candidate pair should be the + // local address for the connection, since different datachannels + // are multiplexed over the same SCTP connection + localAddr, err := manet.FromNetAddr(channel.LocalAddr()) + if err != nil { + return nil, err + } + + // check with the gater if we can dial + if t.gater != nil && !t.gater.InterceptAddrDial(p, remoteMultiaddr) { + return nil, errors.New("not allowed to dial remote peer") + } + + // we can only know the remote public key after the noise handshake, + // but need to set up the callbacks on the peerconnection + conn, err := newConnection( + network.DirOutbound, + pc, + t, + scope, + t.localPeerId, + localAddr, + p, + nil, + remoteMultiaddr, + ) + if err != nil { + return nil, err + } + + secConn, err := t.noiseHandshake(ctx, pc, channel, p, remoteHashFunction, false) + if err != nil { + return conn, err + } + conn.setRemotePublicKey(secConn.RemotePublicKey()) + return conn, nil +} + +func genUfrag() string { + const ( + uFragAlphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" + uFragPrefix = "libp2p+webrtc+v1/" + uFragIdLength = 32 + uFragIdOffset = len(uFragPrefix) + uFragLength = uFragIdOffset + uFragIdLength + ) + + b := make([]byte, uFragLength) + copy(b[:], uFragPrefix[:]) + rand.Read(b[uFragIdOffset:]) + for i := uFragIdOffset; i < uFragLength; i++ { + b[i] = uFragAlphabet[int(b[i])%len(uFragAlphabet)] + } + return string(b) +} + +func (t *WebRTCTransport) getCertificateFingerprint() (webrtc.DTLSFingerprint, error) { + fps, err := t.webrtcConfig.Certificates[0].GetFingerprints() + if err != nil { + return webrtc.DTLSFingerprint{}, err + } + return fps[0], nil +} + +func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash crypto.Hash, inbound bool) ([]byte, error) { + raw := pc.SCTP().Transport().GetRemoteCertificate() + cert, err := x509.ParseCertificate(raw) + if err != nil { + return nil, err + } + + // NOTE: should we want we can fork the cert code as well to avoid + // all the extra allocations due to unneeded string interspersing (hex) + localFp, err := t.getCertificateFingerprint() + if err != nil { + return nil, err + } + + remoteFpBytes, err := internal.Fingerprint(cert, hash) + if err != nil { + return nil, err + } + + localFpBytes, err := encoding.DecodeInterpersedHexFromASCIIString(localFp.Value) + if err != nil { + return nil, err + } + + localEncoded, err := multihash.Encode(localFpBytes, multihash.SHA2_256) + if err != nil { + log.Debugf("could not encode multihash for local fingerprint") + return nil, err + } + remoteEncoded, err := multihash.Encode(remoteFpBytes, multihash.SHA2_256) + if err != nil { + log.Debugf("could not encode multihash for remote fingerprint") + return nil, err + } + + result := []byte("libp2p-webrtc-noise:") + if inbound { + result = append(result, remoteEncoded...) + result = append(result, localEncoded...) + } else { + result = append(result, localEncoded...) + result = append(result, remoteEncoded...) + } + return result, nil +} + +func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, datachannel *webRTCStream, peer peer.ID, hash crypto.Hash, inbound bool) (sec.SecureConn, error) { + prologue, err := t.generateNoisePrologue(pc, hash, inbound) + if err != nil { + return nil, fmt.Errorf("generate prologue: %w", err) + } + sessionTransport, err := t.noiseTpt.WithSessionOptions( + noise.Prologue(prologue), + noise.DisablePeerIDCheck(), + ) + if err != nil { + return nil, fmt.Errorf("instantiate transport: %w", err) + } + var secureConn sec.SecureConn + if inbound { + secureConn, err = sessionTransport.SecureOutbound(ctx, datachannel, peer) + if err != nil { + err = fmt.Errorf("failed to secure inbound [noise outbound]: %w %v", err, ctx.Value("id")) + return secureConn, err + } + } else { + secureConn, err = sessionTransport.SecureInbound(ctx, datachannel, peer) + if err != nil { + err = fmt.Errorf("failed to secure outbound [noise inbound]: %w %v", err, ctx.Value("id")) + return secureConn, err + } + } + return secureConn, nil +} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go new file mode 100644 index 0000000000..1e5de8abaa --- /dev/null +++ b/p2p/transport/webrtc/transport_test.go @@ -0,0 +1,914 @@ +package libp2pwebrtc + +import ( + "context" + "encoding/hex" + "fmt" + "io" + "net" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + ttransport "github.com/libp2p/go-libp2p/p2p/transport/testsuite" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multibase" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/sha3" +) + +func getTransport(t *testing.T, opts ...Option) (*WebRTCTransport, peer.ID) { + t.Helper() + privKey, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1) + require.NoError(t, err) + rcmgr := &network.NullResourceManager{} + transport, err := New(privKey, nil, nil, rcmgr, opts...) + require.NoError(t, err) + peerID, err := peer.IDFromPrivateKey(privKey) + require.NoError(t, err) + t.Cleanup(func() { rcmgr.Close() }) + return transport, peerID +} + +var ( + listenerIp net.IP + dialerIp net.IP +) + +func TestMain(m *testing.M) { + listenerIp, dialerIp = getListenerAndDialerIP() + os.Exit(m.Run()) +} + +func TestTransportWebRTC_CanDial(t *testing.T) { + tr, _ := getTransport(t) + invalid := []string{ + "/ip4/1.2.3.4/udp/1234/p2p-webrtc-direct", + "/dns/test.test/udp/1234/p2p-webrtc-direct", + } + + valid := []string{ + "/ip4/1.2.3.4/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/ip6/0:0:0:0:0:0:0:1/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/ip6/::1/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/dns/test.test/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + } + + for _, addr := range invalid { + ma, err := multiaddr.NewMultiaddr(addr) + require.NoError(t, err) + require.Equal(t, false, tr.CanDial(ma)) + } + + for _, addr := range valid { + ma, err := multiaddr.NewMultiaddr(addr) + require.NoError(t, err) + require.Equal(t, true, tr.CanDial(ma), addr) + } +} + +func TestTransportWebRTC_ListenFailsOnNonWebRTCMultiaddr(t *testing.T) { + tr, _ := getTransport(t) + testAddrs := []string{ + "/ip4/0.0.0.0/udp/0", + "/ip4/0.0.0.0/tcp/0/wss", + } + for _, addr := range testAddrs { + listenMultiaddr, err := multiaddr.NewMultiaddr(addr) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.Error(t, err) + require.Nil(t, listener) + } +} + +// using assert inside goroutines, refer: https://github.com/stretchr/testify/issues/772#issuecomment-945166599 +func TestTransportWebRTC_DialFailsOnUnsupportedHashFunction(t *testing.T) { + tr, _ := getTransport(t) + hash := sha3.New512() + certhash := func() string { + _, err := hash.Write([]byte("test-data")) + require.NoError(t, err) + mh, err := multihash.Encode(hash.Sum([]byte{}), multihash.SHA3_512) + require.NoError(t, err) + certhash, err := multibase.Encode(multibase.Base58BTC, mh) + require.NoError(t, err) + return certhash + }() + testaddr, err := multiaddr.NewMultiaddr("/ip4/1.2.3.4/udp/1234/p2p-webrtc-direct/certhash/" + certhash) + require.NoError(t, err) + _, err = tr.Dial(context.Background(), testaddr, "") + require.ErrorContains(t, err, "unsupported hash function") +} + +func TestTransportWebRTC_CanListenSingle(t *testing.T) { + tr, listeningPeer := getTransport(t) + tr1, connectingPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + _, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + assert.NoError(t, err) + close(done) + }() + + conn, err := listener.Accept() + require.NoError(t, err) + require.NotNil(t, conn) + + require.Equal(t, connectingPeer, conn.RemotePeer()) + select { + case <-done: + case <-time.After(10 * time.Second): + t.FailNow() + } +} + +// WithListenerMaxInFlightConnections sets the maximum number of connections that are in-flight, i.e +// they are being negotiated, or are waiting to be accepted. +func WithListenerMaxInFlightConnections(m uint32) Option { + return func(t *WebRTCTransport) error { + if m == 0 { + t.maxInFlightConnections = DefaultMaxInFlightConnections + } else { + t.maxInFlightConnections = m + } + return nil + } +} + +func TestTransportWebRTC_CanListenMultiple(t *testing.T) { + count := 3 + tr, listeningPeer := getTransport(t, WithListenerMaxInFlightConnections(uint32(count))) + + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + go func() { + for i := 0; i < count; i++ { + conn, err := listener.Accept() + assert.NoError(t, err) + assert.NotNil(t, conn) + } + wg.Wait() + cancel() + }() + + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ctr, _ := getTransport(t) + conn, err := ctr.Dial(ctx, listener.Multiaddr(), listeningPeer) + select { + case <-ctx.Done(): + default: + assert.NoError(t, err) + assert.NotNil(t, conn) + } + }() + } + + select { + case <-ctx.Done(): + case <-time.After(30 * time.Second): + t.Fatalf("timed out") + } + +} + +func TestTransportWebRTC_CanCreateSuccessiveConnections(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + count := 2 + + go func() { + for i := 0; i < count; i++ { + ctr, _ := getTransport(t) + conn, err := ctr.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + require.Equal(t, conn.RemotePeer(), listeningPeer) + } + }() + + for i := 0; i < count; i++ { + _, err := listener.Accept() + require.NoError(t, err) + } +} + +func TestTransportWebRTC_ListenerCanCreateStreams(t *testing.T) { + tr, listeningPeer := getTransport(t) + tr1, connectingPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + streamChan := make(chan network.MuxedStream) + go func() { + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + t.Logf("connection opened by dialer") + stream, err := conn.AcceptStream() + require.NoError(t, err) + t.Logf("dialer accepted stream") + streamChan <- stream + }() + + conn, err := listener.Accept() + require.NoError(t, err) + t.Logf("listener accepted connection") + require.Equal(t, connectingPeer, conn.RemotePeer()) + + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + t.Logf("listener opened stream") + _, err = stream.Write([]byte("test")) + require.NoError(t, err) + + var str network.MuxedStream + select { + case str = <-streamChan: + case <-time.After(3 * time.Second): + t.Fatal("stream opening timed out") + } + buf := make([]byte, 100) + stream.SetReadDeadline(time.Now().Add(3 * time.Second)) + n, err := str.Read(buf) + require.NoError(t, err) + require.Equal(t, "test", string(buf[:n])) + +} + +func TestTransportWebRTC_DialerCanCreateStreams(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + done := make(chan struct{}) + + go func() { + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + + stream, err := lconn.AcceptStream() + require.NoError(t, err) + buf := make([]byte, 100) + n, err := stream.Read(buf) + require.NoError(t, err) + require.Equal(t, "test", string(buf[:n])) + + done <- struct{}{} + }() + + go func() { + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + t.Logf("dialer opened connection") + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + t.Logf("dialer opened stream") + _, err = stream.Write([]byte("test")) + require.NoError(t, err) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timed out") + } + +} + +func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { + count := 5 + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + done := make(chan struct{}) + + go func() { + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + var wg sync.WaitGroup + + for i := 0; i < count; i++ { + stream, err := lconn.AcceptStream() + require.NoError(t, err) + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, 100) + n, err := stream.Read(buf) + require.NoError(t, err) + require.Equal(t, "test", string(buf[:n])) + _, err = stream.Write([]byte("test")) + require.NoError(t, err) + }() + } + + wg.Wait() + done <- struct{}{} + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + t.Logf("dialer opened connection") + + for i := 0; i < count; i++ { + idx := i + go func() { + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + t.Logf("dialer opened stream: %d", idx) + buf := make([]byte, 100) + _, err = stream.Write([]byte("test")) + require.NoError(t, err) + n, err := stream.Read(buf) + require.NoError(t, err) + require.Equal(t, "test", string(buf[:n])) + }() + if i%10 == 0 && i > 0 { + time.Sleep(100 * time.Millisecond) + } + } + select { + case <-done: + case <-time.After(100 * time.Second): + t.Fatal("timed out") + } +} + +func TestTransportWebRTC_Deadline(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + tr1, connectingPeer := getTransport(t) + + t.Run("SetReadDeadline", func(t *testing.T) { + go func() { + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + _, err = lconn.AcceptStream() + require.NoError(t, err) + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + // deadline set to the past + stream.SetReadDeadline(time.Now().Add(-200 * time.Millisecond)) + _, err = stream.Read([]byte{0, 0}) + require.ErrorIs(t, err, ErrTimeout) + + // future deadline exceeded + stream.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + _, err = stream.Read([]byte{0, 0}) + require.ErrorIs(t, err, ErrTimeout) + }) + + t.Run("SetWriteDeadline", func(t *testing.T) { + go func() { + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + _, err = lconn.AcceptStream() + require.NoError(t, err) + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + stream.SetWriteDeadline(time.Now().Add(200 * time.Millisecond)) + largeBuffer := make([]byte, 2*1024*1024) + _, err = stream.Write(largeBuffer) + require.ErrorIs(t, err, ErrTimeout) + }) +} + +func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + + for i := 0; i < 2; i++ { + go func() { + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + _, err = lconn.AcceptStream() + require.NoError(t, err) + }() + + } + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + + errC := make(chan error) + // writers + for i := 0; i < 2; i++ { + go func() { + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + stream.SetWriteDeadline(time.Now().Add(200 * time.Millisecond)) + largeBuffer := make([]byte, 2*1024*1024) + _, err = stream.Write(largeBuffer) + errC <- err + }() + } + + require.ErrorIs(t, <-errC, ErrTimeout) + require.ErrorIs(t, <-errC, ErrTimeout) + +} + +func TestTransportWebRTC_Read(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + + createListener := func() { + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + stream, err := lconn.AcceptStream() + require.NoError(t, err) + _, _ = stream.Write(make([]byte, 2*1024)) + } + + t.Run("read partial message", func(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + createListener() + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + buf := make([]byte, 10) + stream.SetReadDeadline(time.Now().Add(10 * time.Second)) + n, err := stream.Read(buf) + require.NoError(t, err) + require.Equal(t, n, 10) + + wg.Wait() + }) + + t.Run("read zero bytes", func(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + createListener() + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + stream.SetReadDeadline(time.Now().Add(10 * time.Second)) + n, err := stream.Read([]byte{}) + require.NoError(t, err) + require.Equal(t, n, 0) + + wg.Wait() + }) +} + +func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, _ := getTransport(t) + + done := make(chan error) + go func() { + lconn, err := listener.Accept() + if err != nil { + done <- err + return + } + stream, err := lconn.AcceptStream() + if err != nil { + done <- err + return + } + _, err = stream.Write([]byte{1, 2, 3, 4}) + if err != nil { + done <- err + return + } + err = stream.Close() + if err != nil { + done <- err + return + } + close(done) + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + // create a stream + stream, err := conn.OpenStream(context.Background()) + + require.NoError(t, err) + // require write and close to complete + require.NoError(t, <-done) + + stream.SetReadDeadline(time.Now().Add(5 * time.Second)) + + buf := make([]byte, 10) + n, err := stream.Read(buf) + require.NoError(t, err) + require.Equal(t, n, 4) +} + +func TestTransportWebRTC_RemoteReadsAfterClose2(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, _ := getTransport(t) + + awaitStreamClosure := make(chan struct{}) + readBytesResult := make(chan int) + done := make(chan error) + go func() { + lconn, err := listener.Accept() + if err != nil { + done <- err + return + } + stream, err := lconn.AcceptStream() + if err != nil { + done <- err + return + } + + <-awaitStreamClosure + buf := make([]byte, 16) + n, err := stream.Read(buf) + if err != nil { + done <- err + return + } + readBytesResult <- n + close(done) + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + // create a stream + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + _, err = stream.Write([]byte{1, 2, 3, 4}) + require.NoError(t, err) + err = stream.Close() + require.NoError(t, err) + // signal stream closure + close(awaitStreamClosure) + require.Equal(t, <-readBytesResult, 4) +} + +func TestTransportWebRTC_Close(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + + t.Run("StreamCanCloseWhenReadActive", func(t *testing.T) { + done := make(chan struct{}) + + go func() { + lconn, err := listener.Accept() + require.NoError(t, err) + t.Logf("listener accepted connection") + require.Equal(t, connectingPeer, lconn.RemotePeer()) + done <- struct{}{} + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + t.Logf("dialer opened connection") + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + time.AfterFunc(100*time.Millisecond, func() { + err := stream.Close() + require.NoError(t, err) + }) + + _, err = stream.Read(make([]byte, 19)) + require.ErrorIs(t, err, ErrTimeout) + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timed out") + } + }) + + t.Run("RemoteClosesStream", func(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + lconn, err := listener.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + stream, err := lconn.AcceptStream() + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + _ = stream.Close() + + }() + + buf := make([]byte, 2) + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + err = stream.SetReadDeadline(time.Now().Add(2 * time.Second)) + require.NoError(t, err) + _, err = stream.Read(buf) + require.ErrorIs(t, err, io.EOF) + + wg.Wait() + }) +} + +func TestTransportWebRTC_ReceiveFlagsAfterReadClosed(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + done := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + lconn, err := listener.Accept() + require.NoError(t, err) + t.Logf("listener accepted connection") + require.Equal(t, connectingPeer, lconn.RemotePeer()) + stream, err := lconn.AcceptStream() + require.NoError(t, err) + n, err := stream.Read(make([]byte, 10)) + require.NoError(t, err) + require.Equal(t, 10, n) + // stop reader + err = stream.Reset() + require.NoError(t, err) + done <- struct{}{} + }() + + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + t.Logf("dialer opened connection") + stream, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + + err = stream.CloseRead() + require.NoError(t, err) + _, err = stream.Read([]byte{0}) + require.ErrorIs(t, err, io.EOF) + _, err = stream.Write(make([]byte, 10)) + require.NoError(t, err) + <-done + _, err = stream.Write(make([]byte, 2*1024*1024)) + // can be an error closed or timeout + require.Error(t, err) + + wg.Wait() +} + +func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { + // test multihash + encoded, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + require.NoError(t, err) + + testMultihash := &multihash.DecodedMultihash{ + Code: multihash.SHA2_256, + Name: multihash.Codes[multihash.SHA2_256], + Digest: encoded, + Length: len(encoded), + } + + tr, listeningPeer := getTransport(t) + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, _ := getTransport(t) + + go listener.Accept() + + badMultiaddr, _ := multiaddr.SplitFunc(listener.Multiaddr(), func(component multiaddr.Component) bool { + return component.Protocol().Code == multiaddr.P_CERTHASH + }) + + encodedCerthash, err := multihash.Encode(testMultihash.Digest, testMultihash.Code) + require.NoError(t, err) + badEncodedCerthash, err := multibase.Encode(multibase.Base58BTC, encodedCerthash) + require.NoError(t, err) + badCerthash, err := multiaddr.NewMultiaddr(fmt.Sprintf("/certhash/%s", badEncodedCerthash)) + require.NoError(t, err) + badMultiaddr = badMultiaddr.Encapsulate(badCerthash) + + conn, err := tr1.Dial(context.Background(), badMultiaddr, listeningPeer) + require.Nil(t, conn) + t.Log(err) + require.Error(t, err) + + require.ErrorContains(t, err, "failed") +} + +func TestTransportWebRTC_StreamResetOnPeerConnectionFailure(t *testing.T) { + tr, listeningPeer := getTransport(t) + tr.peerConnectionTimeouts.Disconnect = 2 * time.Second + tr.peerConnectionTimeouts.Failed = 3 * time.Second + tr.peerConnectionTimeouts.Keepalive = time.Second + + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + lsnr, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, connectingPeer := getTransport(t) + tr1.peerConnectionTimeouts.Disconnect = 2 * time.Second + tr1.peerConnectionTimeouts.Failed = 3 * time.Second + tr1.peerConnectionTimeouts.Keepalive = time.Second + + done := make(chan struct{}) + go func() { + lconn, err := lsnr.Accept() + require.NoError(t, err) + require.Equal(t, connectingPeer, lconn.RemotePeer()) + + stream, err := lconn.AcceptStream() + require.NoError(t, err) + _, err = stream.Write([]byte("test")) + require.NoError(t, err) + // force close the mux + lsnr.(*listener).mux.Close() + // stream.Write can keep buffering data until failure, + // so we need to loop on writing. + for { + _, err := stream.Write([]byte("test")) + if err != nil { + assert.ErrorIs(t, err, network.ErrReset) + close(done) + return + } + } + }() + + dialctx, dialcancel := context.WithTimeout(context.Background(), 10*time.Second) + defer dialcancel() + conn, err := tr1.Dial(dialctx, lsnr.Multiaddr(), listeningPeer) + require.NoError(t, err) + stream, err := conn.OpenStream(dialctx) + require.NoError(t, err) + _, err = io.ReadAll(stream) + require.Error(t, err) + + select { + case <-done: + case <-time.After(30 * time.Second): + t.Fatal("timed out") + } +} + +func TestTransportWebRTC_MaxInFlightRequests(t *testing.T) { + count := uint32(3) + tr, listeningPeer := getTransport(t, + WithListenerMaxInFlightConnections(count), + ) + tr.peerConnectionTimeouts.Disconnect = 8 * time.Second + tr.peerConnectionTimeouts.Failed = 10 * time.Second + tr.peerConnectionTimeouts.Keepalive = 5 * time.Second + listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + require.NoError(t, err) + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + defer listener.Close() + + dialerCount := count + 2 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + start := make(chan struct{}) + ready := make(chan struct{}, dialerCount) + var success uint32 + for i := 0; uint32(i) < dialerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + dialer, _ := getTransport(t) + dialer.peerConnectionTimeouts.Disconnect = 8 * time.Second + dialer.peerConnectionTimeouts.Failed = 10 * time.Second + dialer.peerConnectionTimeouts.Keepalive = 5 * time.Second + ready <- struct{}{} + <-start + _, err := dialer.Dial(ctx, listener.Multiaddr(), listeningPeer) + if err == nil { + atomic.AddUint32(&success, 1) + } else { + t.Logf("dial error: %v", err) + } + }() + } + + for i := 0; uint32(i) < dialerCount; i++ { + <-ready + } + close(start) + wg.Wait() + successCount := atomic.LoadUint32(&success) + require.Equal(t, count, successCount) +} + +// TestWebrtcTransport implements the standard go-libp2p transport test. +// It's a test that however not works for many transports, and neither does it for WebRTC. +// +// Reason it doens't work for WebRTC is that it opens too many streams too rapidly, which +// in a regular environment would be seen as an attack that we wish to block/stop. +// +// Leaving it here for now only for documentation purposes, +// and to make it explicitly clear this test doesn't work for WebRTC. +func TestWebrtcTransport(t *testing.T) { + t.Skip("This test does not work for WebRTC due to the way it is setup, see comments for more explanation") + ta, _ := getTransport(t) + tb, _ := getTransport(t) + ttransport.SubtestTransport(t, ta, tb, fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp), "peerA") +} diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go new file mode 100644 index 0000000000..ab957591a1 --- /dev/null +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -0,0 +1,293 @@ +package udpmux + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "sync" + + logging "github.com/ipfs/go-log/v2" + pool "github.com/libp2p/go-buffer-pool" + "github.com/pion/ice/v2" + "github.com/pion/stun" +) + +var log = logging.Logger("mux") + +var ( + errConnNotFound = errors.New("connection not found") +) + +const ReceiveMTU = 1500 + +var _ ice.UDPMux = &udpMux{} + +type ufragConnKey struct { + ufrag string + isIPv6 bool +} + +// udpMux multiplexes multiple ICE connections over a single net.PacketConn, +// generally a UDP socket. +// +// The connections are indexed by (ufrag, IP address family) +// and by remote address from which the connection has received valid STUN/RTC +// packets. +// +// When a new packet is received on the underlying net.PacketConn, we +// first check the address map to see if there is a connection associated with the +// remote address. If found we forward the packet to the connection. If an associated +// connection is not found, we check to see if the packet is a STUN packet. We then +// fetch the ufrag of the remote from the STUN packet and use it to check if there +// is a connection associated with the (ufrag, IP address family) pair. If found +// we add the association to the address map. If not found, it is a previously +// unseen IP address and the `unknownUfragCallback` callback is invoked. +type udpMux struct { + socket net.PacketConn + unknownUfragCallback func(string, net.Addr) bool + + storage *udpMuxStorage + + // the context controls the lifecycle of the mux + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc +} + +func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) bool) *udpMux { + ctx, cancel := context.WithCancel(context.Background()) + mux := &udpMux{ + ctx: ctx, + cancel: cancel, + socket: socket, + unknownUfragCallback: unknownUfragCallback, + storage: newUDPMuxStorage(), + } + + mux.wg.Add(1) + go func() { + defer mux.wg.Done() + mux.readLoop() + }() + return mux +} + +// GetListenAddresses implements ice.UDPMux +func (mux *udpMux) GetListenAddresses() []net.Addr { + return []net.Addr{mux.socket.LocalAddr()} +} + +// GetConn implements ice.UDPMux +// It creates a net.PacketConn for a given ufrag if an existing +// one cannot be found. We differentiate IPv4 and IPv6 addresses +// as a remote is capable of being reachable through multiple different +// UDP addresses of the same IP address family (eg. Server-reflexive addresses +// and peer-reflexive addresses). +func (mux *udpMux) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) { + a, ok := addr.(*net.UDPAddr) + if !ok && addr != nil { + return nil, fmt.Errorf("unexpected address type: %T", addr) + } + isIPv6 := ok && a.IP.To4() == nil + return mux.getOrCreateConn(ufrag, isIPv6, addr) +} + +// Close implements ice.UDPMux +func (mux *udpMux) Close() error { + select { + case <-mux.ctx.Done(): + return nil + default: + } + mux.cancel() + mux.socket.Close() + mux.wg.Wait() + return nil +} + +// RemoveConnByUfrag implements ice.UDPMux +func (mux *udpMux) RemoveConnByUfrag(ufrag string) { + if ufrag != "" { + mux.storage.RemoveConnByUfrag(ufrag) + } +} + +func (mux *udpMux) getOrCreateConn(ufrag string, isIPv6 bool, addr net.Addr) (net.PacketConn, error) { + select { + case <-mux.ctx.Done(): + return nil, io.ErrClosedPipe + default: + conn, _, err := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, addr) + return conn, err + } +} + +// writeTo writes a packet to the underlying net.PacketConn +func (mux *udpMux) writeTo(buf []byte, addr net.Addr) (int, error) { + return mux.socket.WriteTo(buf, addr) +} + +func (mux *udpMux) readLoop() { + for { + select { + case <-mux.ctx.Done(): + return + default: + } + + buf := pool.Get(ReceiveMTU) + + n, addr, err := mux.socket.ReadFrom(buf) + if err != nil { + log.Errorf("error reading from socket: %v", err) + pool.Put(buf) + return + } + buf = buf[:n] + + // a non-nil error signifies that the packet was not + // passed on to any connection, and therefore the current + // function has ownership of the packet. Otherwise, the + // ownership of the packet is passed to a connection + if err := mux.processPacket(buf, addr); err != nil { + pool.Put(buf) + } + } +} + +func (mux *udpMux) processPacket(buf []byte, addr net.Addr) error { + udpAddr, ok := addr.(*net.UDPAddr) + if !ok { + return fmt.Errorf("underlying connection did not return a UDP address") + } + isIPv6 := udpAddr.IP.To4() == nil + + // Connections are indexed by remote address. We first + // check if the remote address has a connection associated + // with it. If yes, we push the received packet to the connection + if conn, ok := mux.storage.GetConnByAddr(udpAddr); ok { + err := conn.Push(buf) + if err != nil { + log.Errorf("could not push packet: %v", err) + } + return nil + } + + if !stun.IsMessage(buf) { + return errConnNotFound + } + + msg := &stun.Message{Raw: buf} + if err := msg.Decode(); err != nil || msg.Type != stun.BindingRequest { + log.Debug("incoming message should be a STUN binding request") + return err + } + + ufrag, err := ufragFromSTUNMessage(msg) + if err != nil { + log.Debug("could not find STUN username: %w", err) + return err + } + + var connCreated bool + conn, connCreated, err := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) + if err != nil { + log.Debug("could not find create conn: %w", err) + return err + } + if connCreated && mux.unknownUfragCallback != nil { + if !mux.unknownUfragCallback(ufrag, udpAddr) { + conn.Close() + return io.ErrClosedPipe + } + } + + if err := conn.Push(buf); err != nil { + log.Errorf("could not push packet: %v", err) + } + return nil +} + +// ufragFromSTUNMessage returns the local or ufrag +// from the STUN username attribute. Local ufrag is the ufrag of the +// peer which initiated the connectivity check, e.g in a connectivity +// check from A to B, the username attribute will be B_ufrag:A_ufrag +// with the local ufrag value being A_ufrag. In case of ice-lite, the +// localUfrag value will always be the remote peer's ufrag since ICE-lite +// implementations do not generate connectivity checks. In our specific +// case, since the local and remote ufrag is equal, we can return +// either value. +func ufragFromSTUNMessage(msg *stun.Message) (string, error) { + attr, err := msg.Get(stun.AttrUsername) + if err != nil { + return "", err + } + index := bytes.Index(attr, []byte{':'}) + if index == -1 { + return "", fmt.Errorf("invalid STUN username attribute") + } + return string(attr[index+1:]), nil +} + +type udpMuxStorage struct { + sync.Mutex + + ufragMap map[ufragConnKey]*muxedConnection + addrMap map[string]*muxedConnection +} + +func newUDPMuxStorage() *udpMuxStorage { + return &udpMuxStorage{ + ufragMap: make(map[ufragConnKey]*muxedConnection), + addrMap: make(map[string]*muxedConnection), + } +} + +func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { + s.Lock() + defer s.Unlock() + + for _, isIPv6 := range [...]bool{true, false} { + key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} + if conn, ok := s.ufragMap[key]; ok { + _ = conn.closeConnection() + delete(s.ufragMap, key) + delete(s.addrMap, conn.Address().String()) + } + } +} + +func (s *udpMuxStorage) GetConn(ufrag string, isIPv6 bool) (*muxedConnection, bool) { + key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} + s.Lock() + conn, ok := s.ufragMap[key] + s.Unlock() + return conn, ok +} + +func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *udpMux, addr net.Addr) (*muxedConnection, bool, error) { + key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} + + s.Lock() + defer s.Unlock() + + if conn, ok := s.ufragMap[key]; ok { + return conn, false, nil + } + + conn := newMuxedConnection(mux, ufrag, addr) + s.ufragMap[key] = conn + s.addrMap[addr.String()] = conn + + return conn, true, nil +} + +func (s *udpMuxStorage) GetConnByAddr(addr *net.UDPAddr) (*muxedConnection, bool) { + s.Lock() + conn, ok := s.addrMap[addr.String()] + s.Unlock() + return conn, ok +} diff --git a/p2p/transport/webrtc/udpmux/mux_test.go b/p2p/transport/webrtc/udpmux/mux_test.go new file mode 100644 index 0000000000..5d7fc7960d --- /dev/null +++ b/p2p/transport/webrtc/udpmux/mux_test.go @@ -0,0 +1,87 @@ +package udpmux + +import ( + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var _ net.PacketConn = dummyPacketConn{} + +type dummyPacketConn struct{} + +// Close implements net.PacketConn +func (dummyPacketConn) Close() error { + return nil +} + +// LocalAddr implements net.PacketConn +func (dummyPacketConn) LocalAddr() net.Addr { + return nil +} + +// ReadFrom implements net.PacketConn +func (dummyPacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { + return 0, &net.UDPAddr{}, nil +} + +// SetDeadline implements net.PacketConn +func (dummyPacketConn) SetDeadline(t time.Time) error { + return nil +} + +// SetReadDeadline implements net.PacketConn +func (dummyPacketConn) SetReadDeadline(t time.Time) error { + return nil +} + +// SetWriteDeadline implements net.PacketConn +func (dummyPacketConn) SetWriteDeadline(t time.Time) error { + return nil +} + +// WriteTo implements net.PacketConn +func (dummyPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) { + return 0, nil +} + +func hasConn(m *udpMux, ufrag string, isIPv6 bool) *muxedConnection { + conn, _ := m.storage.GetConn(ufrag, isIPv6) + return conn +} + +var ( + addrV4 = net.UDPAddr{IP: net.IPv4zero, Port: 1234} + addrV6 = net.UDPAddr{IP: net.IPv6zero, Port: 1234} +) + +func TestUDPMux_GetConn(t *testing.T) { + m := NewUDPMux(dummyPacketConn{}, nil) + require.Nil(t, hasConn(m, "test", false)) + conn, err := m.GetConn("test", &addrV4) + require.NoError(t, err) + require.NotNil(t, conn) + + require.Nil(t, hasConn(m, "test", true)) + connv6, err := m.GetConn("test", &addrV6) + require.NoError(t, err) + require.NotNil(t, connv6) + + require.NotEqual(t, conn, connv6) +} + +func TestUDPMux_RemoveConnectionOnClose(t *testing.T) { + mux := NewUDPMux(dummyPacketConn{}, nil) + conn, err := mux.GetConn("test", &addrV4) + require.NoError(t, err) + require.NotNil(t, conn) + + require.NotNil(t, hasConn(mux, "test", false)) + + err = conn.Close() + require.NoError(t, err) + + require.Nil(t, hasConn(mux, "test", false)) +} diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go new file mode 100644 index 0000000000..bafc728b9b --- /dev/null +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -0,0 +1,98 @@ +package udpmux + +import ( + "context" + "errors" + "net" + "time" +) + +var _ net.PacketConn = &muxedConnection{} + +var errAlreadyClosed = errors.New("already closed") + +// muxedConnection provides a net.PacketConn abstraction +// over packetQueue and adds the ability to store addresses +// from which this connection (indexed by ufrag) received +// data. +type muxedConnection struct { + ctx context.Context + cancel context.CancelFunc + pq *packetQueue + ufrag string + addr net.Addr + mux *udpMux +} + +var _ net.PacketConn = (*muxedConnection)(nil) + +func newMuxedConnection(mux *udpMux, ufrag string, addr net.Addr) *muxedConnection { + ctx, cancel := context.WithCancel(mux.ctx) + return &muxedConnection{ + ctx: ctx, + cancel: cancel, + pq: newPacketQueue(), + ufrag: ufrag, + addr: addr, + mux: mux, + } +} + +func (conn *muxedConnection) Push(buf []byte) error { + return conn.pq.Push(conn.ctx, buf) +} + +// Close implements net.PacketConn +func (conn *muxedConnection) Close() error { + _ = conn.closeConnection() + conn.mux.RemoveConnByUfrag(conn.ufrag) + return nil +} + +// LocalAddr implements net.PacketConn +func (conn *muxedConnection) LocalAddr() net.Addr { + return conn.mux.socket.LocalAddr() +} + +func (conn *muxedConnection) Address() net.Addr { + return conn.addr +} + +// ReadFrom implements net.PacketConn +func (conn *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { + n, err := conn.pq.Pop(conn.ctx, p) + return n, conn.addr, err +} + +// SetDeadline implements net.PacketConn +func (*muxedConnection) SetDeadline(t time.Time) error { + // no deadline is desired here + return nil +} + +// SetReadDeadline implements net.PacketConn +func (*muxedConnection) SetReadDeadline(t time.Time) error { + // no read deadline is desired here + return nil +} + +// SetWriteDeadline implements net.PacketConn +func (*muxedConnection) SetWriteDeadline(t time.Time) error { + // no write deadline is desired here + return nil +} + +// WriteTo implements net.PacketConn +func (conn *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { + return conn.mux.writeTo(p, addr) +} + +func (conn *muxedConnection) closeConnection() error { + select { + case <-conn.ctx.Done(): + return errAlreadyClosed + default: + } + conn.cancel() + return nil +} diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go new file mode 100644 index 0000000000..107ee416bc --- /dev/null +++ b/p2p/transport/webrtc/udpmux/packetqueue.go @@ -0,0 +1,92 @@ +package udpmux + +import ( + "context" + "errors" + "sync" + + pool "github.com/libp2p/go-buffer-pool" +) + +type packet struct { + buf []byte +} + +var ( + errTooManyPackets = errors.New("too many packets in queue; dropping") + errEmptyPacketQueue = errors.New("packet queue is empty") + errPacketQueueClosed = errors.New("packet queue closed") +) + +const maxPacketsInQueue = 128 + +type packetQueue struct { + packetsMux sync.Mutex + packetsCh chan struct{} + packets []packet +} + +func newPacketQueue() *packetQueue { + return &packetQueue{ + packetsCh: make(chan struct{}, 1), + } +} + +// Pop reads a packet from the packetQueue or blocks until +// either a packet becomes available or the queue is closed. +func (pq *packetQueue) Pop(ctx context.Context, buf []byte) (int, error) { + select { + case <-pq.packetsCh: + pq.packetsMux.Lock() + defer pq.packetsMux.Unlock() + + if len(pq.packets) == 0 { + return 0, errEmptyPacketQueue + } + p := pq.packets[0] + + n := copy(buf, p.buf) + if n == len(p.buf) { + // only move packet from queue if we read all + pq.packets = pq.packets[1:] + pool.Put(p.buf) + } else { + // otherwise we need to keep the packet in the queue + // but do update the buf + pq.packets[0].buf = p.buf[n:] + } + + if len(pq.packets) > 0 { + // to make sure a next pop call will work + select { + case pq.packetsCh <- struct{}{}: + default: + } + } + + return n, nil + + // It is desired to allow reads of this channel even + // when pq.ctx.Done() is already closed. + case <-ctx.Done(): + return 0, errPacketQueueClosed + } +} + +// Push adds a packet to the packetQueue +func (pq *packetQueue) Push(ctx context.Context, buf []byte) error { + pq.packetsMux.Lock() + defer pq.packetsMux.Unlock() + + if len(pq.packets) >= maxPacketsInQueue { + return errTooManyPackets + } + + pq.packets = append(pq.packets, packet{buf}) + select { + case pq.packetsCh <- struct{}{}: + default: + } + + return nil +} diff --git a/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go b/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go new file mode 100644 index 0000000000..dbbb3b1bc1 --- /dev/null +++ b/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go @@ -0,0 +1,41 @@ +package udpmux + +import ( + "context" + "fmt" + "testing" + + pool "github.com/libp2p/go-buffer-pool" +) + +var sizes = []int{ + 1, + 10, + 100, + maxPacketsInQueue, +} + +func BenchmarkQueue(b *testing.B) { + ctx := context.Background() + for _, dequeue := range [...]bool{true, false} { + for _, input := range sizes { + testCase := fmt.Sprintf("enqueue_%d", input) + if dequeue { + testCase = testCase + "_dequeue" + } + b.Run(testCase, func(b *testing.B) { + pq := newPacketQueue() + buf := make([]byte, 256) + b.ResetTimer() + for i := 0; i < b.N; i++ { + for k := 0; k < input; k++ { + pq.Push(ctx, pool.Get(255)) + } + for k := 0; k < input; k++ { + pq.Pop(ctx, buf) + } + } + }) + } + } +} diff --git a/p2p/transport/webrtc/udpmux/packetqueue_test.go b/p2p/transport/webrtc/udpmux/packetqueue_test.go new file mode 100644 index 0000000000..7abe0c1a4b --- /dev/null +++ b/p2p/transport/webrtc/udpmux/packetqueue_test.go @@ -0,0 +1,61 @@ +package udpmux + +import ( + "context" + "testing" + "time" + + pool "github.com/libp2p/go-buffer-pool" + "github.com/stretchr/testify/require" +) + +func TestPacketQueue_QueuePacketsForRead(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pq := newPacketQueue() + pq.Push(ctx, []byte{1, 2, 3}) + pq.Push(ctx, []byte{5, 6, 7, 8}) + + buf := pool.Get(100) + size, err := pq.Pop(ctx, buf) + require.NoError(t, err) + require.Equal(t, size, 3) + + size, err = pq.Pop(ctx, buf) + require.NoError(t, err) + require.Equal(t, size, 4) +} + +func TestPacketQueue_WaitsForData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pq := newPacketQueue() + buf := pool.Get(100) + + timer := time.AfterFunc(200*time.Millisecond, func() { + pq.Push(ctx, []byte{5, 6, 7, 8}) + }) + + defer timer.Stop() + size, err := pq.Pop(ctx, buf) + require.NoError(t, err) + require.Equal(t, size, 4) +} + +func TestPacketQueue_DropsPacketsWhenQueueIsFull(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pq := newPacketQueue() + for i := 0; i < maxPacketsInQueue; i++ { + buf := pool.Get(255) + err := pq.Push(ctx, buf) + require.NoError(t, err) + } + + buf := pool.Get(255) + err := pq.Push(ctx, buf) + require.ErrorIs(t, err, errTooManyPackets) +} diff --git a/p2p/transport/webrtc/webrtc.go b/p2p/transport/webrtc/webrtc.go new file mode 100644 index 0000000000..35bfcbc7d0 --- /dev/null +++ b/p2p/transport/webrtc/webrtc.go @@ -0,0 +1,15 @@ +// libp2pwebrtc implements the WebRTC transport for go-libp2p, +// as officially described in https://github.com/libp2p/specs/tree/cfcf0230b2f5f11ed6dd060f97305faa973abed2/webrtc. +// +// Benchmarks on how this transport compares to other transports can be found in +// https://github.com/libp2p/libp2p-go-webrtc-benchmarks. +// +// Entrypoint for the logic of this Transport can be found in `transport.go`, where the WebRTC transport is implemented, +// used both by the client for Dialing as well as the server for Listening. Starting from there you should be able to follow +// the logic from start to finish. +// +// In the udpmux subpackage you can find the logic for multiplexing multiple WebRTC (ICE) connections over a single UDP socket. +// +// The pb subpackage contains the protobuf definitions for the signaling protocol used by this transport, +// which is taken verbatim from the "Multiplexing" chapter of the WebRTC spec. +package libp2pwebrtc diff --git a/test-plans/go.mod b/test-plans/go.mod index 03fb0bd08d..93554a2356 100644 --- a/test-plans/go.mod +++ b/test-plans/go.mod @@ -8,6 +8,18 @@ require ( github.com/multiformats/go-multiaddr v0.11.0 ) +require ( + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -81,15 +93,6 @@ require ( go.uber.org/fx v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect - golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect - google.golang.org/protobuf v1.31.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) From e6d7d9f8c6f791a068132f9bc86f0c81ddd7922d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jun 2023 14:17:25 +0300 Subject: [PATCH 02/86] webrtc: cleanup internal mess --- p2p/transport/webrtc/internal/fingerprint.go | 31 +++++++++++++++++ .../internal/{util_test.go => sdp_test.go} | 0 p2p/transport/webrtc/internal/util.go | 34 ------------------- 3 files changed, 31 insertions(+), 34 deletions(-) rename p2p/transport/webrtc/internal/{util_test.go => sdp_test.go} (100%) delete mode 100644 p2p/transport/webrtc/internal/util.go diff --git a/p2p/transport/webrtc/internal/fingerprint.go b/p2p/transport/webrtc/internal/fingerprint.go index bd8305ef14..4472f320c8 100644 --- a/p2p/transport/webrtc/internal/fingerprint.go +++ b/p2p/transport/webrtc/internal/fingerprint.go @@ -4,6 +4,13 @@ import ( "crypto" "crypto/x509" "errors" + + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" + + ma "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multibase" + mh "github.com/multiformats/go-multihash" + "github.com/pion/webrtc/v3" ) // Fingerprint is forked from pion to avoid bytes to string alloc, @@ -24,3 +31,27 @@ func Fingerprint(cert *x509.Certificate, algo crypto.Hash) ([]byte, error) { h.Write(cert.Raw) return h.Sum(nil), nil } + +func DecodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { + remoteFingerprintMultibase, err := maddr.ValueForProtocol(ma.P_CERTHASH) + if err != nil { + return nil, err + } + _, data, err := multibase.Decode(remoteFingerprintMultibase) + if err != nil { + return nil, err + } + return mh.Decode(data) +} + +func EncodeDTLSFingerprint(fp webrtc.DTLSFingerprint) (string, error) { + digest, err := encoding.DecodeInterpersedHexFromASCIIString(fp.Value) + if err != nil { + return "", err + } + encoded, err := mh.Encode(digest, mh.SHA2_256) + if err != nil { + return "", err + } + return multibase.Encode(multibase.Base64url, encoded) +} diff --git a/p2p/transport/webrtc/internal/util_test.go b/p2p/transport/webrtc/internal/sdp_test.go similarity index 100% rename from p2p/transport/webrtc/internal/util_test.go rename to p2p/transport/webrtc/internal/sdp_test.go diff --git a/p2p/transport/webrtc/internal/util.go b/p2p/transport/webrtc/internal/util.go deleted file mode 100644 index 2bde1988e8..0000000000 --- a/p2p/transport/webrtc/internal/util.go +++ /dev/null @@ -1,34 +0,0 @@ -package internal - -import ( - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" - ma "github.com/multiformats/go-multiaddr" - "github.com/multiformats/go-multibase" - mh "github.com/multiformats/go-multihash" - - "github.com/pion/webrtc/v3" -) - -func DecodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { - remoteFingerprintMultibase, err := maddr.ValueForProtocol(ma.P_CERTHASH) - if err != nil { - return nil, err - } - _, data, err := multibase.Decode(remoteFingerprintMultibase) - if err != nil { - return nil, err - } - return mh.Decode(data) -} - -func EncodeDTLSFingerprint(fp webrtc.DTLSFingerprint) (string, error) { - digest, err := encoding.DecodeInterpersedHexFromASCIIString(fp.Value) - if err != nil { - return "", err - } - encoded, err := mh.Encode(digest, mh.SHA2_256) - if err != nil { - return "", err - } - return multibase.Encode(multibase.Base64url, encoded) -} From 8813f040e976fb9853160d2e123ec639d23c20fd Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jun 2023 14:31:15 +0300 Subject: [PATCH 03/86] webrtc: clean up messy hex table used for SDP encoding --- p2p/transport/webrtc/internal/encoding/hex.go | 37 ++++++++----------- .../webrtc/internal/encoding/hex_test.go | 13 ++++--- p2p/transport/webrtc/internal/sdp.go | 4 +- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/p2p/transport/webrtc/internal/encoding/hex.go b/p2p/transport/webrtc/internal/encoding/hex.go index a08c5cf20c..03d92c3935 100644 --- a/p2p/transport/webrtc/internal/encoding/hex.go +++ b/p2p/transport/webrtc/internal/encoding/hex.go @@ -9,7 +9,6 @@ package encoding import ( "encoding/hex" "errors" - "strings" ) // EncodeInterspersedHex encodes a byte slice into a string of hex characters, @@ -17,28 +16,25 @@ import ( // // Example: { 0x01, 0x02, 0x03 } -> "01:02:03" func EncodeInterspersedHex(src []byte) string { - var builder strings.Builder - EncodeInterspersedHexToBuilder(src, &builder) - return builder.String() -} - -// EncodeInterspersedHexToBuilder encodes a byte slice into a of hex characters, -// separating each encoded byte with a colon (':'). String is written to the builder. -// -// Example: { 0x01, 0x02, 0x03 } -> "01:02:03" -func EncodeInterspersedHexToBuilder(src []byte, builder *strings.Builder) { if len(src) == 0 { - return + return "" } - builder.Grow(len(src)*3 - 1) - v := src[0] - builder.WriteByte(hextable[v>>4]) - builder.WriteByte(hextable[v&0x0f]) - for _, v = range src[1:] { - builder.WriteByte(':') - builder.WriteByte(hextable[v>>4]) - builder.WriteByte(hextable[v&0x0f]) + s := hex.EncodeToString(src) + n := len(s) + // Determine number of colons + colons := n / 2 + if n%2 == 0 { + colons-- + } + buffer := make([]byte, n+colons) + + for i, j := 0, 0; i < n; i, j = i+2, j+3 { + copy(buffer[j:j+2], s[i:i+2]) + if j+3 < len(buffer) { + buffer[j+2] = ':' + } } + return string(buffer) } // DecodeInterspersedHex decodes a byte slice string of hex characters into a byte slice, @@ -140,7 +136,6 @@ var ( ) const ( - hextable = "0123456789abcdef" reverseHexTable = "" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + diff --git a/p2p/transport/webrtc/internal/encoding/hex_test.go b/p2p/transport/webrtc/internal/encoding/hex_test.go index 71f83647c1..a2a46166b4 100644 --- a/p2p/transport/webrtc/internal/encoding/hex_test.go +++ b/p2p/transport/webrtc/internal/encoding/hex_test.go @@ -14,12 +14,13 @@ func TestEncodeInterspersedHex(t *testing.T) { require.Equal(t, "ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad", EncodeInterspersedHex(b)) } -func TestEncodeInterspersedHexToBuilder(t *testing.T) { - b, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") - require.NoError(t, err) - var builder strings.Builder - EncodeInterspersedHexToBuilder(b, &builder) - require.Equal(t, "ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad", builder.String()) +func BenchmarkEncodeInterspersedHex(b *testing.B) { + data, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + require.NoError(b, err) + + for i := 0; i < b.N; i++ { + EncodeInterspersedHex(data) + } } func TestDecodeInterpersedHexStringLowerCase(t *testing.T) { diff --git a/p2p/transport/webrtc/internal/sdp.go b/p2p/transport/webrtc/internal/sdp.go index 9372ba97e4..6d8a0e4b12 100644 --- a/p2p/transport/webrtc/internal/sdp.go +++ b/p2p/transport/webrtc/internal/sdp.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" - multihash "github.com/multiformats/go-multihash" + "github.com/multiformats/go-multihash" ) // clientSDP describes an SDP format string which can be used @@ -85,7 +85,7 @@ func RenderServerSDP(addr *net.UDPAddr, ufrag string, fingerprint multihash.Deco builder.Grow(len(fingerprint.Digest)*3 + 8) builder.WriteString(sdpString) builder.WriteByte(' ') - encoding.EncodeInterspersedHexToBuilder(fingerprint.Digest, &builder) + builder.WriteString(encoding.EncodeInterspersedHex(fingerprint.Digest)) fp := builder.String() return fmt.Sprintf( From 49148cf3d6e412c15dba5ce61be8d3ca195fd38f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jun 2023 14:51:43 +0300 Subject: [PATCH 04/86] webrtc: clean up messy SDP decoding --- p2p/transport/webrtc/internal/encoding/hex.go | 128 +++--------------- .../webrtc/internal/encoding/hex_test.go | 43 +++--- p2p/transport/webrtc/internal/fingerprint.go | 2 +- p2p/transport/webrtc/transport.go | 2 +- 4 files changed, 47 insertions(+), 128 deletions(-) diff --git a/p2p/transport/webrtc/internal/encoding/hex.go b/p2p/transport/webrtc/internal/encoding/hex.go index 03d92c3935..bf5b235c26 100644 --- a/p2p/transport/webrtc/internal/encoding/hex.go +++ b/p2p/transport/webrtc/internal/encoding/hex.go @@ -37,120 +37,34 @@ func EncodeInterspersedHex(src []byte) string { return string(buffer) } -// DecodeInterspersedHex decodes a byte slice string of hex characters into a byte slice, -// where the hex characters are expected to be separated by a colon (':'). -// -// Example: {'0', '1', ':', '0', '2', ':', '0', '3'} -> { 0x01, 0x02, 0x03 } -func DecodeInterspersedHex(src []byte) ([]byte, error) { - if len(src) == 0 { - return []byte{}, nil - } - if len(src) < 2 { - return nil, hex.ErrLength - } - - dst := make([]byte, (len(src)+1)/3) - i, j := 0, 1 - for ; j < len(src); j += 3 { // jump one extra byte for the separator (:) - p := src[j-1] - q := src[j] - if j+1 < len(src) && src[j+1] != ':' { - return nil, errUnexpectedIntersperseHexChar - } - - a := reverseHexTable[p] - b := reverseHexTable[q] - if a > 0x0f { - return nil, hex.InvalidByteError(p) - } - if b > 0x0f { - return nil, hex.InvalidByteError(q) - } - dst[i] = (a << 4) | b - i++ - } - if (len(src)+1)%3 != 0 { - if len(src)%3 == 0 { - j -= 1 - } - // Check for invalid char before reporting bad length, - // since the invalid char (if present) is an earlier problem. - if reverseHexTable[src[j-1]] > 0x0f { - return nil, hex.InvalidByteError(src[j-1]) - } - return nil, hex.ErrLength - } - return dst[:i], nil -} +var errUnexpectedIntersperseHexChar = errors.New("unexpected character in interspersed hex string") -// DecodeInterpersedHexFromASCIIString decodes an ASCII string of hex characters into a byte slice, +// DecodeInterspersedHexFromASCIIString decodes an ASCII string of hex characters into a byte slice, // where the hex characters are expected to be separated by a colon (':'). // // NOTE that this function returns an error in case the input string contains non-ASCII characters. // // Example: "01:02:03" -> { 0x01, 0x02, 0x03 } -func DecodeInterpersedHexFromASCIIString(src string) ([]byte, error) { - if len(src) == 0 { - return []byte{}, nil - } - if len(src) < 2 { - return nil, hex.ErrLength - } - - dst := make([]byte, (len(src)+1)/3) - i, j := 0, 1 - for ; j < len(src); j += 3 { // jump one extra byte for the separator (:) - p := src[j-1] - q := src[j] - if j+1 < len(src) && src[j+1] != ':' { - return nil, errUnexpectedIntersperseHexChar - } - - a := reverseHexTable[p] - b := reverseHexTable[q] - if a > 0x0f { - return nil, hex.InvalidByteError(p) - } - if b > 0x0f { - return nil, hex.InvalidByteError(q) +func DecodeInterspersedHexFromASCIIString(s string) ([]byte, error) { + n := len(s) + buffer := make([]byte, n/3*2+n%3) + j := 0 + for i := 0; i < n; i++ { + if i%3 == 2 { + if s[i] != ':' { + return nil, errUnexpectedIntersperseHexChar + } + } else { + if s[i] == ':' { + return nil, errUnexpectedIntersperseHexChar + } + buffer[j] = s[i] + j++ } - dst[i] = (a << 4) | b - i++ } - if (len(src)+1)%3 != 0 { - if len(src)%3 == 0 { - j -= 1 - } - // Check for invalid char before reporting bad length, - // since the invalid char (if present) is an earlier problem. - if reverseHexTable[src[j-1]] > 0x0f { - return nil, hex.InvalidByteError(src[j-1]) - } - return nil, hex.ErrLength + dst := make([]byte, hex.DecodedLen(len(buffer))) + if _, err := hex.Decode(dst, buffer); err != nil { + return nil, err } - return dst[:i], nil + return dst, nil } - -var ( - errUnexpectedIntersperseHexChar = errors.New("unexpected character in interspersed hex string") -) - -const ( - reverseHexTable = "" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" + - "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" -) diff --git a/p2p/transport/webrtc/internal/encoding/hex_test.go b/p2p/transport/webrtc/internal/encoding/hex_test.go index a2a46166b4..6f8bcb75f7 100644 --- a/p2p/transport/webrtc/internal/encoding/hex_test.go +++ b/p2p/transport/webrtc/internal/encoding/hex_test.go @@ -24,37 +24,44 @@ func BenchmarkEncodeInterspersedHex(b *testing.B) { } func TestDecodeInterpersedHexStringLowerCase(t *testing.T) { - b, err := DecodeInterpersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") + b, err := DecodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func TestDecodeInterpersedHexStringMixedCase(t *testing.T) { - b, err := DecodeInterpersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") + b, err := DecodeInterspersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func TestDecodeInterpersedHexStringOneByte(t *testing.T) { - b, err := DecodeInterpersedHexFromASCIIString("ba") + b, err := DecodeInterspersedHexFromASCIIString("ba") require.NoError(t, err) require.Equal(t, "ba", hex.EncodeToString(b)) } func TestDecodeInterpersedHexBytesLowerCase(t *testing.T) { - b, err := DecodeInterspersedHex([]byte("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad")) + b, err := DecodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } +func BenchmarkDecode(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := DecodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") + require.NoError(b, err) + } +} + func TestDecodeInterpersedHexBytesMixedCase(t *testing.T) { - b, err := DecodeInterspersedHex([]byte("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad")) + b, err := DecodeInterspersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func TestDecodeInterpersedHexBytesOneByte(t *testing.T) { - b, err := DecodeInterspersedHex([]byte("ba")) + b, err := DecodeInterspersedHexFromASCIIString("ba") require.NoError(t, err) require.Equal(t, "ba", hex.EncodeToString(b)) } @@ -64,49 +71,47 @@ func TestEncodeInterperseHexNilSlice(t *testing.T) { require.Equal(t, "", EncodeInterspersedHex([]byte{})) } -func TestDecodeInterspersedHexNilSlice(t *testing.T) { - for _, v := range [][]byte{nil, {}} { - b, err := DecodeInterspersedHex(v) - require.NoError(t, err) - require.Equal(t, []byte{}, b) - } +func TestDecodeInterspersedHexEmpty(t *testing.T) { + b, err := DecodeInterspersedHexFromASCIIString("") + require.NoError(t, err) + require.Equal(t, []byte{}, b) } func TestDecodeInterpersedHexFromASCIIStringEmpty(t *testing.T) { - b, err := DecodeInterpersedHexFromASCIIString("") + b, err := DecodeInterspersedHexFromASCIIString("") require.NoError(t, err) require.Equal(t, []byte{}, b) } func TestDecodeInterpersedHexInvalid(t *testing.T) { for _, v := range []string{"0", "0000", "000"} { - _, err := DecodeInterspersedHex([]byte(v)) + _, err := DecodeInterspersedHexFromASCIIString(v) require.Error(t, err) } } func TestDecodeInterpersedHexValid(t *testing.T) { - b, err := DecodeInterspersedHex([]byte("00")) + b, err := DecodeInterspersedHexFromASCIIString("00") require.NoError(t, err) require.Equal(t, []byte{0}, b) } func TestDecodeInterpersedHexFromASCIIStringInvalid(t *testing.T) { for _, v := range []string{"0", "0000", "000"} { - _, err := DecodeInterpersedHexFromASCIIString(v) + _, err := DecodeInterspersedHexFromASCIIString(v) require.Error(t, err) } } func TestDecodeInterpersedHexFromASCIIStringValid(t *testing.T) { - b, err := DecodeInterpersedHexFromASCIIString("00") + b, err := DecodeInterspersedHexFromASCIIString("00") require.NoError(t, err) require.Equal(t, []byte{0}, b) } func FuzzInterpersedHex(f *testing.F) { f.Fuzz(func(t *testing.T, b []byte) { - decoded, err := DecodeInterspersedHex(b) + decoded, err := DecodeInterspersedHexFromASCIIString(string(b)) if err != nil { return } @@ -117,7 +122,7 @@ func FuzzInterpersedHex(f *testing.F) { func FuzzInterspersedHexASCII(f *testing.F) { f.Fuzz(func(t *testing.T, s string) { - decoded, err := DecodeInterpersedHexFromASCIIString(s) + decoded, err := DecodeInterspersedHexFromASCIIString(s) if err != nil { return } diff --git a/p2p/transport/webrtc/internal/fingerprint.go b/p2p/transport/webrtc/internal/fingerprint.go index 4472f320c8..470f6b0071 100644 --- a/p2p/transport/webrtc/internal/fingerprint.go +++ b/p2p/transport/webrtc/internal/fingerprint.go @@ -45,7 +45,7 @@ func DecodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { } func EncodeDTLSFingerprint(fp webrtc.DTLSFingerprint) (string, error) { - digest, err := encoding.DecodeInterpersedHexFromASCIIString(fp.Value) + digest, err := encoding.DecodeInterspersedHexFromASCIIString(fp.Value) if err != nil { return "", err } diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index cbe8ba7548..68d7be7a6c 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -448,7 +448,7 @@ func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash return nil, err } - localFpBytes, err := encoding.DecodeInterpersedHexFromASCIIString(localFp.Value) + localFpBytes, err := encoding.DecodeInterspersedHexFromASCIIString(localFp.Value) if err != nil { return nil, err } From ca7e2561641c0384529750bb6ba8f10a1f6e27cf Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jun 2023 15:02:56 +0300 Subject: [PATCH 05/86] webrtc: dissolve messy internal packages --- .../webrtc/{internal => }/fingerprint.go | 20 ++++----- .../webrtc/{internal/encoding => }/hex.go | 10 ++--- .../{internal/encoding => }/hex_test.go | 44 +++++++++---------- p2p/transport/webrtc/listener.go | 3 +- p2p/transport/webrtc/{internal => }/sdp.go | 13 +++--- .../webrtc/{internal => }/sdp_test.go | 10 ++--- p2p/transport/webrtc/transport.go | 16 +++---- 7 files changed, 54 insertions(+), 62 deletions(-) rename p2p/transport/webrtc/{internal => }/fingerprint.go (60%) rename p2p/transport/webrtc/{internal/encoding => }/hex.go (86%) rename p2p/transport/webrtc/{internal/encoding => }/hex_test.go (73%) rename p2p/transport/webrtc/{internal => }/sdp.go (88%) rename p2p/transport/webrtc/{internal => }/sdp_test.go (93%) diff --git a/p2p/transport/webrtc/internal/fingerprint.go b/p2p/transport/webrtc/fingerprint.go similarity index 60% rename from p2p/transport/webrtc/internal/fingerprint.go rename to p2p/transport/webrtc/fingerprint.go index 470f6b0071..fd715865a8 100644 --- a/p2p/transport/webrtc/internal/fingerprint.go +++ b/p2p/transport/webrtc/fingerprint.go @@ -1,27 +1,23 @@ -package internal +package libp2pwebrtc import ( "crypto" "crypto/x509" "errors" - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" - ma "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" "github.com/pion/webrtc/v3" ) -// Fingerprint is forked from pion to avoid bytes to string alloc, +// parseFingerprint is forked from pion to avoid bytes to string alloc, // and to avoid the entire hex interspersing when we do not need it anyway -var ( - errHashUnavailable = errors.New("fingerprint: hash algorithm is not linked into the binary") -) +var errHashUnavailable = errors.New("fingerprint: hash algorithm is not linked into the binary") -// Fingerprint creates a fingerprint for a certificate using the specified hash algorithm -func Fingerprint(cert *x509.Certificate, algo crypto.Hash) ([]byte, error) { +// parseFingerprint creates a fingerprint for a certificate using the specified hash algorithm +func parseFingerprint(cert *x509.Certificate, algo crypto.Hash) ([]byte, error) { if !algo.Available() { return nil, errHashUnavailable } @@ -32,7 +28,7 @@ func Fingerprint(cert *x509.Certificate, algo crypto.Hash) ([]byte, error) { return h.Sum(nil), nil } -func DecodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { +func decodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { remoteFingerprintMultibase, err := maddr.ValueForProtocol(ma.P_CERTHASH) if err != nil { return nil, err @@ -44,8 +40,8 @@ func DecodeRemoteFingerprint(maddr ma.Multiaddr) (*mh.DecodedMultihash, error) { return mh.Decode(data) } -func EncodeDTLSFingerprint(fp webrtc.DTLSFingerprint) (string, error) { - digest, err := encoding.DecodeInterspersedHexFromASCIIString(fp.Value) +func encodeDTLSFingerprint(fp webrtc.DTLSFingerprint) (string, error) { + digest, err := decodeInterspersedHexFromASCIIString(fp.Value) if err != nil { return "", err } diff --git a/p2p/transport/webrtc/internal/encoding/hex.go b/p2p/transport/webrtc/hex.go similarity index 86% rename from p2p/transport/webrtc/internal/encoding/hex.go rename to p2p/transport/webrtc/hex.go index bf5b235c26..482036540a 100644 --- a/p2p/transport/webrtc/internal/encoding/hex.go +++ b/p2p/transport/webrtc/hex.go @@ -1,4 +1,4 @@ -package encoding +package libp2pwebrtc // The code in this file is adapted from the Go standard library's hex package. // As found in https://cs.opensource.google/go/go/+/refs/tags/go1.20.2:src/encoding/hex/hex.go @@ -11,11 +11,11 @@ import ( "errors" ) -// EncodeInterspersedHex encodes a byte slice into a string of hex characters, +// encodeInterspersedHex encodes a byte slice into a string of hex characters, // separating each encoded byte with a colon (':'). // // Example: { 0x01, 0x02, 0x03 } -> "01:02:03" -func EncodeInterspersedHex(src []byte) string { +func encodeInterspersedHex(src []byte) string { if len(src) == 0 { return "" } @@ -39,13 +39,13 @@ func EncodeInterspersedHex(src []byte) string { var errUnexpectedIntersperseHexChar = errors.New("unexpected character in interspersed hex string") -// DecodeInterspersedHexFromASCIIString decodes an ASCII string of hex characters into a byte slice, +// decodeInterspersedHexFromASCIIString decodes an ASCII string of hex characters into a byte slice, // where the hex characters are expected to be separated by a colon (':'). // // NOTE that this function returns an error in case the input string contains non-ASCII characters. // // Example: "01:02:03" -> { 0x01, 0x02, 0x03 } -func DecodeInterspersedHexFromASCIIString(s string) ([]byte, error) { +func decodeInterspersedHexFromASCIIString(s string) ([]byte, error) { n := len(s) buffer := make([]byte, n/3*2+n%3) j := 0 diff --git a/p2p/transport/webrtc/internal/encoding/hex_test.go b/p2p/transport/webrtc/hex_test.go similarity index 73% rename from p2p/transport/webrtc/internal/encoding/hex_test.go rename to p2p/transport/webrtc/hex_test.go index 6f8bcb75f7..c8a7147498 100644 --- a/p2p/transport/webrtc/internal/encoding/hex_test.go +++ b/p2p/transport/webrtc/hex_test.go @@ -1,4 +1,4 @@ -package encoding +package libp2pwebrtc import ( "encoding/hex" @@ -11,7 +11,7 @@ import ( func TestEncodeInterspersedHex(t *testing.T) { b, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") require.NoError(t, err) - require.Equal(t, "ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad", EncodeInterspersedHex(b)) + require.Equal(t, "ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad", encodeInterspersedHex(b)) } func BenchmarkEncodeInterspersedHex(b *testing.B) { @@ -19,114 +19,114 @@ func BenchmarkEncodeInterspersedHex(b *testing.B) { require.NoError(b, err) for i := 0; i < b.N; i++ { - EncodeInterspersedHex(data) + encodeInterspersedHex(data) } } func TestDecodeInterpersedHexStringLowerCase(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") + b, err := decodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func TestDecodeInterpersedHexStringMixedCase(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") + b, err := decodeInterspersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func TestDecodeInterpersedHexStringOneByte(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("ba") + b, err := decodeInterspersedHexFromASCIIString("ba") require.NoError(t, err) require.Equal(t, "ba", hex.EncodeToString(b)) } func TestDecodeInterpersedHexBytesLowerCase(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") + b, err := decodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func BenchmarkDecode(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := DecodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") + _, err := decodeInterspersedHexFromASCIIString("ba:78:16:bf:8f:01:cf:ea:41:41:40:de:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:ff:61:f2:00:15:ad") require.NoError(b, err) } } func TestDecodeInterpersedHexBytesMixedCase(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") + b, err := decodeInterspersedHexFromASCIIString("Ba:78:16:BF:8F:01:cf:ea:41:41:40:De:5d:ae:22:23:b0:03:61:a3:96:17:7a:9c:b4:10:FF:61:f2:00:15:ad") require.NoError(t, err) require.Equal(t, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", hex.EncodeToString(b)) } func TestDecodeInterpersedHexBytesOneByte(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("ba") + b, err := decodeInterspersedHexFromASCIIString("ba") require.NoError(t, err) require.Equal(t, "ba", hex.EncodeToString(b)) } func TestEncodeInterperseHexNilSlice(t *testing.T) { - require.Equal(t, "", EncodeInterspersedHex(nil)) - require.Equal(t, "", EncodeInterspersedHex([]byte{})) + require.Equal(t, "", encodeInterspersedHex(nil)) + require.Equal(t, "", encodeInterspersedHex([]byte{})) } func TestDecodeInterspersedHexEmpty(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("") + b, err := decodeInterspersedHexFromASCIIString("") require.NoError(t, err) require.Equal(t, []byte{}, b) } func TestDecodeInterpersedHexFromASCIIStringEmpty(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("") + b, err := decodeInterspersedHexFromASCIIString("") require.NoError(t, err) require.Equal(t, []byte{}, b) } func TestDecodeInterpersedHexInvalid(t *testing.T) { for _, v := range []string{"0", "0000", "000"} { - _, err := DecodeInterspersedHexFromASCIIString(v) + _, err := decodeInterspersedHexFromASCIIString(v) require.Error(t, err) } } func TestDecodeInterpersedHexValid(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("00") + b, err := decodeInterspersedHexFromASCIIString("00") require.NoError(t, err) require.Equal(t, []byte{0}, b) } func TestDecodeInterpersedHexFromASCIIStringInvalid(t *testing.T) { for _, v := range []string{"0", "0000", "000"} { - _, err := DecodeInterspersedHexFromASCIIString(v) + _, err := decodeInterspersedHexFromASCIIString(v) require.Error(t, err) } } func TestDecodeInterpersedHexFromASCIIStringValid(t *testing.T) { - b, err := DecodeInterspersedHexFromASCIIString("00") + b, err := decodeInterspersedHexFromASCIIString("00") require.NoError(t, err) require.Equal(t, []byte{0}, b) } func FuzzInterpersedHex(f *testing.F) { f.Fuzz(func(t *testing.T, b []byte) { - decoded, err := DecodeInterspersedHexFromASCIIString(string(b)) + decoded, err := decodeInterspersedHexFromASCIIString(string(b)) if err != nil { return } - encoded := EncodeInterspersedHex(decoded) + encoded := encodeInterspersedHex(decoded) require.Equal(t, strings.ToLower(string(b)), encoded) }) } func FuzzInterspersedHexASCII(f *testing.F) { f.Fuzz(func(t *testing.T, s string) { - decoded, err := DecodeInterspersedHexFromASCIIString(s) + decoded, err := decodeInterspersedHexFromASCIIString(s) if err != nil { return } - encoded := EncodeInterspersedHex(decoded) + encoded := encodeInterspersedHex(decoded) require.Equal(t, strings.ToLower(s), encoded) }) } diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index be42f820fa..eed6e6cb63 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -12,7 +12,6 @@ import ( "time" "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/udpmux" tpt "github.com/libp2p/go-libp2p/core/transport" @@ -216,7 +215,7 @@ func (l *listener) setupConnection( errC := awaitPeerConnectionOpen(addr.ufrag, pc) // we infer the client sdp from the incoming STUN connectivity check // by setting the ice-ufrag equal to the incoming check. - clientSdpString := internal.RenderClientSDP(addr.raddr, addr.ufrag) + clientSdpString := createClientSDP(addr.raddr, addr.ufrag) clientSdp := webrtc.SessionDescription{SDP: clientSdpString, Type: webrtc.SDPTypeOffer} pc.SetRemoteDescription(clientSdp) diff --git a/p2p/transport/webrtc/internal/sdp.go b/p2p/transport/webrtc/sdp.go similarity index 88% rename from p2p/transport/webrtc/internal/sdp.go rename to p2p/transport/webrtc/sdp.go index 6d8a0e4b12..878b668a18 100644 --- a/p2p/transport/webrtc/internal/sdp.go +++ b/p2p/transport/webrtc/sdp.go @@ -1,4 +1,4 @@ -package internal +package libp2pwebrtc import ( "crypto" @@ -6,7 +6,6 @@ import ( "net" "strings" - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" "github.com/multiformats/go-multihash" ) @@ -32,7 +31,7 @@ a=sctp-port:5000 a=max-message-size:16384 ` -func RenderClientSDP(addr *net.UDPAddr, ufrag string) string { +func createClientSDP(addr *net.UDPAddr, ufrag string) string { ipVersion := "IP4" if addr.IP.To4() == nil { ipVersion = "IP6" @@ -70,7 +69,7 @@ a=candidate:1 1 UDP 1 %[2]s %[3]d typ host a=end-of-candidates ` -func RenderServerSDP(addr *net.UDPAddr, ufrag string, fingerprint multihash.DecodedMultihash) (string, error) { +func createServerSDP(addr *net.UDPAddr, ufrag string, fingerprint multihash.DecodedMultihash) (string, error) { ipVersion := "IP4" if addr.IP.To4() == nil { ipVersion = "IP6" @@ -85,7 +84,7 @@ func RenderServerSDP(addr *net.UDPAddr, ufrag string, fingerprint multihash.Deco builder.Grow(len(fingerprint.Digest)*3 + 8) builder.WriteString(sdpString) builder.WriteByte(' ') - builder.WriteString(encoding.EncodeInterspersedHex(fingerprint.Digest)) + builder.WriteString(encodeInterspersedHex(fingerprint.Digest)) fp := builder.String() return fmt.Sprintf( @@ -98,10 +97,10 @@ func RenderServerSDP(addr *net.UDPAddr, ufrag string, fingerprint multihash.Deco ), nil } -// GetSupportedSDPHash converts a multihash code to the +// getSupportedSDPHash converts a multihash code to the // corresponding crypto.Hash for supported protocols. If a // crypto.Hash cannot be found, it returns `(0, false)` -func GetSupportedSDPHash(code uint64) (crypto.Hash, bool) { +func getSupportedSDPHash(code uint64) (crypto.Hash, bool) { switch code { case multihash.MD5: return crypto.MD5, true diff --git a/p2p/transport/webrtc/internal/sdp_test.go b/p2p/transport/webrtc/sdp_test.go similarity index 93% rename from p2p/transport/webrtc/internal/sdp_test.go rename to p2p/transport/webrtc/sdp_test.go index 20fc9c7874..2e7ac5b3e8 100644 --- a/p2p/transport/webrtc/internal/sdp_test.go +++ b/p2p/transport/webrtc/sdp_test.go @@ -1,4 +1,4 @@ -package internal +package libp2pwebrtc import ( "encoding/hex" @@ -43,7 +43,7 @@ func TestRenderServerSDP(t *testing.T) { ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" fingerprint := testMultihash - sdp, err := RenderServerSDP(addr, ufrag, fingerprint) + sdp, err := createServerSDP(addr, ufrag, fingerprint) require.NoError(t, err) require.Equal(t, expectedServerSDP, sdp) } @@ -68,7 +68,7 @@ a=max-message-size:16384 func TestRenderClientSDP(t *testing.T) { addr := &net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: 37826} ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" - sdp := RenderClientSDP(addr, ufrag) + sdp := createClientSDP(addr, ufrag) require.Equal(t, expectedClientSDP, sdp) } @@ -77,7 +77,7 @@ func BenchmarkRenderClientSDP(b *testing.B) { ufrag := "d2c0fc07-8bb3-42ae-bae2-a6fce8a0b581" for i := 0; i < b.N; i++ { - RenderClientSDP(addr, ufrag) + createClientSDP(addr, ufrag) } } @@ -95,7 +95,7 @@ func BenchmarkRenderServerSDP(b *testing.B) { fingerprint := testMultihash for i := 0; i < b.N; i++ { - RenderServerSDP(addr, ufrag, fingerprint) + createServerSDP(addr, ufrag, fingerprint) } } diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 68d7be7a6c..59ce356298 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -20,16 +20,14 @@ import ( "github.com/libp2p/go-libp2p/core/sec" tpt "github.com/libp2p/go-libp2p/core/transport" "github.com/libp2p/go-libp2p/p2p/security/noise" - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal" - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/internal/encoding" logging "github.com/ipfs/go-log/v2" ma "github.com/multiformats/go-multiaddr" mafmt "github.com/multiformats/go-multiaddr-fmt" manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multihash" - pionlogger "github.com/pion/logging" + pionlogger "github.com/pion/logging" "github.com/pion/webrtc/v3" ) @@ -195,7 +193,7 @@ func (t *WebRTCTransport) listenSocket(socket *net.UDPConn) (tpt.Listener, error return nil, err } - encodedLocalFingerprint, err := internal.EncodeDTLSFingerprint(listenerFingerprint) + encodedLocalFingerprint, err := encodeDTLSFingerprint(listenerFingerprint) if err != nil { return nil, err } @@ -251,11 +249,11 @@ func (t *WebRTCTransport) dial( } }() - remoteMultihash, err := internal.DecodeRemoteFingerprint(remoteMultiaddr) + remoteMultihash, err := decodeRemoteFingerprint(remoteMultiaddr) if err != nil { return nil, fmt.Errorf("decode fingerprint: %w", err) } - remoteHashFunction, ok := internal.GetSupportedSDPHash(remoteMultihash.Code) + remoteHashFunction, ok := getSupportedSDPHash(remoteMultihash.Code) if !ok { return nil, fmt.Errorf("unsupported hash function: %w", nil) } @@ -329,7 +327,7 @@ func (t *WebRTCTransport) dial( return nil, fmt.Errorf("set local description: %w", err) } - answerSDPString, err := internal.RenderServerSDP(raddr, ufrag, *remoteMultihash) + answerSDPString, err := createServerSDP(raddr, ufrag, *remoteMultihash) if err != nil { return nil, fmt.Errorf("render server SDP: %w", err) } @@ -443,12 +441,12 @@ func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash return nil, err } - remoteFpBytes, err := internal.Fingerprint(cert, hash) + remoteFpBytes, err := parseFingerprint(cert, hash) if err != nil { return nil, err } - localFpBytes, err := encoding.DecodeInterspersedHexFromASCIIString(localFp.Value) + localFpBytes, err := decodeInterspersedHexFromASCIIString(localFp.Value) if err != nil { return nil, err } From 3078315bf17ad4dbf7e989bfa5e22429a6697f1b Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jun 2023 15:20:25 +0300 Subject: [PATCH 06/86] webrtc: fix incorrect protocol registration --- p2p/transport/webrtc/transport.go | 4 +- p2p/transport/webrtc/transport_test.go | 93 +++++++++++--------------- 2 files changed, 41 insertions(+), 56 deletions(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 59ce356298..76c6f700f3 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -141,7 +141,7 @@ func New(privKey ic.PrivKey, psk pnet.PSK, gater connmgr.ConnectionGater, rcmgr } func (t *WebRTCTransport) Protocols() []int { - return []int{ma.P_P2P_WEBRTC_DIRECT} + return []int{ma.P_WEBRTC_DIRECT} } func (t *WebRTCTransport) Proxy() bool { @@ -152,7 +152,7 @@ func (t *WebRTCTransport) CanDial(addr ma.Multiaddr) bool { return dialMatcher.Matches(addr) } -var webRTCMultiAddr = ma.StringCast("/p2p-webrtc-direct") +var webRTCMultiAddr = ma.StringCast("/webrtc-direct") func (t *WebRTCTransport) Listen(addr ma.Multiaddr) (tpt.Listener, error) { addr, wrtcComponent := ma.SplitLast(addr) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 1e5de8abaa..ae3c8a6ce2 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -12,16 +12,18 @@ import ( "testing" "time" + "golang.org/x/crypto/sha3" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" ttransport "github.com/libp2p/go-libp2p/p2p/transport/testsuite" - "github.com/multiformats/go-multiaddr" + + ma "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/crypto/sha3" ) func getTransport(t *testing.T, opts ...Option) (*WebRTCTransport, peer.ID) { @@ -38,39 +40,37 @@ func getTransport(t *testing.T, opts ...Option) (*WebRTCTransport, peer.ID) { } var ( - listenerIp net.IP - dialerIp net.IP + listenerIP net.IP + dialerIP net.IP ) func TestMain(m *testing.M) { - listenerIp, dialerIp = getListenerAndDialerIP() + listenerIP, dialerIP = getListenerAndDialerIP() os.Exit(m.Run()) } func TestTransportWebRTC_CanDial(t *testing.T) { tr, _ := getTransport(t) invalid := []string{ - "/ip4/1.2.3.4/udp/1234/p2p-webrtc-direct", - "/dns/test.test/udp/1234/p2p-webrtc-direct", + "/ip4/1.2.3.4/udp/1234/webrtc-direct", + "/dns/test.test/udp/1234/webrtc-direct", } valid := []string{ - "/ip4/1.2.3.4/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", - "/ip6/0:0:0:0:0:0:0:1/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", - "/ip6/::1/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", - "/dns/test.test/udp/1234/p2p-webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/ip4/1.2.3.4/udp/1234/webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/ip6/0:0:0:0:0:0:0:1/udp/1234/webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/ip6/::1/udp/1234/webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", + "/dns/test.test/udp/1234/webrtc-direct/certhash/uEiAsGPzpiPGQzSlVHRXrUCT5EkTV7YFrV4VZ3hpEKTd_zg", } for _, addr := range invalid { - ma, err := multiaddr.NewMultiaddr(addr) - require.NoError(t, err) - require.Equal(t, false, tr.CanDial(ma)) + a := ma.StringCast(addr) + require.Equal(t, false, tr.CanDial(a)) } for _, addr := range valid { - ma, err := multiaddr.NewMultiaddr(addr) - require.NoError(t, err) - require.Equal(t, true, tr.CanDial(ma), addr) + a := ma.StringCast(addr) + require.Equal(t, true, tr.CanDial(a), addr) } } @@ -81,7 +81,7 @@ func TestTransportWebRTC_ListenFailsOnNonWebRTCMultiaddr(t *testing.T) { "/ip4/0.0.0.0/tcp/0/wss", } for _, addr := range testAddrs { - listenMultiaddr, err := multiaddr.NewMultiaddr(addr) + listenMultiaddr, err := ma.NewMultiaddr(addr) require.NoError(t, err) listener, err := tr.Listen(listenMultiaddr) require.Error(t, err) @@ -102,7 +102,7 @@ func TestTransportWebRTC_DialFailsOnUnsupportedHashFunction(t *testing.T) { require.NoError(t, err) return certhash }() - testaddr, err := multiaddr.NewMultiaddr("/ip4/1.2.3.4/udp/1234/p2p-webrtc-direct/certhash/" + certhash) + testaddr, err := ma.NewMultiaddr("/ip4/1.2.3.4/udp/1234/webrtc-direct/certhash/" + certhash) require.NoError(t, err) _, err = tr.Dial(context.Background(), testaddr, "") require.ErrorContains(t, err, "unsupported hash function") @@ -111,8 +111,7 @@ func TestTransportWebRTC_DialFailsOnUnsupportedHashFunction(t *testing.T) { func TestTransportWebRTC_CanListenSingle(t *testing.T) { tr, listeningPeer := getTransport(t) tr1, connectingPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -153,8 +152,7 @@ func TestTransportWebRTC_CanListenMultiple(t *testing.T) { count := 3 tr, listeningPeer := getTransport(t, WithListenerMaxInFlightConnections(uint32(count))) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -197,8 +195,7 @@ func TestTransportWebRTC_CanListenMultiple(t *testing.T) { func TestTransportWebRTC_CanCreateSuccessiveConnections(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) count := 2 @@ -221,8 +218,7 @@ func TestTransportWebRTC_CanCreateSuccessiveConnections(t *testing.T) { func TestTransportWebRTC_ListenerCanCreateStreams(t *testing.T) { tr, listeningPeer := getTransport(t) tr1, connectingPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -264,8 +260,7 @@ func TestTransportWebRTC_ListenerCanCreateStreams(t *testing.T) { func TestTransportWebRTC_DialerCanCreateStreams(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -308,8 +303,7 @@ func TestTransportWebRTC_DialerCanCreateStreams(t *testing.T) { func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { count := 5 tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -371,8 +365,7 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { func TestTransportWebRTC_Deadline(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) tr1, connectingPeer := getTransport(t) @@ -425,8 +418,7 @@ func TestTransportWebRTC_Deadline(t *testing.T) { func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -467,8 +459,7 @@ func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { func TestTransportWebRTC_Read(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -529,8 +520,7 @@ func TestTransportWebRTC_Read(t *testing.T) { func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -580,8 +570,7 @@ func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { func TestTransportWebRTC_RemoteReadsAfterClose2(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -629,8 +618,7 @@ func TestTransportWebRTC_RemoteReadsAfterClose2(t *testing.T) { func TestTransportWebRTC_Close(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -701,8 +689,7 @@ func TestTransportWebRTC_Close(t *testing.T) { func TestTransportWebRTC_ReceiveFlagsAfterReadClosed(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -762,7 +749,7 @@ func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { } tr, listeningPeer := getTransport(t) - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) require.NoError(t, err) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -771,15 +758,15 @@ func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { go listener.Accept() - badMultiaddr, _ := multiaddr.SplitFunc(listener.Multiaddr(), func(component multiaddr.Component) bool { - return component.Protocol().Code == multiaddr.P_CERTHASH + badMultiaddr, _ := ma.SplitFunc(listener.Multiaddr(), func(component ma.Component) bool { + return component.Protocol().Code == ma.P_CERTHASH }) encodedCerthash, err := multihash.Encode(testMultihash.Digest, testMultihash.Code) require.NoError(t, err) badEncodedCerthash, err := multibase.Encode(multibase.Base58BTC, encodedCerthash) require.NoError(t, err) - badCerthash, err := multiaddr.NewMultiaddr(fmt.Sprintf("/certhash/%s", badEncodedCerthash)) + badCerthash, err := ma.NewMultiaddr(fmt.Sprintf("/certhash/%s", badEncodedCerthash)) require.NoError(t, err) badMultiaddr = badMultiaddr.Encapsulate(badCerthash) @@ -797,8 +784,7 @@ func TestTransportWebRTC_StreamResetOnPeerConnectionFailure(t *testing.T) { tr.peerConnectionTimeouts.Failed = 3 * time.Second tr.peerConnectionTimeouts.Keepalive = time.Second - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) lsnr, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -855,8 +841,7 @@ func TestTransportWebRTC_MaxInFlightRequests(t *testing.T) { tr.peerConnectionTimeouts.Disconnect = 8 * time.Second tr.peerConnectionTimeouts.Failed = 10 * time.Second tr.peerConnectionTimeouts.Keepalive = 5 * time.Second - listenMultiaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp)) - require.NoError(t, err) + listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) defer listener.Close() @@ -910,5 +895,5 @@ func TestWebrtcTransport(t *testing.T) { t.Skip("This test does not work for WebRTC due to the way it is setup, see comments for more explanation") ta, _ := getTransport(t) tb, _ := getTransport(t) - ttransport.SubtestTransport(t, ta, tb, fmt.Sprintf("/ip4/%s/udp/0/p2p-webrtc-direct", listenerIp), "peerA") + ttransport.SubtestTransport(t, ta, tb, fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP), "peerA") } From 5176981809d92e1256ac5aaa1cf0053157b0e153 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Jun 2023 15:39:08 +0300 Subject: [PATCH 07/86] webrtc: remove incorrect call to ConnectionGater.InterceptAddrDial This function is called from the swarm, not from every individual transport. --- p2p/transport/webrtc/transport.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 76c6f700f3..34556f0bf0 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -231,12 +231,8 @@ func (t *WebRTCTransport) Dial(ctx context.Context, remoteMultiaddr ma.Multiaddr return conn, nil } -func (t *WebRTCTransport) dial( - ctx context.Context, - scope network.ConnManagementScope, - remoteMultiaddr ma.Multiaddr, - p peer.ID, -) (tConn tpt.CapableConn, err error) { +func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagementScope, remoteMultiaddr ma.Multiaddr, p peer.ID) (tConn tpt.CapableConn, err error) { + fmt.Println(remoteMultiaddr) var pc *webrtc.PeerConnection defer func() { if err != nil { @@ -371,11 +367,6 @@ func (t *WebRTCTransport) dial( return nil, err } - // check with the gater if we can dial - if t.gater != nil && !t.gater.InterceptAddrDial(p, remoteMultiaddr) { - return nil, errors.New("not allowed to dial remote peer") - } - // we can only know the remote public key after the noise handshake, // but need to set up the callbacks on the peerconnection conn, err := newConnection( From 633db08e50fa4465164db63fde0a3ff03d08925a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 13 Jun 2023 14:12:13 +0300 Subject: [PATCH 08/86] webrtc: add missing call to ConnectionGater.InterceptSecured for dialing --- p2p/test/transport/gating_test.go | 2 +- p2p/transport/webrtc/transport.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/p2p/test/transport/gating_test.go b/p2p/test/transport/gating_test.go index 426fc906e5..ed0b6e0945 100644 --- a/p2p/test/transport/gating_test.go +++ b/p2p/test/transport/gating_test.go @@ -97,7 +97,7 @@ func TestInterceptSecuredOutgoing(t *testing.T) { connGater.EXPECT().InterceptPeerDial(h2.ID()).Return(true), connGater.EXPECT().InterceptAddrDial(h2.ID(), gomock.Any()).Return(true), connGater.EXPECT().InterceptSecured(network.DirOutbound, h2.ID(), gomock.Any()).Do(func(_ network.Direction, _ peer.ID, addrs network.ConnMultiaddrs) { - // remove the certhash component from WebTransport addresses + // remove the certhash component from WebTransport and WebRTC addresses require.Equal(t, stripCertHash(h2.Addrs()[0]).String(), addrs.RemoteMultiaddr().String()) }), ) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 34556f0bf0..a1ebb8faee 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -232,7 +232,6 @@ func (t *WebRTCTransport) Dial(ctx context.Context, remoteMultiaddr ma.Multiaddr } func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagementScope, remoteMultiaddr ma.Multiaddr, p peer.ID) (tConn tpt.CapableConn, err error) { - fmt.Println(remoteMultiaddr) var pc *webrtc.PeerConnection defer func() { if err != nil { @@ -388,6 +387,10 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement if err != nil { return conn, err } + if t.gater != nil && !t.gater.InterceptSecured(network.DirOutbound, p, conn) { + secConn.Close() + return nil, fmt.Errorf("secured connection gated") + } conn.setRemotePublicKey(secConn.RemotePublicKey()) return conn, nil } From 12520be1053318bda9a1f19517838beaac6ab92c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 13 Jun 2023 14:20:59 +0300 Subject: [PATCH 09/86] webrtc: don't include the certhash in Connection.RemoteMultiaddr --- p2p/transport/webrtc/transport.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index a1ebb8faee..c3e13605c9 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -366,6 +366,8 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, err } + remoteMultiaddrWithoutCerthash, _ := ma.SplitFunc(remoteMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) + // we can only know the remote public key after the noise handshake, // but need to set up the callbacks on the peerconnection conn, err := newConnection( @@ -377,7 +379,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement localAddr, p, nil, - remoteMultiaddr, + remoteMultiaddrWithoutCerthash, ) if err != nil { return nil, err From ed1c137c073752dfd2d1c2dc72ffa4ee28b2e931 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 13 Jun 2023 14:26:42 +0300 Subject: [PATCH 10/86] webrtc: add missing call to ConnectionGater.InterceptAccept --- p2p/test/transport/gating_test.go | 6 +++++- p2p/transport/webrtc/listener.go | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/p2p/test/transport/gating_test.go b/p2p/test/transport/gating_test.go index ed0b6e0945..3079007bee 100644 --- a/p2p/test/transport/gating_test.go +++ b/p2p/test/transport/gating_test.go @@ -166,7 +166,11 @@ func TestInterceptAccept(t *testing.T) { h1.Peerstore().AddAddrs(h2.ID(), h2.Addrs(), time.Hour) _, err := h1.NewStream(ctx, h2.ID(), protocol.TestingID) require.Error(t, err) - require.NotErrorIs(t, err, context.DeadlineExceeded) + if _, err := h2.Addrs()[0].ValueForProtocol(ma.P_WEBRTC_DIRECT); err != nil { + // WebRTC rejects connection attempt before an error can be sent to the client. + // This means that the connection attempt will time out. + require.NotErrorIs(t, err, context.DeadlineExceeded) + } }) } } diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index eed6e6cb63..d6aeac1f2b 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -4,6 +4,7 @@ import ( "context" "crypto" "encoding/hex" + "errors" "fmt" "net" "os" @@ -25,7 +26,14 @@ import ( "github.com/pion/webrtc/v3" ) -var _ tpt.Listener = &listener{} +type connMultiaddrs struct { + local, remote ma.Multiaddr +} + +var _ network.ConnMultiaddrs = &connMultiaddrs{} + +func (c *connMultiaddrs) LocalMultiaddr() ma.Multiaddr { return c.local } +func (c *connMultiaddrs) RemoteMultiaddr() ma.Multiaddr { return c.remote } const ( candidateSetupTimeout = 20 * time.Second @@ -65,6 +73,8 @@ type listener struct { cancel context.CancelFunc } +var _ tpt.Listener = &listener{} + func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.PacketConn, config webrtc.Configuration) (*listener, error) { localFingerprints, err := config.Certificates[0].GetFingerprints() if err != nil { @@ -148,6 +158,14 @@ func (l *listener) handleCandidate(ctx context.Context, addr *candidateAddr) (tp if err != nil { return nil, err } + if l.transport.gater != nil { + localAddr, _ := ma.SplitFunc(l.localMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) + if !l.transport.gater.InterceptAccept(&connMultiaddrs{local: localAddr, remote: remoteMultiaddr}) { + // The connection attempt is rejected before we can send the client an error. + // This means that the connection attempt will time out. + return nil, errors.New("connection gated") + } + } scope, err := l.transport.rcmgr.OpenConnection(network.DirInbound, false, remoteMultiaddr) if err != nil { return nil, err From e5fc8e702627c0098c24c36824d8edd585e8744d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 13 Jun 2023 14:36:09 +0300 Subject: [PATCH 11/86] webrtc: add missing call to ConnectionGater.InterceptSecured --- p2p/transport/webrtc/listener.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index d6aeac1f2b..f699bf72cf 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -175,6 +175,10 @@ func (l *listener) handleCandidate(ctx context.Context, addr *candidateAddr) (tp scope.Done() return nil, err } + if l.transport.gater != nil && !l.transport.gater.InterceptSecured(network.DirInbound, conn.RemotePeer(), conn) { + conn.Close() + return nil, errors.New("connection gated") + } return conn, nil } @@ -262,6 +266,8 @@ func (l *listener) setupConnection( return nil, err } + localMultiaddrWithoutCerthash, _ := ma.SplitFunc(l.localMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) + handshakeChannel := newStream(nil, rawDatachannel, rwc, l.localAddr, addr.raddr) // The connection is instantiated before performing the Noise handshake. This is // to handle the case where the remote is faster and attempts to initiate a stream @@ -272,7 +278,7 @@ func (l *listener) setupConnection( l.transport, scope, l.transport.localPeerId, - l.localMultiaddr, + localMultiaddrWithoutCerthash, "", // remotePeer nil, // remoteKey remoteMultiaddr, From 6ccf888bcd70b37df0931ed30a53dce56a181ab8 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 13 Jun 2023 14:43:32 +0300 Subject: [PATCH 12/86] webrtc: fix rcmgr OpenConnection call claiming that it uses a fd --- p2p/test/transport/rcmgr_test.go | 6 +++--- p2p/transport/webrtc/transport.go | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/p2p/test/transport/rcmgr_test.go b/p2p/test/transport/rcmgr_test.go index 378d1f9bab..239e474db7 100644 --- a/p2p/test/transport/rcmgr_test.go +++ b/p2p/test/transport/rcmgr_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - gomock "github.com/golang/mock/gomock" + "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" mocknetwork "github.com/libp2p/go-libp2p/core/network/mocks" @@ -55,7 +55,7 @@ func TestResourceManagerIsUsed(t *testing.T) { } expectFd := true - if strings.Contains(tc.Name, "QUIC") || strings.Contains(tc.Name, "WebTransport") { + if strings.Contains(tc.Name, "QUIC") || strings.Contains(tc.Name, "WebTransport") || strings.Contains(tc.Name, "WebRTC") { expectFd = false } @@ -86,7 +86,7 @@ func TestResourceManagerIsUsed(t *testing.T) { } return nil }) - connScope.EXPECT().Done() + connScope.EXPECT().Done().MinTimes(1) var allStreamsDone sync.WaitGroup diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index c3e13605c9..5ec3938d32 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -214,12 +214,11 @@ func (t *WebRTCTransport) listenSocket(socket *net.UDPConn) (tpt.Listener, error } func (t *WebRTCTransport) Dial(ctx context.Context, remoteMultiaddr ma.Multiaddr, p peer.ID) (tpt.CapableConn, error) { - scope, err := t.rcmgr.OpenConnection(network.DirOutbound, true, remoteMultiaddr) + scope, err := t.rcmgr.OpenConnection(network.DirOutbound, false, remoteMultiaddr) if err != nil { return nil, err } - err = scope.SetPeer(p) - if err != nil { + if err := scope.SetPeer(p); err != nil { scope.Done() return nil, err } From 233addbe03d5bd95d436a534a09023a7fd69c7af Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jun 2023 11:29:44 -0700 Subject: [PATCH 13/86] webrtc: fix dial matcher --- p2p/transport/webrtc/transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 5ec3938d32..1b4a5cdf85 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -33,7 +33,7 @@ import ( var log = logging.Logger("webrtc-transport") -var dialMatcher = mafmt.And(mafmt.UDP, mafmt.Base(ma.P_P2P_WEBRTC_DIRECT), mafmt.Base(ma.P_CERTHASH)) +var dialMatcher = mafmt.And(mafmt.UDP, mafmt.Base(ma.P_WEBRTC_DIRECT), mafmt.Base(ma.P_CERTHASH)) const ( // handshakeChannelNegotiated is used to specify that the From b971d4c5749b37930fca920c0830dffb42f1b0d7 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jun 2023 11:47:44 -0700 Subject: [PATCH 14/86] webrtc: rewrite super messy in-flight limiting test case This test took ~18s. That's not ok. --- p2p/transport/webrtc/transport_test.go | 44 +++++++------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index ae3c8a6ce2..1e582eeb57 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -833,54 +833,34 @@ func TestTransportWebRTC_StreamResetOnPeerConnectionFailure(t *testing.T) { } } -func TestTransportWebRTC_MaxInFlightRequests(t *testing.T) { - count := uint32(3) +func TestMaxInFlightRequests(t *testing.T) { + const count = 3 tr, listeningPeer := getTransport(t, WithListenerMaxInFlightConnections(count), ) - tr.peerConnectionTimeouts.Disconnect = 8 * time.Second - tr.peerConnectionTimeouts.Failed = 10 * time.Second - tr.peerConnectionTimeouts.Keepalive = 5 * time.Second - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) - listener, err := tr.Listen(listenMultiaddr) + listener, err := tr.Listen(ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP))) require.NoError(t, err) defer listener.Close() - dialerCount := count + 2 - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var wg sync.WaitGroup - start := make(chan struct{}) - ready := make(chan struct{}, dialerCount) - var success uint32 - for i := 0; uint32(i) < dialerCount; i++ { + var success, fails atomic.Int32 + for i := 0; i < count+1; i++ { wg.Add(1) go func() { defer wg.Done() dialer, _ := getTransport(t) - dialer.peerConnectionTimeouts.Disconnect = 8 * time.Second - dialer.peerConnectionTimeouts.Failed = 10 * time.Second - dialer.peerConnectionTimeouts.Keepalive = 5 * time.Second - ready <- struct{}{} - <-start - _, err := dialer.Dial(ctx, listener.Multiaddr(), listeningPeer) - if err == nil { - atomic.AddUint32(&success, 1) + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + if _, err := dialer.Dial(ctx, listener.Multiaddr(), listeningPeer); err == nil { + success.Add(1) } else { - t.Logf("dial error: %v", err) + fails.Add(1) } }() } - - for i := 0; uint32(i) < dialerCount; i++ { - <-ready - } - close(start) wg.Wait() - successCount := atomic.LoadUint32(&success) - require.Equal(t, count, successCount) + require.Equal(t, count, int(success.Load()), "expected exactly 3 dial successes") + require.Equal(t, 1, int(fails.Load()), "expected exactly 1 dial failure") } // TestWebrtcTransport implements the standard go-libp2p transport test. From e00968538f2ea4eab9a9712f4e4efee2454e878c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jun 2023 11:49:07 -0700 Subject: [PATCH 15/86] webrtc: speed up messy connection failure test --- p2p/transport/webrtc/transport_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 1e582eeb57..b0f9669aab 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -778,20 +778,20 @@ func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { require.ErrorContains(t, err, "failed") } -func TestTransportWebRTC_StreamResetOnPeerConnectionFailure(t *testing.T) { +func TestStreamResetOnPeerConnectionFailure(t *testing.T) { tr, listeningPeer := getTransport(t) - tr.peerConnectionTimeouts.Disconnect = 2 * time.Second - tr.peerConnectionTimeouts.Failed = 3 * time.Second - tr.peerConnectionTimeouts.Keepalive = time.Second + tr.peerConnectionTimeouts.Disconnect = 100 * time.Millisecond + tr.peerConnectionTimeouts.Failed = 150 * time.Millisecond + tr.peerConnectionTimeouts.Keepalive = 50 * time.Millisecond listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) lsnr, err := tr.Listen(listenMultiaddr) require.NoError(t, err) tr1, connectingPeer := getTransport(t) - tr1.peerConnectionTimeouts.Disconnect = 2 * time.Second - tr1.peerConnectionTimeouts.Failed = 3 * time.Second - tr1.peerConnectionTimeouts.Keepalive = time.Second + tr1.peerConnectionTimeouts.Disconnect = 100 * time.Millisecond + tr1.peerConnectionTimeouts.Failed = 150 * time.Millisecond + tr1.peerConnectionTimeouts.Keepalive = 50 * time.Millisecond done := make(chan struct{}) go func() { From 65821bc0f4846afcd6b2205ed0980b84d5d10475 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 29 Jun 2023 12:23:50 -0700 Subject: [PATCH 16/86] webrtc: fix convoluted use of TestMain --- p2p/transport/webrtc/transport_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index b0f9669aab..17c8a7b95b 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net" - "os" "sync" "sync/atomic" "testing" @@ -39,14 +38,10 @@ func getTransport(t *testing.T, opts ...Option) (*WebRTCTransport, peer.ID) { return transport, peerID } -var ( - listenerIP net.IP - dialerIP net.IP -) +var listenerIP, dialerIP net.IP -func TestMain(m *testing.M) { +func init() { listenerIP, dialerIP = getListenerAndDialerIP() - os.Exit(m.Run()) } func TestTransportWebRTC_CanDial(t *testing.T) { From 4a75582e1f29389afacf7b1a0547c9742eafd467 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 12:27:39 -0700 Subject: [PATCH 17/86] webrtc: fix messy and flaky TestReceiveFlagsAfterReadClosed --- p2p/transport/webrtc/transport_test.go | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 17c8a7b95b..f1e4f5d0e6 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -682,7 +682,7 @@ func TestTransportWebRTC_Close(t *testing.T) { }) } -func TestTransportWebRTC_ReceiveFlagsAfterReadClosed(t *testing.T) { +func TestReceiveFlagsAfterReadClosed(t *testing.T) { tr, listeningPeer := getTransport(t) listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) listener, err := tr.Listen(listenMultiaddr) @@ -691,44 +691,44 @@ func TestTransportWebRTC_ReceiveFlagsAfterReadClosed(t *testing.T) { tr1, connectingPeer := getTransport(t) done := make(chan struct{}) - var wg sync.WaitGroup - wg.Add(1) go func() { - defer wg.Done() - lconn, err := listener.Accept() require.NoError(t, err) - t.Logf("listener accepted connection") require.Equal(t, connectingPeer, lconn.RemotePeer()) stream, err := lconn.AcceptStream() require.NoError(t, err) - n, err := stream.Read(make([]byte, 10)) + b := make([]byte, 6) + n, err := stream.Read(b) require.NoError(t, err) - require.Equal(t, 10, n) + require.Equal(t, []byte("foobar"), b[:n]) // stop reader - err = stream.Reset() - require.NoError(t, err) + require.NoError(t, stream.Reset()) done <- struct{}{} }() conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) require.NoError(t, err) - t.Logf("dialer opened connection") stream, err := conn.OpenStream(context.Background()) require.NoError(t, err) - err = stream.CloseRead() - require.NoError(t, err) + require.NoError(t, stream.CloseRead()) _, err = stream.Read([]byte{0}) require.ErrorIs(t, err, io.EOF) - _, err = stream.Write(make([]byte, 10)) + _, err = stream.Write([]byte("foobar")) require.NoError(t, err) <-done - _, err = stream.Write(make([]byte, 2*1024*1024)) - // can be an error closed or timeout - require.Error(t, err) - - wg.Wait() + // at some point we should receive the stream reset + start := time.Now() + for { + if _, err := stream.Write([]byte{0x42}); err != nil { + require.ErrorIs(t, err, io.ErrClosedPipe) + break + } + if time.Since(start) > 3*time.Second { + require.Fail(t, "didn't receive stream reset") + } + time.Sleep(time.Millisecond) + } } func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { From 35ff94d5d137af69cffbed2a9529aa5889433a49 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 12:58:25 -0700 Subject: [PATCH 18/86] webrtc: remove unused context from packetQueue.Push --- .../webrtc/udpmux/muxed_connection.go | 2 +- p2p/transport/webrtc/udpmux/packetqueue.go | 2 +- .../webrtc/udpmux/packetqueue_bench_test.go | 5 ++-- .../webrtc/udpmux/packetqueue_test.go | 29 ++++++------------- 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index bafc728b9b..e92897cff2 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -39,7 +39,7 @@ func newMuxedConnection(mux *udpMux, ufrag string, addr net.Addr) *muxedConnecti } func (conn *muxedConnection) Push(buf []byte) error { - return conn.pq.Push(conn.ctx, buf) + return conn.pq.Push(buf) } // Close implements net.PacketConn diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go index 107ee416bc..d168cc7c45 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue.go +++ b/p2p/transport/webrtc/udpmux/packetqueue.go @@ -74,7 +74,7 @@ func (pq *packetQueue) Pop(ctx context.Context, buf []byte) (int, error) { } // Push adds a packet to the packetQueue -func (pq *packetQueue) Push(ctx context.Context, buf []byte) error { +func (pq *packetQueue) Push(buf []byte) error { pq.packetsMux.Lock() defer pq.packetsMux.Unlock() diff --git a/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go b/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go index dbbb3b1bc1..aaf8688d85 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go +++ b/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go @@ -16,7 +16,6 @@ var sizes = []int{ } func BenchmarkQueue(b *testing.B) { - ctx := context.Background() for _, dequeue := range [...]bool{true, false} { for _, input := range sizes { testCase := fmt.Sprintf("enqueue_%d", input) @@ -29,10 +28,10 @@ func BenchmarkQueue(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for k := 0; k < input; k++ { - pq.Push(ctx, pool.Get(255)) + pq.Push(pool.Get(255)) } for k := 0; k < input; k++ { - pq.Pop(ctx, buf) + pq.Pop(context.Background(), buf) } } }) diff --git a/p2p/transport/webrtc/udpmux/packetqueue_test.go b/p2p/transport/webrtc/udpmux/packetqueue_test.go index 7abe0c1a4b..74cfa6b00a 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue_test.go +++ b/p2p/transport/webrtc/udpmux/packetqueue_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - pool "github.com/libp2p/go-buffer-pool" "github.com/stretchr/testify/require" ) @@ -14,15 +13,14 @@ func TestPacketQueue_QueuePacketsForRead(t *testing.T) { defer cancel() pq := newPacketQueue() - pq.Push(ctx, []byte{1, 2, 3}) - pq.Push(ctx, []byte{5, 6, 7, 8}) + pq.Push([]byte{1, 2, 3}) + pq.Push([]byte{5, 6, 7, 8}) - buf := pool.Get(100) - size, err := pq.Pop(ctx, buf) + size, err := pq.Pop(ctx, make([]byte, 6)) require.NoError(t, err) require.Equal(t, size, 3) - size, err = pq.Pop(ctx, buf) + size, err = pq.Pop(ctx, make([]byte, 6)) require.NoError(t, err) require.Equal(t, size, 4) } @@ -32,30 +30,21 @@ func TestPacketQueue_WaitsForData(t *testing.T) { defer cancel() pq := newPacketQueue() - buf := pool.Get(100) timer := time.AfterFunc(200*time.Millisecond, func() { - pq.Push(ctx, []byte{5, 6, 7, 8}) + pq.Push([]byte("foobar")) }) defer timer.Stop() - size, err := pq.Pop(ctx, buf) + size, err := pq.Pop(ctx, make([]byte, 6)) require.NoError(t, err) - require.Equal(t, size, 4) + require.Equal(t, size, 6) } func TestPacketQueue_DropsPacketsWhenQueueIsFull(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - pq := newPacketQueue() for i := 0; i < maxPacketsInQueue; i++ { - buf := pool.Get(255) - err := pq.Push(ctx, buf) - require.NoError(t, err) + require.NoError(t, pq.Push(make([]byte, 10))) } - - buf := pool.Get(255) - err := pq.Push(ctx, buf) - require.ErrorIs(t, err, errTooManyPackets) + require.ErrorIs(t, pq.Push(make([]byte, 10)), errTooManyPackets) } From 8a9dd503d26e4fbbe59b53bd20e2b7f86d7afdcd Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:15:30 -0700 Subject: [PATCH 19/86] webrtc: remove errEmptyPacketQueue --- p2p/transport/webrtc/udpmux/packetqueue.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go index d168cc7c45..8f5bdb4e26 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue.go +++ b/p2p/transport/webrtc/udpmux/packetqueue.go @@ -14,7 +14,6 @@ type packet struct { var ( errTooManyPackets = errors.New("too many packets in queue; dropping") - errEmptyPacketQueue = errors.New("packet queue is empty") errPacketQueueClosed = errors.New("packet queue closed") ) @@ -35,14 +34,17 @@ func newPacketQueue() *packetQueue { // Pop reads a packet from the packetQueue or blocks until // either a packet becomes available or the queue is closed. func (pq *packetQueue) Pop(ctx context.Context, buf []byte) (int, error) { +start: select { case <-pq.packetsCh: pq.packetsMux.Lock() - defer pq.packetsMux.Unlock() if len(pq.packets) == 0 { - return 0, errEmptyPacketQueue + pq.packetsMux.Unlock() + goto start } + + defer pq.packetsMux.Unlock() p := pq.packets[0] n := copy(buf, p.buf) From 75718f20f44a080d0c6c7b57b41d290294d97e1f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:15:56 -0700 Subject: [PATCH 20/86] webrtc: remove nonsensical comment from packetQueue.Pop --- p2p/transport/webrtc/udpmux/packetqueue.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go index 8f5bdb4e26..d0ad81560f 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue.go +++ b/p2p/transport/webrtc/udpmux/packetqueue.go @@ -31,8 +31,7 @@ func newPacketQueue() *packetQueue { } } -// Pop reads a packet from the packetQueue or blocks until -// either a packet becomes available or the queue is closed. +// Pop reads a packet from the packetQueue. func (pq *packetQueue) Pop(ctx context.Context, buf []byte) (int, error) { start: select { @@ -67,9 +66,6 @@ start: } return n, nil - - // It is desired to allow reads of this channel even - // when pq.ctx.Done() is already closed. case <-ctx.Done(): return 0, errPacketQueueClosed } From b2f29445cffc3de2d31c74998c9e3fea43358f32 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:22:33 -0700 Subject: [PATCH 21/86] webrtc: remove option to perform partial reads from packet queue --- p2p/transport/webrtc/udpmux/packetqueue.go | 27 ++++++++-------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go index d0ad81560f..379b0f5220 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue.go +++ b/p2p/transport/webrtc/udpmux/packetqueue.go @@ -45,26 +45,16 @@ start: defer pq.packetsMux.Unlock() p := pq.packets[0] - n := copy(buf, p.buf) - if n == len(p.buf) { - // only move packet from queue if we read all - pq.packets = pq.packets[1:] - pool.Put(p.buf) - } else { - // otherwise we need to keep the packet in the queue - // but do update the buf - pq.packets[0].buf = p.buf[n:] + if n < len(p.buf) { + log.Debugf("short read, had %d, read %d", len(p.buf), n) } - + pq.packets = pq.packets[1:] + pool.Put(p.buf) if len(pq.packets) > 0 { // to make sure a next pop call will work - select { - case pq.packetsCh <- struct{}{}: - default: - } + pq.notify() } - return n, nil case <-ctx.Done(): return 0, errPacketQueueClosed @@ -81,10 +71,13 @@ func (pq *packetQueue) Push(buf []byte) error { } pq.packets = append(pq.packets, packet{buf}) + pq.notify() + return nil +} + +func (pq *packetQueue) notify() { select { case pq.packetsCh <- struct{}{}: default: } - - return nil } From 827d756c0d1aee4d8b66164406d6b9ab9f7af971 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:23:55 -0700 Subject: [PATCH 22/86] webrtc: remove unneeded errPacketQueueClosed --- p2p/transport/webrtc/udpmux/packetqueue.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go index 379b0f5220..054419818a 100644 --- a/p2p/transport/webrtc/udpmux/packetqueue.go +++ b/p2p/transport/webrtc/udpmux/packetqueue.go @@ -12,10 +12,7 @@ type packet struct { buf []byte } -var ( - errTooManyPackets = errors.New("too many packets in queue; dropping") - errPacketQueueClosed = errors.New("packet queue closed") -) +var errTooManyPackets = errors.New("too many packets in queue; dropping") const maxPacketsInQueue = 128 @@ -57,7 +54,7 @@ start: } return n, nil case <-ctx.Done(): - return 0, errPacketQueueClosed + return 0, ctx.Err() } } From 8674088373be3017d892ee6516c51caafe82bb62 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:37:09 -0700 Subject: [PATCH 23/86] webrtc: remove unused udpMuxStorage.GetConn --- p2p/transport/webrtc/udpmux/mux.go | 22 +++++++--------------- p2p/transport/webrtc/udpmux/mux_test.go | 16 +++++++++------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index ab957591a1..4f982af474 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -23,13 +23,6 @@ var ( const ReceiveMTU = 1500 -var _ ice.UDPMux = &udpMux{} - -type ufragConnKey struct { - ufrag string - isIPv6 bool -} - // udpMux multiplexes multiple ICE connections over a single net.PacketConn, // generally a UDP socket. // @@ -57,6 +50,8 @@ type udpMux struct { cancel context.CancelFunc } +var _ ice.UDPMux = &udpMux{} + func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) bool) *udpMux { ctx, cancel := context.WithCancel(context.Background()) mux := &udpMux{ @@ -211,6 +206,11 @@ func (mux *udpMux) processPacket(buf []byte, addr net.Addr) error { return nil } +type ufragConnKey struct { + ufrag string + isIPv6 bool +} + // ufragFromSTUNMessage returns the local or ufrag // from the STUN username attribute. Local ufrag is the ufrag of the // peer which initiated the connectivity check, e.g in a connectivity @@ -260,14 +260,6 @@ func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { } } -func (s *udpMuxStorage) GetConn(ufrag string, isIPv6 bool) (*muxedConnection, bool) { - key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} - s.Lock() - conn, ok := s.ufragMap[key] - s.Unlock() - return conn, ok -} - func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *udpMux, addr net.Addr) (*muxedConnection, bool, error) { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} diff --git a/p2p/transport/webrtc/udpmux/mux_test.go b/p2p/transport/webrtc/udpmux/mux_test.go index 5d7fc7960d..f6dc0e2aa3 100644 --- a/p2p/transport/webrtc/udpmux/mux_test.go +++ b/p2p/transport/webrtc/udpmux/mux_test.go @@ -47,9 +47,11 @@ func (dummyPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) { return 0, nil } -func hasConn(m *udpMux, ufrag string, isIPv6 bool) *muxedConnection { - conn, _ := m.storage.GetConn(ufrag, isIPv6) - return conn +func hasConn(m *udpMux, ufrag string, isIPv6 bool) bool { + m.storage.Lock() + _, ok := m.storage.ufragMap[ufragConnKey{ufrag: ufrag, isIPv6: isIPv6}] + m.storage.Unlock() + return ok } var ( @@ -59,12 +61,12 @@ var ( func TestUDPMux_GetConn(t *testing.T) { m := NewUDPMux(dummyPacketConn{}, nil) - require.Nil(t, hasConn(m, "test", false)) + require.False(t, hasConn(m, "test", false)) conn, err := m.GetConn("test", &addrV4) require.NoError(t, err) require.NotNil(t, conn) - require.Nil(t, hasConn(m, "test", true)) + require.False(t, hasConn(m, "test", true)) connv6, err := m.GetConn("test", &addrV6) require.NoError(t, err) require.NotNil(t, connv6) @@ -78,10 +80,10 @@ func TestUDPMux_RemoveConnectionOnClose(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.NotNil(t, hasConn(mux, "test", false)) + require.True(t, hasConn(mux, "test", false)) err = conn.Close() require.NoError(t, err) - require.Nil(t, hasConn(mux, "test", false)) + require.False(t, hasConn(mux, "test", false)) } From c7c0f1d8309a5fc6ae70d44575e5727ef8ea3a8b Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:49:39 -0700 Subject: [PATCH 24/86] webrtc: remove unused special case for a nil unknownUfragCallback --- p2p/transport/webrtc/udpmux/mux.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 4f982af474..12ae56bb80 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -193,11 +193,9 @@ func (mux *udpMux) processPacket(buf []byte, addr net.Addr) error { log.Debug("could not find create conn: %w", err) return err } - if connCreated && mux.unknownUfragCallback != nil { - if !mux.unknownUfragCallback(ufrag, udpAddr) { - conn.Close() - return io.ErrClosedPipe - } + if connCreated && !mux.unknownUfragCallback(ufrag, udpAddr) { + conn.Close() + return io.ErrClosedPipe } if err := conn.Push(buf); err != nil { From ddec80ca6a614df1985f2fec32d57c7bca384fae Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:57:14 -0700 Subject: [PATCH 25/86] webrtc: clean up and fix error handling in udpMux.processPacket --- p2p/transport/webrtc/udpmux/mux.go | 51 ++++++++++++------------------ 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 12ae56bb80..853a5d51ef 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -3,7 +3,6 @@ package udpmux import ( "bytes" "context" - "errors" "fmt" "io" "net" @@ -17,10 +16,6 @@ import ( var log = logging.Logger("mux") -var ( - errConnNotFound = errors.New("connection not found") -) - const ReceiveMTU = 1500 // udpMux multiplexes multiple ICE connections over a single net.PacketConn, @@ -115,8 +110,8 @@ func (mux *udpMux) getOrCreateConn(ufrag string, isIPv6 bool, addr net.Addr) (ne case <-mux.ctx.Done(): return nil, io.ErrClosedPipe default: - conn, _, err := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, addr) - return conn, err + _, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, addr) + return conn, nil } } @@ -143,20 +138,17 @@ func (mux *udpMux) readLoop() { } buf = buf[:n] - // a non-nil error signifies that the packet was not - // passed on to any connection, and therefore the current - // function has ownership of the packet. Otherwise, the - // ownership of the packet is passed to a connection - if err := mux.processPacket(buf, addr); err != nil { + if processed := mux.processPacket(buf, addr); !processed { pool.Put(buf) } } } -func (mux *udpMux) processPacket(buf []byte, addr net.Addr) error { +func (mux *udpMux) processPacket(buf []byte, addr net.Addr) (processed bool) { udpAddr, ok := addr.(*net.UDPAddr) if !ok { - return fmt.Errorf("underlying connection did not return a UDP address") + log.Errorf("received a non-UDP address: %s", addr) + return false } isIPv6 := udpAddr.IP.To4() == nil @@ -164,44 +156,41 @@ func (mux *udpMux) processPacket(buf []byte, addr net.Addr) error { // check if the remote address has a connection associated // with it. If yes, we push the received packet to the connection if conn, ok := mux.storage.GetConnByAddr(udpAddr); ok { - err := conn.Push(buf) - if err != nil { + if err := conn.Push(buf); err != nil { log.Errorf("could not push packet: %v", err) + return false } - return nil + return true } if !stun.IsMessage(buf) { - return errConnNotFound + log.Debug("incoming message is not a STUN message") + return false } msg := &stun.Message{Raw: buf} if err := msg.Decode(); err != nil || msg.Type != stun.BindingRequest { log.Debug("incoming message should be a STUN binding request") - return err + return false } ufrag, err := ufragFromSTUNMessage(msg) if err != nil { log.Debug("could not find STUN username: %w", err) - return err + return false } - var connCreated bool - conn, connCreated, err := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) - if err != nil { - log.Debug("could not find create conn: %w", err) - return err - } + connCreated, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) if connCreated && !mux.unknownUfragCallback(ufrag, udpAddr) { conn.Close() - return io.ErrClosedPipe + return false } if err := conn.Push(buf); err != nil { log.Errorf("could not push packet: %v", err) + return false } - return nil + return true } type ufragConnKey struct { @@ -258,21 +247,21 @@ func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { } } -func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *udpMux, addr net.Addr) (*muxedConnection, bool, error) { +func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *udpMux, addr net.Addr) (created bool, _ *muxedConnection) { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} s.Lock() defer s.Unlock() if conn, ok := s.ufragMap[key]; ok { - return conn, false, nil + return false, conn } conn := newMuxedConnection(mux, ufrag, addr) s.ufragMap[key] = conn s.addrMap[addr.String()] = conn - return conn, true, nil + return true, conn } func (s *udpMuxStorage) GetConnByAddr(addr *net.UDPAddr) (*muxedConnection, bool) { From 07449bc2b62cc381e0c8cc76ba1ad731fe55bf22 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 13:59:28 -0700 Subject: [PATCH 26/86] webrtc: fix go-log logger name for the udpmux --- p2p/transport/webrtc/udpmux/mux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 853a5d51ef..3577ee58ed 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -14,7 +14,7 @@ import ( "github.com/pion/stun" ) -var log = logging.Logger("mux") +var log = logging.Logger("webrtc-udpmux") const ReceiveMTU = 1500 From 0fed1f44724cd1b9a3698443f90f9cd2cee69e13 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 14:08:21 -0700 Subject: [PATCH 27/86] webrtc: clean up connection closing for the muxedConnection --- p2p/transport/webrtc/udpmux/mux.go | 7 +- .../webrtc/udpmux/muxed_connection.go | 69 +++++++++---------- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 3577ee58ed..08b105a704 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -234,13 +234,16 @@ func newUDPMuxStorage() *udpMuxStorage { } func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { + s.removeConnByUfrag(ufrag, true) +} + +func (s *udpMuxStorage) removeConnByUfrag(ufrag string, closeConn bool) { s.Lock() defer s.Unlock() for _, isIPv6 := range [...]bool{true, false} { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} if conn, ok := s.ufragMap[key]; ok { - _ = conn.closeConnection() delete(s.ufragMap, key) delete(s.addrMap, conn.Address().String()) } @@ -257,7 +260,7 @@ func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *udpMux, return false, conn } - conn := newMuxedConnection(mux, ufrag, addr) + conn := newMuxedConnection(mux, func() { s.removeConnByUfrag(ufrag, false) }, addr) s.ufragMap[key] = conn s.addrMap[addr.String()] = conn diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index e92897cff2..28f007ed42 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -16,52 +16,57 @@ var errAlreadyClosed = errors.New("already closed") // from which this connection (indexed by ufrag) received // data. type muxedConnection struct { - ctx context.Context - cancel context.CancelFunc - pq *packetQueue - ufrag string - addr net.Addr - mux *udpMux + ctx context.Context + cancel context.CancelFunc + onClose func() + pq *packetQueue + addr net.Addr + mux *udpMux } var _ net.PacketConn = (*muxedConnection)(nil) -func newMuxedConnection(mux *udpMux, ufrag string, addr net.Addr) *muxedConnection { +func newMuxedConnection(mux *udpMux, onClose func(), addr net.Addr) *muxedConnection { ctx, cancel := context.WithCancel(mux.ctx) return &muxedConnection{ - ctx: ctx, - cancel: cancel, - pq: newPacketQueue(), - ufrag: ufrag, - addr: addr, - mux: mux, + ctx: ctx, + cancel: cancel, + pq: newPacketQueue(), + onClose: onClose, + addr: addr, + mux: mux, } } -func (conn *muxedConnection) Push(buf []byte) error { - return conn.pq.Push(buf) +func (c *muxedConnection) Push(buf []byte) error { + return c.pq.Push(buf) } // Close implements net.PacketConn -func (conn *muxedConnection) Close() error { - _ = conn.closeConnection() - conn.mux.RemoveConnByUfrag(conn.ufrag) +func (c *muxedConnection) Close() error { + select { + case <-c.ctx.Done(): + return errAlreadyClosed + default: + } + c.onClose() + c.cancel() return nil } // LocalAddr implements net.PacketConn -func (conn *muxedConnection) LocalAddr() net.Addr { - return conn.mux.socket.LocalAddr() +func (c *muxedConnection) LocalAddr() net.Addr { + return c.mux.socket.LocalAddr() } -func (conn *muxedConnection) Address() net.Addr { - return conn.addr +func (c *muxedConnection) Address() net.Addr { + return c.addr } // ReadFrom implements net.PacketConn -func (conn *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { - n, err := conn.pq.Pop(conn.ctx, p) - return n, conn.addr, err +func (c *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { + n, err := c.pq.Pop(c.ctx, p) + return n, c.addr, err } // SetDeadline implements net.PacketConn @@ -83,16 +88,6 @@ func (*muxedConnection) SetWriteDeadline(t time.Time) error { } // WriteTo implements net.PacketConn -func (conn *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { - return conn.mux.writeTo(p, addr) -} - -func (conn *muxedConnection) closeConnection() error { - select { - case <-conn.ctx.Done(): - return errAlreadyClosed - default: - } - conn.cancel() - return nil +func (c *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { + return c.mux.writeTo(p, addr) } From e96e711a1936f5848f1f057b579d2da739ae1644 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 14:09:20 -0700 Subject: [PATCH 28/86] webrtc: allow multiple calls to muxedConnection.Close --- p2p/transport/webrtc/udpmux/muxed_connection.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index 28f007ed42..dbfc5d6b75 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -2,15 +2,12 @@ package udpmux import ( "context" - "errors" "net" "time" ) var _ net.PacketConn = &muxedConnection{} -var errAlreadyClosed = errors.New("already closed") - // muxedConnection provides a net.PacketConn abstraction // over packetQueue and adds the ability to store addresses // from which this connection (indexed by ufrag) received @@ -46,7 +43,7 @@ func (c *muxedConnection) Push(buf []byte) error { func (c *muxedConnection) Close() error { select { case <-c.ctx.Done(): - return errAlreadyClosed + return nil default: } c.onClose() From 22e81a0cdb71eeefbeb5b7761845d95d56a1a3cf Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 16:40:31 -0700 Subject: [PATCH 29/86] webrtc: fix construction of listener multiaddresses --- p2p/transport/webrtc/transport.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 1b4a5cdf85..2aa756f1b6 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -35,6 +35,16 @@ var log = logging.Logger("webrtc-transport") var dialMatcher = mafmt.And(mafmt.UDP, mafmt.Base(ma.P_WEBRTC_DIRECT), mafmt.Base(ma.P_CERTHASH)) +var webrtcComponent *ma.Component + +func init() { + var err error + webrtcComponent, err = ma.NewComponent(ma.ProtocolWithCode(ma.P_WEBRTC_DIRECT).Name, "") + if err != nil { + log.Fatal(err) + } +} + const ( // handshakeChannelNegotiated is used to specify that the // handshake data channel does not need negotiation via DCEP. @@ -152,11 +162,9 @@ func (t *WebRTCTransport) CanDial(addr ma.Multiaddr) bool { return dialMatcher.Matches(addr) } -var webRTCMultiAddr = ma.StringCast("/webrtc-direct") - func (t *WebRTCTransport) Listen(addr ma.Multiaddr) (tpt.Listener, error) { addr, wrtcComponent := ma.SplitLast(addr) - isWebrtc := wrtcComponent.Equal(webRTCMultiAddr) + isWebrtc := wrtcComponent.Equal(webrtcComponent) if !isWebrtc { return nil, fmt.Errorf("must listen on webrtc multiaddr") } @@ -198,12 +206,11 @@ func (t *WebRTCTransport) listenSocket(socket *net.UDPConn) (tpt.Listener, error return nil, err } - certMultiaddress, err := ma.NewMultiaddr(fmt.Sprintf("/p2p-webrtc-direct/certhash/%s", encodedLocalFingerprint)) + certComp, err := ma.NewComponent(ma.ProtocolWithCode(ma.P_CERTHASH).Name, encodedLocalFingerprint) if err != nil { return nil, err } - - listenerMultiaddr = listenerMultiaddr.Encapsulate(certMultiaddress) + listenerMultiaddr = listenerMultiaddr.Encapsulate(webrtcComponent).Encapsulate(certComp) return newListener( t, From 8826c907f00ee1c2f6845d66f9e26e87c5ce9045 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 17:22:39 -0700 Subject: [PATCH 30/86] webrtc: rewrite the connection timeout test --- p2p/transport/webrtc/transport_test.go | 86 +++++++++++++++----------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index f1e4f5d0e6..ce01a9b8dc 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -11,6 +11,10 @@ import ( "testing" "time" + manet "github.com/multiformats/go-multiaddr/net" + + quicproxy "github.com/quic-go/quic-go/integrationtests/tools/proxy" + "golang.org/x/crypto/sha3" "github.com/libp2p/go-libp2p/core/crypto" @@ -773,59 +777,71 @@ func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { require.ErrorContains(t, err, "failed") } -func TestStreamResetOnPeerConnectionFailure(t *testing.T) { +func TestConnectionTimeoutOnListener(t *testing.T) { tr, listeningPeer := getTransport(t) tr.peerConnectionTimeouts.Disconnect = 100 * time.Millisecond tr.peerConnectionTimeouts.Failed = 150 * time.Millisecond tr.peerConnectionTimeouts.Keepalive = 50 * time.Millisecond listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) - lsnr, err := tr.Listen(listenMultiaddr) + ln, err := tr.Listen(listenMultiaddr) require.NoError(t, err) + defer ln.Close() - tr1, connectingPeer := getTransport(t) - tr1.peerConnectionTimeouts.Disconnect = 100 * time.Millisecond - tr1.peerConnectionTimeouts.Failed = 150 * time.Millisecond - tr1.peerConnectionTimeouts.Keepalive = 50 * time.Millisecond + var drop atomic.Bool + proxy, err := quicproxy.NewQuicProxy(fmt.Sprintf("%s:0", listenerIP), &quicproxy.Opts{ + RemoteAddr: fmt.Sprintf("%s:%d", listenerIP, ln.Addr().(*net.UDPAddr).Port), + DropPacket: func(quicproxy.Direction, []byte) bool { return drop.Load() }, + }) + require.NoError(t, err) + defer proxy.Close() - done := make(chan struct{}) + tr1, connectingPeer := getTransport(t) go func() { - lconn, err := lsnr.Accept() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + addr, err := manet.FromNetAddr(proxy.LocalAddr()) require.NoError(t, err) - require.Equal(t, connectingPeer, lconn.RemotePeer()) - - stream, err := lconn.AcceptStream() + _, webrtcComponent := ma.SplitFunc(ln.Multiaddr(), func(c ma.Component) bool { return c.Protocol().Code == ma.P_WEBRTC_DIRECT }) + addr = addr.Encapsulate(webrtcComponent) + conn, err := tr1.Dial(ctx, addr, listeningPeer) require.NoError(t, err) - _, err = stream.Write([]byte("test")) + str, err := conn.OpenStream(ctx) require.NoError(t, err) - // force close the mux - lsnr.(*listener).mux.Close() - // stream.Write can keep buffering data until failure, - // so we need to loop on writing. - for { - _, err := stream.Write([]byte("test")) - if err != nil { - assert.ErrorIs(t, err, network.ErrReset) - close(done) - return - } - } + str.Write([]byte("foobar")) }() - dialctx, dialcancel := context.WithTimeout(context.Background(), 10*time.Second) - defer dialcancel() - conn, err := tr1.Dial(dialctx, lsnr.Multiaddr(), listeningPeer) - require.NoError(t, err) - stream, err := conn.OpenStream(dialctx) + conn, err := ln.Accept() require.NoError(t, err) - _, err = io.ReadAll(stream) - require.Error(t, err) + require.Equal(t, connectingPeer, conn.RemotePeer()) - select { - case <-done: - case <-time.After(30 * time.Second): - t.Fatal("timed out") + str, err := conn.AcceptStream() + require.NoError(t, err) + _, err = str.Write([]byte("test")) + require.NoError(t, err) + // start dropping all packets + drop.Store(true) + start := time.Now() + // TODO: return timeout errors here + for { + if _, err := str.Write([]byte("test")); err != nil { + require.ErrorIs(t, err, io.ErrClosedPipe) + break + } + if time.Since(start) > 5*time.Second { + t.Fatal("timeout") + } + // make sure to not write too often, we don't want to fill the flow control window + time.Sleep(5 * time.Millisecond) } + // make sure that accepting a stream also returns an error... + _, err = conn.AcceptStream() + // TODO: return timeout errors here + require.Error(t, err) + // ... as well as opening a new stream + _, err = conn.OpenStream(context.Background()) + // TODO: return timeout errors here + require.Error(t, err) } func TestMaxInFlightRequests(t *testing.T) { From fe6eca711657a4df78c597aaf50e9a467b2324d2 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 1 Jul 2023 22:42:04 -0700 Subject: [PATCH 31/86] webrtc: remove custom definition of timeout error --- p2p/transport/webrtc/stream_error.go | 25 ------------------------- p2p/transport/webrtc/stream_read.go | 5 +---- p2p/transport/webrtc/stream_write.go | 5 +---- p2p/transport/webrtc/transport_test.go | 14 +++++++------- 4 files changed, 9 insertions(+), 40 deletions(-) delete mode 100644 p2p/transport/webrtc/stream_error.go diff --git a/p2p/transport/webrtc/stream_error.go b/p2p/transport/webrtc/stream_error.go deleted file mode 100644 index b2546b8dec..0000000000 --- a/p2p/transport/webrtc/stream_error.go +++ /dev/null @@ -1,25 +0,0 @@ -package libp2pwebrtc - -import "net" - -type Error struct { - message string - temporary, timeout bool -} - -var _ net.Error = &Error{} - -func (e *Error) Error() string { - return e.message -} - -func (e *Error) Temporary() bool { - return e.temporary -} - -func (e *Error) Timeout() bool { - return e.timeout -} - -// ErrTimeout is used when an i/o deadline is reached -var ErrTimeout = &Error{message: "i/o deadline reached", timeout: true, temporary: true} diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 852af3a22d..e8ffe6e9ee 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -8,7 +8,7 @@ import ( "time" "github.com/libp2p/go-libp2p/core/network" - pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" ) // Read from the underlying datachannel. This also @@ -34,9 +34,6 @@ func (s *webRTCStream) Read(b []byte) (int, error) { } read, readErr = s.readMessage(b) } - if errors.Is(readErr, os.ErrDeadlineExceeded) { - return read, ErrTimeout - } return read, readErr } diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index ebc5791fe3..24be04ffe6 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -9,7 +9,7 @@ import ( "time" "github.com/libp2p/go-libp2p/core/network" - pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" "google.golang.org/protobuf/proto" ) @@ -38,9 +38,6 @@ func (s *webRTCStream) Write(b []byte) (int, error) { n += end b = b[end:] if err != nil { - if errors.Is(err, os.ErrDeadlineExceeded) { - err = ErrTimeout - } return n, err } } diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index ce01a9b8dc..2020591915 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "os" "sync" "sync/atomic" "testing" @@ -386,12 +387,12 @@ func TestTransportWebRTC_Deadline(t *testing.T) { // deadline set to the past stream.SetReadDeadline(time.Now().Add(-200 * time.Millisecond)) _, err = stream.Read([]byte{0, 0}) - require.ErrorIs(t, err, ErrTimeout) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) // future deadline exceeded stream.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) _, err = stream.Read([]byte{0, 0}) - require.ErrorIs(t, err, ErrTimeout) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) }) t.Run("SetWriteDeadline", func(t *testing.T) { @@ -411,7 +412,7 @@ func TestTransportWebRTC_Deadline(t *testing.T) { stream.SetWriteDeadline(time.Now().Add(200 * time.Millisecond)) largeBuffer := make([]byte, 2*1024*1024) _, err = stream.Write(largeBuffer) - require.ErrorIs(t, err, ErrTimeout) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) }) } @@ -451,9 +452,8 @@ func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { }() } - require.ErrorIs(t, <-errC, ErrTimeout) - require.ErrorIs(t, <-errC, ErrTimeout) - + require.ErrorIs(t, <-errC, os.ErrDeadlineExceeded) + require.ErrorIs(t, <-errC, os.ErrDeadlineExceeded) } func TestTransportWebRTC_Read(t *testing.T) { @@ -646,7 +646,7 @@ func TestTransportWebRTC_Close(t *testing.T) { }) _, err = stream.Read(make([]byte, 19)) - require.ErrorIs(t, err, ErrTimeout) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) select { case <-done: From dd959d42d48038b374b05b07006e17f85b4b8d77 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jul 2023 09:39:49 -0700 Subject: [PATCH 32/86] webrtc: use a callback to remove closed streams from the streams map --- p2p/transport/webrtc/connection.go | 49 ++++++++---------------------- p2p/transport/webrtc/listener.go | 2 +- p2p/transport/webrtc/stream.go | 15 ++++----- p2p/transport/webrtc/transport.go | 2 +- 4 files changed, 23 insertions(+), 45 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 6b24b7c744..ac926c2835 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -14,7 +14,7 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" tpt "github.com/libp2p/go-libp2p/core/transport" - pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" "github.com/libp2p/go-msgio" ma "github.com/multiformats/go-multiaddr" "github.com/pion/datachannel" @@ -179,11 +179,11 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error return nil, fmt.Errorf("open stream: %w", err) } stream := newStream( - c, dc, rwc, nil, nil, + func() { c.removeStream(*streamID) }, ) err = c.addStream(stream) if err != nil { @@ -197,13 +197,13 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { select { case <-c.ctx.Done(): return nil, os.ErrClosed - case accStream := <-c.acceptQueue: + case str := <-c.acceptQueue: stream := newStream( - c, - accStream.channel, - accStream.stream, + str.channel, + str.stream, nil, nil, + func() { c.removeStream(*str.channel.ID()) }, ) if err := c.addStream(stream); err != nil { stream.Close() @@ -213,36 +213,13 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { } } -// implement network.ConnSecurity -func (c *connection) LocalPeer() peer.ID { - return c.localPeer -} - -func (c *connection) RemotePeer() peer.ID { - return c.remotePeer -} - -func (c *connection) RemotePublicKey() ic.PubKey { - return c.remoteKey -} - -// implement network.ConnMultiaddrs -func (c *connection) LocalMultiaddr() ma.Multiaddr { - return c.localMultiaddr -} - -func (c *connection) RemoteMultiaddr() ma.Multiaddr { - return c.remoteMultiaddr -} - -// implement network.ConnScoper -func (c *connection) Scope() network.ConnScope { - return c.scope -} - -func (c *connection) Transport() tpt.Transport { - return c.transport -} +func (c *connection) LocalPeer() peer.ID { return c.localPeer } +func (c *connection) RemotePeer() peer.ID { return c.remotePeer } +func (c *connection) RemotePublicKey() ic.PubKey { return c.remoteKey } +func (c *connection) LocalMultiaddr() ma.Multiaddr { return c.localMultiaddr } +func (c *connection) RemoteMultiaddr() ma.Multiaddr { return c.remoteMultiaddr } +func (c *connection) Scope() network.ConnScope { return c.scope } +func (c *connection) Transport() tpt.Transport { return c.transport } func (c *connection) addStream(stream *webRTCStream) error { c.m.Lock() diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index f699bf72cf..e72e81b94c 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -268,7 +268,7 @@ func (l *listener) setupConnection( localMultiaddrWithoutCerthash, _ := ma.SplitFunc(l.localMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) - handshakeChannel := newStream(nil, rawDatachannel, rwc, l.localAddr, addr.raddr) + handshakeChannel := newStream(rawDatachannel, rwc, l.localAddr, addr.raddr, func() {}) // The connection is instantiated before performing the Noise handshake. This is // to handle the case where the remote is faster and attempts to initiate a stream // before the ondatachannel callback can be set. diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 829a042a89..f28cce71da 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -7,8 +7,9 @@ import ( "time" "github.com/libp2p/go-libp2p/core/network" - pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" "github.com/libp2p/go-msgio/pbio" + "github.com/pion/datachannel" "github.com/pion/webrtc/v3" ) @@ -71,8 +72,8 @@ type webRTCStream struct { stateHandler webRTCStreamState - conn *connection - id uint16 + onDone func() + id uint16 // for logging purposes dataChannel *datachannel.DataChannel laddr net.Addr @@ -85,10 +86,10 @@ type webRTCStream struct { } func newStream( - connection *connection, channel *webrtc.DataChannel, rwc datachannel.ReadWriteCloser, laddr, raddr net.Addr, + onDone func(), ) *webRTCStream { ctx, cancel := context.WithCancel(context.Background()) @@ -99,9 +100,9 @@ func newStream( writerDeadlineUpdated: make(chan struct{}, 1), writeAvailable: make(chan struct{}, 1), - conn: connection, id: *channel.ID(), dataChannel: rwc.(*datachannel.DataChannel), + onDone: onDone, laddr: laddr, raddr: raddr, @@ -180,8 +181,8 @@ func (s *webRTCStream) close(isReset bool, notifyConnection bool) error { // close the channel. We do not care about the error message in // this case err = s.dataChannel.Close() - if notifyConnection && s.conn != nil { - s.conn.removeStream(s.id) + if notifyConnection { + s.onDone() } }) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 2aa756f1b6..c6e9933198 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -363,7 +363,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement } laddr := &net.UDPAddr{IP: net.ParseIP(cp.Local.Address), Port: int(cp.Local.Port)} - channel := newStream(nil, rawHandshakeChannel, detached, laddr, raddr) + channel := newStream(rawHandshakeChannel, detached, laddr, raddr, func() {}) // the local address of the selected candidate pair should be the // local address for the connection, since different datachannels // are multiplexed over the same SCTP connection From 615da30b554b8cc8dfebfc765983a60a62e0557b Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jul 2023 12:43:38 -0700 Subject: [PATCH 33/86] webrtc: remove nonsensical context value in Noise handshake error --- p2p/transport/webrtc/transport.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index c6e9933198..bd78cde11e 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -491,14 +491,12 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerCon if inbound { secureConn, err = sessionTransport.SecureOutbound(ctx, datachannel, peer) if err != nil { - err = fmt.Errorf("failed to secure inbound [noise outbound]: %w %v", err, ctx.Value("id")) - return secureConn, err + return secureConn, fmt.Errorf("failed to secure inbound connection: %w", err) } } else { secureConn, err = sessionTransport.SecureInbound(ctx, datachannel, peer) if err != nil { - err = fmt.Errorf("failed to secure outbound [noise inbound]: %w %v", err, ctx.Value("id")) - return secureConn, err + return secureConn, fmt.Errorf("failed to secure outbound connection: %w", err) } } return secureConn, nil From 505053e83ebae47fb6918c03aed41c9eb3fe6d1f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jul 2023 13:17:06 -0700 Subject: [PATCH 34/86] webrtc: add missing error check when setting the remote's description --- p2p/transport/webrtc/listener.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index e72e81b94c..1caa5592c8 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -235,19 +235,18 @@ func (l *listener) setupConnection( } errC := awaitPeerConnectionOpen(addr.ufrag, pc) - // we infer the client sdp from the incoming STUN connectivity check - // by setting the ice-ufrag equal to the incoming check. - clientSdpString := createClientSDP(addr.raddr, addr.ufrag) - clientSdp := webrtc.SessionDescription{SDP: clientSdpString, Type: webrtc.SDPTypeOffer} - pc.SetRemoteDescription(clientSdp) - + // Infer the client SDP from the incoming STUN message by setting the ice-ufrag. + if err := pc.SetRemoteDescription(webrtc.SessionDescription{ + SDP: createClientSDP(addr.raddr, addr.ufrag), + Type: webrtc.SDPTypeOffer, + }); err != nil { + return nil, err + } answer, err := pc.CreateAnswer(nil) if err != nil { return nil, err } - - err = pc.SetLocalDescription(answer) - if err != nil { + if err := pc.SetLocalDescription(answer); err != nil { return nil, err } @@ -294,8 +293,7 @@ func (l *listener) setupConnection( } // earliest point where we know the remote's peerID - err = scope.SetPeer(secureConn.RemotePeer()) - if err != nil { + if err := scope.SetPeer(secureConn.RemotePeer()); err != nil { return nil, err } From 5af980b3c62e9850aa71b626e61362e72106ed7e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jul 2023 16:35:07 -0700 Subject: [PATCH 35/86] webrtc: clean up messy TestTransportWebRTC_PeerConnectionDTLSFailed --- p2p/transport/webrtc/transport_test.go | 33 +++++++------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 2020591915..7a53534e6a 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -736,45 +736,28 @@ func TestReceiveFlagsAfterReadClosed(t *testing.T) { } func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { - // test multihash - encoded, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") - require.NoError(t, err) - - testMultihash := &multihash.DecodedMultihash{ - Code: multihash.SHA2_256, - Name: multihash.Codes[multihash.SHA2_256], - Digest: encoded, - Length: len(encoded), - } - tr, listeningPeer := getTransport(t) listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + ln, err := tr.Listen(listenMultiaddr) require.NoError(t, err) - listener, err := tr.Listen(listenMultiaddr) - require.NoError(t, err) - - tr1, _ := getTransport(t) - - go listener.Accept() - - badMultiaddr, _ := ma.SplitFunc(listener.Multiaddr(), func(component ma.Component) bool { - return component.Protocol().Code == ma.P_CERTHASH - }) + defer ln.Close() - encodedCerthash, err := multihash.Encode(testMultihash.Digest, testMultihash.Code) + encoded, err := hex.DecodeString("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + require.NoError(t, err) + encodedCerthash, err := multihash.Encode(encoded, multihash.SHA2_256) require.NoError(t, err) badEncodedCerthash, err := multibase.Encode(multibase.Base58BTC, encodedCerthash) require.NoError(t, err) badCerthash, err := ma.NewMultiaddr(fmt.Sprintf("/certhash/%s", badEncodedCerthash)) require.NoError(t, err) + badMultiaddr, _ := ma.SplitFunc(ln.Multiaddr(), func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) badMultiaddr = badMultiaddr.Encapsulate(badCerthash) + tr1, _ := getTransport(t) conn, err := tr1.Dial(context.Background(), badMultiaddr, listeningPeer) - require.Nil(t, conn) - t.Log(err) require.Error(t, err) - require.ErrorContains(t, err, "failed") + require.Nil(t, conn) } func TestConnectionTimeoutOnListener(t *testing.T) { From c89798bf0d6e7f7719787d831e1d491a81d02423 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 09:22:03 -0700 Subject: [PATCH 36/86] webrtc: set the ICEPrflxAcceptanceMinWait to 0 when dialing --- p2p/transport/webrtc/transport.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index bd78cde11e..e3a88a3349 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -284,6 +284,8 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement settingEngine.SetICECredentials(ufrag, ufrag) settingEngine.DetachDataChannels() + // use the first best address candidate + settingEngine.SetPrflxAcceptanceMinWait(0) settingEngine.SetICETimeouts( t.peerConnectionTimeouts.Disconnect, t.peerConnectionTimeouts.Failed, From 514172b92e4d7e41ada12a4715e093108368dfd8 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jul 2023 14:43:01 -0700 Subject: [PATCH 37/86] webrtc: giant stream refactor --- p2p/transport/webrtc/connection.go | 70 +++---- p2p/transport/webrtc/stream.go | 215 ++++++++++++--------- p2p/transport/webrtc/stream_read.go | 140 +++++--------- p2p/transport/webrtc/stream_state.go | 136 ------------- p2p/transport/webrtc/stream_test.go | 227 ++++++++++++++++++++++ p2p/transport/webrtc/stream_write.go | 258 +++++++++++++------------ p2p/transport/webrtc/transport.go | 2 +- p2p/transport/webrtc/transport_test.go | 63 +----- 8 files changed, 575 insertions(+), 536 deletions(-) delete mode 100644 p2p/transport/webrtc/stream_state.go create mode 100644 p2p/transport/webrtc/stream_test.go diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index ac926c2835..a0942c576d 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -6,7 +6,7 @@ import ( "fmt" "math" "math/rand" - "os" + "net" "sync" "sync/atomic" @@ -24,9 +24,15 @@ import ( var _ tpt.CapableConn = &connection{} -const ( - maxAcceptQueueLen = 10 -) +const maxAcceptQueueLen = 10 + +type errConnectionTimeout struct{} + +var _ net.Error = &errConnectionTimeout{} + +func (errConnectionTimeout) Error() string { return "connection timeout" } +func (errConnectionTimeout) Timeout() bool { return true } +func (errConnectionTimeout) Temporary() bool { return false } type acceptStream struct { stream datachannel.ReadWriteCloser @@ -40,6 +46,8 @@ type connection struct { transport *WebRTCTransport scope network.ConnManagementScope + closeErr error + localPeer peer.ID localMultiaddr ma.Multiaddr @@ -48,7 +56,7 @@ type connection struct { remoteMultiaddr ma.Multiaddr m sync.Mutex - streams map[uint16]*webRTCStream + streams map[uint16]*stream acceptQueue chan acceptStream idAllocator *sidAllocator @@ -87,7 +95,7 @@ func newConnection( remoteMultiaddr: remoteMultiaddr, ctx: ctx, cancel: cancel, - streams: make(map[uint16]*webRTCStream), + streams: make(map[uint16]*stream), idAllocator: idAllocator, acceptQueue: make(chan acceptStream, maxAcceptQueueLen), @@ -119,26 +127,9 @@ func newConnection( return conn, nil } -func (c *connection) resetStreams() { - if c.IsClosed() { - return - } - c.m.Lock() - defer c.m.Unlock() - for k, stream := range c.streams { - // reset the streams, but we do not need to be notified - // of stream closure - stream.close(true, false) - delete(c.streams, k) - } - -} - // ConnState implements transport.CapableConn func (c *connection) ConnState() network.ConnectionState { - return network.ConnectionState{ - Transport: "p2p-webrtc-direct", - } + return network.ConnectionState{Transport: "p2p-webrtc-direct"} } // Close closes the underlying peerconnection. @@ -148,6 +139,7 @@ func (c *connection) Close() error { } c.scope.Done() + c.closeErr = errors.New("connection closed") c.cancel() return c.pc.Close() } @@ -163,7 +155,7 @@ func (c *connection) IsClosed() bool { func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error) { if c.IsClosed() { - return nil, os.ErrClosed + return nil, c.closeErr } streamID, err := c.idAllocator.nextID() @@ -178,25 +170,24 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if err != nil { return nil, fmt.Errorf("open stream: %w", err) } - stream := newStream( + str := newStream( dc, rwc, nil, nil, func() { c.removeStream(*streamID) }, ) - err = c.addStream(stream) - if err != nil { - stream.Close() + if err := c.addStream(str); err != nil { + str.Close() return nil, err } - return stream, nil + return str, nil } func (c *connection) AcceptStream() (network.MuxedStream, error) { select { case <-c.ctx.Done(): - return nil, os.ErrClosed + return nil, c.closeErr case str := <-c.acceptQueue: stream := newStream( str.channel, @@ -221,7 +212,7 @@ func (c *connection) RemoteMultiaddr() ma.Multiaddr { return c.remoteMultiaddr } func (c *connection) Scope() network.ConnScope { return c.scope } func (c *connection) Transport() tpt.Transport { return c.transport } -func (c *connection) addStream(stream *webRTCStream) error { +func (c *connection) addStream(stream *stream) error { c.m.Lock() defer c.m.Unlock() if _, ok := c.streams[stream.id]; ok { @@ -238,10 +229,19 @@ func (c *connection) removeStream(id uint16) { } func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { - log.Debugf("[%s][%d] handling peerconnection state: %s", c.localPeer, c.dbgId, state) + fmt.Printf("[%s][%d] handling peerconnection state: %s\n", c.localPeer, c.dbgId, state) if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed { // reset any streams - c.resetStreams() + if c.IsClosed() { + return + } + c.m.Lock() + defer c.m.Unlock() + c.closeErr = errConnectionTimeout{} + for k, str := range c.streams { + str.setCloseError(c.closeErr) + delete(c.streams, k) + } c.cancel() c.scope.Done() c.pc.Close() @@ -276,7 +276,7 @@ func (c *connection) detachChannel(ctx context.Context, dc *webrtc.DataChannel) }) select { case <-c.ctx.Done(): - return nil, errors.New("connection closed") + return nil, c.closeErr case <-ctx.Done(): return nil, ctx.Err() case <-done: diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index f28cce71da..6ddae290b3 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -1,7 +1,6 @@ package libp2pwebrtc import ( - "context" "net" "sync" "time" @@ -14,10 +13,8 @@ import ( "github.com/pion/webrtc/v3" ) -var _ network.MuxedStream = &webRTCStream{} - const ( - // maxMessageSize is limited to 16384 bytes in the SDP. + // maxMessageSize is the maximum message size of the Protobuf message we send / receive. maxMessageSize = 16384 // Pion SCTP association has an internal receive buffer of 1MB (roughly, 1MB per connection). // We can change this value in the SettingEngine before creating the peerconnection. @@ -43,62 +40,73 @@ const ( varintOverhead = 2 ) +type receiveState uint8 + +const ( + receiveStateReceiving receiveState = iota + receiveStateDataRead // received and read the FIN + receiveStateReset // either by calling CloseRead locally, or by receiving +) + +type sendState uint8 + +const ( + sendStateSending sendState = iota + sendStateDataSent + sendStateReset +) + // Package pion detached data channel into a net.Conn // and then a network.MuxedStream -type webRTCStream struct { - reader pbio.Reader +type stream struct { // pbio.Reader is not thread safe, // and while our Read is not promised to be thread safe, // we ourselves internally read from multiple routines... - readerMux sync.Mutex + readMu sync.Mutex + reader pbio.Reader // this buffer is limited up to a single message. Reason we need it // is because a reader might read a message midway, and so we need a // wait to buffer that for as long as the remaining part is not (yet) read - readBuffer []byte - - writer pbio.Writer - // public write API is not promised to be thread safe, however we also write from - // read functionality due to our spec limiations, so (e.g. 1 channel for state and data) - // and thus we do need to protect the writer - writerMux sync.Mutex - - writerDeadline time.Time - writerDeadlineMux sync.Mutex - - writerDeadlineUpdated chan struct{} - writeAvailable chan struct{} + nextMessage *pb.Message + receiveState receiveState + + // The public Write API is not promised to be thread safe, + // but we need to be able to write control messages. + writeMu sync.Mutex + writer pbio.Writer + sendStateChanged chan struct{} + sendState sendState + controlMsgQueue []*pb.Message + writeDeadline time.Time + writeDeadlineUpdated chan struct{} + writeAvailable chan struct{} readLoopOnce sync.Once - stateHandler webRTCStreamState - onDone func() id uint16 // for logging purposes dataChannel *datachannel.DataChannel + closeErr error laddr net.Addr raddr net.Addr - - ctx context.Context - cancel context.CancelFunc - - closeOnce sync.Once } +var _ network.MuxedStream = &stream{} + func newStream( channel *webrtc.DataChannel, rwc datachannel.ReadWriteCloser, laddr, raddr net.Addr, onDone func(), -) *webRTCStream { - ctx, cancel := context.WithCancel(context.Background()) - - result := &webRTCStream{ +) *stream { + s := &stream{ reader: pbio.NewDelimitedReader(rwc, maxMessageSize), writer: pbio.NewDelimitedWriter(rwc), - writerDeadlineUpdated: make(chan struct{}, 1), - writeAvailable: make(chan struct{}, 1), + sendStateChanged: make(chan struct{}, 1), + writeDeadlineUpdated: make(chan struct{}, 1), + writeAvailable: make(chan struct{}, 1), id: *channel.ID(), dataChannel: rwc.(*datachannel.DataChannel), @@ -106,94 +114,109 @@ func newStream( laddr: laddr, raddr: raddr, - - ctx: ctx, - cancel: cancel, } channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) channel.OnBufferedAmountLow(func() { + s.writeMu.Lock() + defer s.writeMu.Unlock() + // first send out queued control messages + for len(s.controlMsgQueue) > 0 { + msg := s.controlMsgQueue[0] + available := s.availableSendSpace() + if controlMsgSize < available { + s.writer.WriteMsg(msg) // TODO: handle error + s.controlMsgQueue = s.controlMsgQueue[1:] + } else { + return + } + } + + s.maybeDeclareStreamDone() + select { - case result.writeAvailable <- struct{}{}: + case s.writeAvailable <- struct{}{}: default: } }) - - return result + return s } -func (s *webRTCStream) Close() error { - return s.close(false, true) +func (s *stream) Close() error { + closeWriteErr := s.CloseWrite() + closeReadErr := s.CloseRead() + if closeWriteErr != nil { + return closeWriteErr + } + return closeReadErr } -func (s *webRTCStream) Reset() error { - return s.close(true, true) +func (s *stream) Reset() error { + cancelWriteErr := s.cancelWrite() + closeReadErr := s.CloseRead() + if cancelWriteErr != nil { + return cancelWriteErr + } + return closeReadErr } -func (s *webRTCStream) LocalAddr() net.Addr { - return s.laddr -} +func (s *stream) LocalAddr() net.Addr { return s.laddr } +func (s *stream) RemoteAddr() net.Addr { return s.raddr } -func (s *webRTCStream) RemoteAddr() net.Addr { - return s.raddr -} - -func (s *webRTCStream) SetDeadline(t time.Time) error { +func (s *stream) SetDeadline(t time.Time) error { return s.SetWriteDeadline(t) } -func (s *webRTCStream) processIncomingFlag(flag pb.Message_Flag) { - if s.isClosed() { +// processIncomingFlag process the flag on an incoming message +// It needs to be called with msg.Flag, not msg.GetFlag(), +// otherwise we'd misinterpret the default value. +func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { + if flag == nil { return } - state, reset := s.stateHandler.HandleInboundFlag(flag) - if state == stateClosed { - log.Debug("closing: after handle inbound flag") - s.close(reset, true) - } -} -// this is used to force reset a stream -func (s *webRTCStream) close(isReset bool, notifyConnection bool) error { - if s.isClosed() { - return nil - } + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.readMu.Lock() + defer s.readMu.Unlock() - var err error - s.closeOnce.Do(func() { - log.Debugf("closing stream %d: reset: %t, notify: %t", s.id, isReset, notifyConnection) - if isReset { - s.stateHandler.Reset() - // write the RESET message. The error is explicitly ignored - // because we do not know if the remote is still connected - s.writeMessageToWriter(&pb.Message{Flag: pb.Message_RESET.Enum()}) - } else { - s.stateHandler.Close() - // write a FIN message for standard stream closure - // we write directly to the underlying writer since we do not care about - // cancelled contexts or deadlines for this. - s.writeMessageToWriter(&pb.Message{Flag: pb.Message_FIN.Enum()}) + switch *flag { + case pb.Message_FIN: + if s.receiveState == receiveStateReceiving { + s.receiveState = receiveStateDataRead } - // close the context - s.cancel() - // force close reads - s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) // pion ignores zero times - // close the channel. We do not care about the error message in - // this case - err = s.dataChannel.Close() - if notifyConnection { - s.onDone() + case pb.Message_STOP_SENDING: + if s.sendState == sendStateSending { + s.sendState = sendStateReset } - }) - - return err + select { + case s.sendStateChanged <- struct{}{}: + default: + } + case pb.Message_RESET: + if s.receiveState == receiveStateReceiving { + s.receiveState = receiveStateReset + } + } + s.maybeDeclareStreamDone() } -func (s *webRTCStream) isClosed() bool { - select { - case <-s.ctx.Done(): - return true - default: - return false +// this is used to force reset a stream +func (s *stream) maybeDeclareStreamDone() { + if (s.sendState == sendStateReset || s.sendState == sendStateDataSent) && + (s.receiveState == receiveStateReset || s.receiveState == receiveStateDataRead) && + len(s.controlMsgQueue) == 0 { + _ = s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) // pion ignores zero times + _ = s.dataChannel.Close() + // TODO: write for the spawned reader to return + s.onDone() } } + +func (s *stream) setCloseError(e error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.readMu.Lock() + defer s.readMu.Unlock() + s.closeErr = e +} diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index e8ffe6e9ee..23e317f314 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -2,116 +2,82 @@ package libp2pwebrtc import ( "errors" - "fmt" "io" - "os" "time" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" ) -// Read from the underlying datachannel. This also -// process sctp control messages such as DCEP, which is -// handled internally by pion, and stream closure which -// is signaled by `Read` on the datachannel returning -// io.EOF. -func (s *webRTCStream) Read(b []byte) (int, error) { - if len(b) == 0 { - return 0, nil +// Read from the underlying datachannel. +// This also process SCTP control messages such as DCEP, which is handled internally by pion, +// and stream closure which is signaled by `Read` on the datachannel returning io.EOF. +func (s *stream) Read(b []byte) (int, error) { + if s.closeErr != nil { + return 0, s.closeErr } - - var ( - readErr error - read int - ) - for read == 0 && readErr == nil { - if s.isClosed() { - if s.stateHandler.IsReset() { - return 0, network.ErrReset - } - return 0, io.EOF - } - read, readErr = s.readMessage(b) + switch s.receiveState { + case receiveStateDataRead: + return 0, io.EOF + case receiveStateReset: + return 0, network.ErrReset } - return read, readErr -} - -func (s *webRTCStream) readMessage(b []byte) (int, error) { - read := copy(b, s.readBuffer) - s.readBuffer = s.readBuffer[read:] - remaining := len(s.readBuffer) - if remaining == 0 && !s.stateHandler.AllowRead() { - log.Debug("[2] stream has no more data to read") - if read != 0 { - return read, nil - } - return read, io.EOF - } - - if read > 0 { - return read, nil - } - - // read from datachannel - var msg pb.Message - err := s.readMessageFromDataChannel(&msg) - if err != nil { - // This case occurs when the remote node goes away - // without writing a FIN message - if errors.Is(err, io.EOF) { - // if the channel was properly closed, return EOF - if !s.stateHandler.AllowRead() && !s.stateHandler.IsReset() { - return 0, io.EOF + if s.nextMessage == nil { + // load the next message + var msg pb.Message + if err := s.readMessageFromDataChannel(&msg); err != nil { + if err == io.EOF { + // if the channel was properly closed, return EOF + if s.receiveState == receiveStateDataRead { + return 0, io.EOF + } + // This case occurs when the remote node closes the stream without writing a FIN message + // There's little we can do here + return 0, errors.New("didn't receive final state for stream") } - return 0, network.ErrReset - } - - if errors.Is(err, os.ErrDeadlineExceeded) && s.stateHandler.IsReset() { - return 0, network.ErrReset + return 0, err } - return read, err + s.nextMessage = &msg } - // append incoming data to read readBuffer - if s.stateHandler.AllowRead() && msg.Message != nil { - s.readBuffer = append(s.readBuffer, msg.GetMessage()...) + n := copy(b, s.nextMessage.Message) + s.nextMessage.Message = s.nextMessage.Message[n:] + if len(s.nextMessage.Message) > 0 { + return n, nil } - // process any flags on the message - if msg.Flag != nil { - s.processIncomingFlag(msg.GetFlag()) + // process flags on the message after reading all the data + s.processIncomingFlag(s.nextMessage.Flag) + s.nextMessage = nil + if s.closeErr != nil { + return n, s.closeErr + } + switch s.receiveState { + case receiveStateDataRead: + return n, io.EOF + case receiveStateReset: + return n, network.ErrReset + default: + return n, nil } - - return read, nil } -func (s *webRTCStream) readMessageFromDataChannel(msg *pb.Message) error { - s.readerMux.Lock() - defer s.readerMux.Unlock() +func (s *stream) readMessageFromDataChannel(msg *pb.Message) error { + s.readMu.Lock() + defer s.readMu.Unlock() return s.reader.ReadMsg(msg) } -func (s *webRTCStream) SetReadDeadline(t time.Time) error { - return s.dataChannel.SetReadDeadline(t) -} +func (s *stream) SetReadDeadline(t time.Time) error { return s.dataChannel.SetReadDeadline(t) } -func (s *webRTCStream) CloseRead() error { - if s.isClosed() { - return nil +func (s *stream) CloseRead() error { + s.receiveState = receiveStateReset + if s.nextMessage != nil { + s.processIncomingFlag(s.nextMessage.Flag) + s.nextMessage = nil } - var err error - s.closeOnce.Do(func() { - err = s.writeMessageToWriter(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) - if err != nil { - log.Debug("could not write STOP_SENDING message") - err = fmt.Errorf("could not close stream for reading: %w", err) - return - } - if s.stateHandler.CloseRead() == stateClosed { - s.close(false, true) - } - }) + err := s.sendControlMessage(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) + s.maybeDeclareStreamDone() return err } diff --git a/p2p/transport/webrtc/stream_state.go b/p2p/transport/webrtc/stream_state.go deleted file mode 100644 index a7b05ec937..0000000000 --- a/p2p/transport/webrtc/stream_state.go +++ /dev/null @@ -1,136 +0,0 @@ -package libp2pwebrtc - -import ( - "sync" - - pb "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" -) - -type channelState uint8 - -const ( - stateOpen channelState = iota - stateReadClosed - stateWriteClosed - stateClosed -) - -type webRTCStreamState struct { - mu sync.RWMutex - state channelState - reset bool -} - -func (ss *webRTCStreamState) HandleInboundFlag(flag pb.Message_Flag) (channelState, bool) { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.state == stateClosed { - return ss.state, ss.reset - } - - switch flag { - case pb.Message_FIN: - ss.closeReadInner() - - case pb.Message_STOP_SENDING: - ss.closeWriteInner() - - case pb.Message_RESET: - ss.closeInner(true) - default: - // ignore values that are invalid for flags - } - - return ss.state, ss.reset -} - -func (ss *webRTCStreamState) State() channelState { - ss.mu.RLock() - defer ss.mu.RUnlock() - return ss.state -} - -func (ss *webRTCStreamState) AllowRead() bool { - ss.mu.RLock() - defer ss.mu.RUnlock() - return ss.state == stateOpen || ss.state == stateWriteClosed -} - -func (ss *webRTCStreamState) CloseRead() channelState { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.state == stateClosed { - return ss.state - } - - ss.closeReadInner() - return ss.state -} - -func (ss *webRTCStreamState) closeReadInner() { - if ss.state == stateOpen { - ss.state = stateReadClosed - } else if ss.state == stateWriteClosed { - ss.closeInner(false) - } -} - -func (ss *webRTCStreamState) AllowWrite() bool { - ss.mu.RLock() - defer ss.mu.RUnlock() - return ss.state == stateOpen || ss.state == stateReadClosed -} - -func (ss *webRTCStreamState) CloseWrite() channelState { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.state == stateClosed { - return ss.state - } - - ss.closeWriteInner() - return ss.state -} - -func (ss *webRTCStreamState) closeWriteInner() { - if ss.state == stateOpen { - ss.state = stateWriteClosed - } else if ss.state == stateReadClosed { - ss.closeInner(false) - } -} - -func (ss *webRTCStreamState) Closed() bool { - ss.mu.RLock() - defer ss.mu.RUnlock() - return ss.state == stateClosed -} -func (ss *webRTCStreamState) Close() { - ss.mu.Lock() - defer ss.mu.Unlock() - ss.state = stateClosed - ss.reset = false -} - -func (ss *webRTCStreamState) Reset() { - ss.mu.Lock() - defer ss.mu.Unlock() - ss.state = stateClosed - ss.reset = true -} - -func (ss *webRTCStreamState) IsReset() bool { - ss.mu.Lock() - defer ss.mu.Unlock() - return ss.reset -} - -func (ss *webRTCStreamState) closeInner(reset bool) { - if ss.state != stateClosed { - ss.state = stateClosed - ss.reset = reset - } -} diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go new file mode 100644 index 0000000000..0de126aed5 --- /dev/null +++ b/p2p/transport/webrtc/stream_test.go @@ -0,0 +1,227 @@ +package libp2pwebrtc + +import ( + "crypto/rand" + "errors" + "io" + "os" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/network" + + "github.com/pion/datachannel" + "github.com/pion/webrtc/v3" + "github.com/stretchr/testify/require" +) + +type detachedChan struct { + rwc datachannel.ReadWriteCloser + dc *webrtc.DataChannel +} + +func getDetachedDataChannels(t *testing.T) (detachedChan, detachedChan) { + s := webrtc.SettingEngine{} + s.DetachDataChannels() + api := webrtc.NewAPI(webrtc.WithSettingEngine(s)) + + offerPC, err := api.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + t.Cleanup(func() { offerPC.Close() }) + offerRWCChan := make(chan datachannel.ReadWriteCloser, 1) + offerDC, err := offerPC.CreateDataChannel("data", nil) + require.NoError(t, err) + offerDC.OnOpen(func() { + rwc, err := offerDC.Detach() + require.NoError(t, err) + offerRWCChan <- rwc + }) + + answerPC, err := api.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + + answerChan := make(chan detachedChan, 1) + answerPC.OnDataChannel(func(dc *webrtc.DataChannel) { + dc.OnOpen(func() { + rwc, err := dc.Detach() + require.NoError(t, err) + answerChan <- detachedChan{rwc: rwc, dc: dc} + }) + }) + t.Cleanup(func() { answerPC.Close() }) + + // Set ICE Candidate handlers. As soon as a PeerConnection has gathered a candidate send it to the other peer + answerPC.OnICECandidate(func(candidate *webrtc.ICECandidate) { + if candidate != nil { + require.NoError(t, offerPC.AddICECandidate(candidate.ToJSON())) + } + }) + offerPC.OnICECandidate(func(candidate *webrtc.ICECandidate) { + if candidate != nil { + require.NoError(t, answerPC.AddICECandidate(candidate.ToJSON())) + } + }) + + // Set the handler for Peer connection state + // This will notify you when the peer has connected/disconnected + offerPC.OnConnectionStateChange(func(s webrtc.PeerConnectionState) { + if s == webrtc.PeerConnectionStateFailed { + t.Log("peer connection failed on offerer") + } + }) + + // Set the handler for Peer connection state + // This will notify you when the peer has connected/disconnected + answerPC.OnConnectionStateChange(func(s webrtc.PeerConnectionState) { + if s == webrtc.PeerConnectionStateFailed { + t.Log("peer connection failed on answerer") + } + }) + + // Now, create an offer + offer, err := offerPC.CreateOffer(nil) + require.NoError(t, err) + require.NoError(t, offerPC.SetLocalDescription(offer)) + require.NoError(t, answerPC.SetRemoteDescription(offer)) + + answer, err := answerPC.CreateAnswer(nil) + require.NoError(t, err) + require.NoError(t, answerPC.SetLocalDescription(answer)) + require.NoError(t, offerPC.SetRemoteDescription(answer)) + + return <-answerChan, detachedChan{rwc: <-offerRWCChan, dc: offerDC} +} + +func TestStreamSimpleReadWriteClose(t *testing.T) { + client, server := getDetachedDataChannels(t) + + var clientDone, serverDone bool + clientStr := newStream(client.dc, client.rwc, nil, nil, func() { clientDone = true }) + serverStr := newStream(server.dc, server.rwc, nil, nil, func() { serverDone = true }) + + // send a foobar from the client + _, err := clientStr.Write([]byte("foobar")) + require.NoError(t, err) + require.NoError(t, clientStr.CloseWrite()) + // writing after closing should error + _, err = clientStr.Write([]byte("foobar")) + require.Error(t, err) + require.False(t, clientDone) + + // now read all the data on the server side + b, err := io.ReadAll(serverStr) + require.NoError(t, err) + require.Equal(t, []byte("foobar"), b) + // reading again should give another io.EOF + n, err := serverStr.Read(make([]byte, 10)) + require.Zero(t, n) + require.ErrorIs(t, err, io.EOF) + require.False(t, serverDone) + + // send something back + _, err = serverStr.Write([]byte("lorem ipsum")) + require.NoError(t, err) + require.NoError(t, serverStr.CloseWrite()) + require.True(t, serverDone) + // and read it at the client + require.False(t, clientDone) + b, err = io.ReadAll(clientStr) + require.NoError(t, err) + require.Equal(t, []byte("lorem ipsum"), b) + require.True(t, clientDone) +} + +func TestStreamResets(t *testing.T) { + client, server := getDetachedDataChannels(t) + + var clientDone, serverDone bool + clientStr := newStream(client.dc, client.rwc, nil, nil, func() { clientDone = true }) + serverStr := newStream(server.dc, server.rwc, nil, nil, func() { serverDone = true }) + + // send a foobar from the client + _, err := clientStr.Write([]byte("foobar")) + require.NoError(t, err) + _, err = serverStr.Write([]byte("lorem ipsum")) + require.NoError(t, err) + require.NoError(t, clientStr.Reset()) // resetting resets both directions + require.True(t, clientDone) + // attempting to write more data should result in a reset error + _, err = clientStr.Write([]byte("foobar")) + require.ErrorIs(t, err, network.ErrReset) + // read what the server sent + b, err := io.ReadAll(clientStr) + require.Empty(t, b) + require.ErrorIs(t, err, network.ErrReset) + + // read the data on the server side + require.False(t, serverDone) + b, err = io.ReadAll(serverStr) + require.Equal(t, []byte("foobar"), b) + require.ErrorIs(t, err, network.ErrReset) + require.Eventually(t, func() bool { + _, err := serverStr.Write([]byte("foobar")) + return errors.Is(err, network.ErrReset) + }, time.Second, 50*time.Millisecond) + require.True(t, serverDone) +} + +func TestStreamReadDeadlineAsync(t *testing.T) { + client, server := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) + serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + + timeout := 100 * time.Millisecond + if os.Getenv("CI") != "" { + timeout *= 5 + } + start := time.Now() + clientStr.SetReadDeadline(start.Add(timeout)) + _, err := clientStr.Read([]byte{0}) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + took := time.Since(start) + require.GreaterOrEqual(t, took, timeout) + require.LessOrEqual(t, took, timeout*3/2) + // repeated calls should return immediately + start = time.Now() + _, err = clientStr.Read([]byte{0}) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + require.LessOrEqual(t, time.Since(start), timeout/3) + // clear the deadline + clientStr.SetReadDeadline(time.Time{}) + _, err = serverStr.Write([]byte("foobar")) + require.NoError(t, err) + _, err = clientStr.Read([]byte{0}) + require.NoError(t, err) + require.LessOrEqual(t, time.Since(start), timeout/3) +} + +func TestStreamWriteDeadlineAsync(t *testing.T) { + client, server := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) + serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + _ = serverStr + + b := make([]byte, 1024) + rand.Read(b) + start := time.Now() + timeout := 100 * time.Millisecond + if os.Getenv("CI") != "" { + timeout *= 5 + } + clientStr.SetWriteDeadline(start.Add(timeout)) + var hitDeadline bool + for i := 0; i < 2000; i++ { + if _, err := clientStr.Write(b); err != nil { + t.Logf("wrote %d kB", i) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + hitDeadline = true + break + } + } + require.True(t, hitDeadline) + took := time.Since(start) + require.GreaterOrEqual(t, took, timeout) + require.LessOrEqual(t, took, timeout*3/2) +} diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 24be04ffe6..2b2bfcd9eb 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -2,178 +2,188 @@ package libp2pwebrtc import ( "errors" - "fmt" - "io" - "math" "os" "time" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" - "google.golang.org/protobuf/proto" ) -func (s *webRTCStream) Write(b []byte) (int, error) { - if !s.stateHandler.AllowWrite() { - return 0, io.ErrClosedPipe +var errWriteAfterClose = errors.New("write after close") + +// If we have less space than minMessageSize, we don't put a new message on the data channel. +// Instead, we wait until more space opens up. +const minMessageSize = 1 << 10 + +func (s *stream) Write(b []byte) (int, error) { + s.writeMu.Lock() + defer s.writeMu.Unlock() + + if s.closeErr != nil { + return 0, s.closeErr + } + switch s.sendState { + case sendStateReset: + return 0, network.ErrReset + case sendStateDataSent: + return 0, errWriteAfterClose } // Check if there is any message on the wire. This is used for control // messages only when the read side of the stream is closed - if s.stateHandler.State() == stateReadClosed { + if s.receiveState != receiveStateReceiving { s.readLoopOnce.Do(s.spawnControlMessageReader) } - const chunkSize = maxMessageSize - protoOverhead - varintOverhead + var writeDeadlineTimer *time.Timer + defer func() { + if writeDeadlineTimer != nil { + writeDeadlineTimer.Stop() + } + }() var n int - for len(b) > 0 { - end := len(b) - if chunkSize < end { - end = chunkSize + if s.closeErr != nil { + return n, s.closeErr + } + switch s.sendState { + case sendStateReset: + return n, network.ErrReset + case sendStateDataSent: + return n, errWriteAfterClose } - err := s.writeMessage(&pb.Message{Message: b[:end]}) - n += end - b = b[end:] - if err != nil { + writeDeadline := s.writeDeadline + // deadline deleted, stop and remove the timer + if writeDeadline.IsZero() && writeDeadlineTimer != nil { + writeDeadlineTimer.Stop() + writeDeadlineTimer = nil + } + var writeDeadlineChan <-chan time.Time + if !writeDeadline.IsZero() { + if writeDeadlineTimer == nil { + writeDeadlineTimer = time.NewTimer(time.Until(writeDeadline)) + } else { + if !writeDeadlineTimer.Stop() { + <-writeDeadlineTimer.C + } + writeDeadlineTimer.Reset(time.Until(writeDeadline)) + } + writeDeadlineChan = writeDeadlineTimer.C + } + + availableSpace := s.availableSendSpace() + if availableSpace < minMessageSize { + s.writeMu.Unlock() + select { + case <-s.writeAvailable: + case <-writeDeadlineChan: + s.writeMu.Lock() + return n, os.ErrDeadlineExceeded + case <-s.sendStateChanged: + case <-s.writeDeadlineUpdated: + } + s.writeMu.Lock() + continue + } + end := maxMessageSize + if end > availableSpace { + end = availableSpace + } + end -= protoOverhead + varintOverhead + if end > len(b) { + end = len(b) + } + msg := &pb.Message{Message: b[:end]} + if err := s.writer.WriteMsg(msg); err != nil { return n, err } + n += end + b = b[end:] } - return n, nil } // used for reading control messages while writing, in case the reader is closed, // as to ensure we do still get control messages. This is important as according to the spec // our data and control channels are intermixed on the same conn. -func (s *webRTCStream) spawnControlMessageReader() { +func (s *stream) spawnControlMessageReader() { + if s.nextMessage != nil { + s.processIncomingFlag(s.nextMessage.Flag) + s.nextMessage = nil + } go func() { - // zero the read deadline, so read call only returns - // when the underlying datachannel closes or there is - // a message on the channel - s.dataChannel.SetReadDeadline(time.Time{}) - var msg pb.Message + // no deadline needed, Read will return once there's a new message, or an error occurred + _ = s.dataChannel.SetReadDeadline(time.Time{}) for { - if s.stateHandler.Closed() { + var msg pb.Message + if err := s.readMessageFromDataChannel(&msg); err != nil { return } - err := s.readMessageFromDataChannel(&msg) - if err != nil { - if errors.Is(err, io.EOF) { - s.close(true, true) - } - return - } - if msg.Flag != nil { - state, reset := s.stateHandler.HandleInboundFlag(msg.GetFlag()) - if state == stateClosed { - log.Debug("closing: after handle inbound flag") - s.close(reset, true) - } - } + s.processIncomingFlag(msg.Flag) } }() } -func (s *webRTCStream) writeMessage(msg *pb.Message) error { - var writeDeadlineTimer *time.Timer - defer func() { - if writeDeadlineTimer != nil { - writeDeadlineTimer.Stop() - } - }() - - for { - if !s.stateHandler.AllowWrite() { - return io.ErrClosedPipe - } - - writeDeadline, hasWriteDeadline := s.getWriteDeadline() - if !hasWriteDeadline { - // writeDeadline = time.Unix(999999999999999999, 0) - // Does this cause the timer to overflow ? https://cs.opensource.google/go/go/+/master:src/time/sleep.go;l=32?q=runtimeTimer&ss=go%2Fgo - // this could be causing an overflow above https://cs.opensource.google/go/go/+/master:src/time/time.go;l=1123?q=unixTim&ss=go%2Fgo - // Go adds 62135596800 to the seconds parameter of `time.Unix` - writeDeadline = time.Unix(math.MaxInt64-62135596801, 0) - } - if writeDeadlineTimer == nil { - writeDeadlineTimer = time.NewTimer(time.Until(writeDeadline)) - } else { - if !writeDeadlineTimer.Stop() { - <-writeDeadlineTimer.C - } - writeDeadlineTimer.Reset(time.Until(writeDeadline)) - } +func (s *stream) SetWriteDeadline(t time.Time) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + s.writeDeadline = t + select { + case s.writeDeadlineUpdated <- struct{}{}: + default: + } + return nil +} - bufferedAmount := int(s.dataChannel.BufferedAmount()) - addedBuffer := bufferedAmount + varintOverhead + proto.Size(msg) - if addedBuffer > maxBufferedAmount { - select { - case <-writeDeadlineTimer.C: - return os.ErrDeadlineExceeded - case <-s.writeAvailable: - return s.writeMessageToWriter(msg) - case <-s.ctx.Done(): - if s.stateHandler.IsReset() { - return network.ErrReset - } - return io.ErrClosedPipe - case <-s.writerDeadlineUpdated: - } - } else { - return s.writeMessageToWriter(msg) - } +func (s *stream) availableSendSpace() int { + buffered := int(s.dataChannel.BufferedAmount()) + availableSpace := maxBufferedAmount - buffered + if availableSpace < 0 { // this should never happen, but better check + log.Errorw("data channel buffered more data than the maximum amount", "max", maxBufferedAmount, "buffered", buffered) } + return availableSpace } -func (s *webRTCStream) writeMessageToWriter(msg *pb.Message) error { - s.writerMux.Lock() - defer s.writerMux.Unlock() - return s.writer.WriteMsg(msg) +const controlMsgSize = 100 // TODO: use actual message size + +func (s *stream) sendControlMessage(msg *pb.Message) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + + available := s.availableSendSpace() + if controlMsgSize < available { + return s.writer.WriteMsg(msg) + } + s.controlMsgQueue = append(s.controlMsgQueue, msg) + return nil } -func (s *webRTCStream) SetWriteDeadline(t time.Time) error { - s.writerDeadlineMux.Lock() - defer s.writerDeadlineMux.Unlock() - s.writerDeadline = t +func (s *stream) cancelWrite() error { + if s.sendState != sendStateSending { + return nil + } + s.sendState = sendStateReset select { - case s.writerDeadlineUpdated <- struct{}{}: + case s.sendStateChanged <- struct{}{}: default: } + if err := s.sendControlMessage(&pb.Message{Flag: pb.Message_RESET.Enum()}); err != nil { + return err + } + s.maybeDeclareStreamDone() return nil } -func (s *webRTCStream) getWriteDeadline() (time.Time, bool) { - s.writerDeadlineMux.Lock() - defer s.writerDeadlineMux.Unlock() - return s.writerDeadline, !s.writerDeadline.IsZero() -} - -func (s *webRTCStream) CloseWrite() error { - if s.isClosed() { +func (s *stream) CloseWrite() error { + if s.sendState != sendStateSending { return nil } - var err error - s.closeOnce.Do(func() { - err = s.writeMessage(&pb.Message{Flag: pb.Message_FIN.Enum()}) - if err != nil { - log.Debug("could not write FIN message") - err = fmt.Errorf("close stream for writing: %w", err) - return - } - // if successfully written, process the outgoing flag - state := s.stateHandler.CloseWrite() - // unblock and fail any ongoing writes - select { - case s.writeAvailable <- struct{}{}: - default: - } - // check if closure required - if state == stateClosed { - s.close(false, true) - } - }) - return err + s.sendState = sendStateDataSent + if err := s.sendControlMessage(&pb.Message{Flag: pb.Message_FIN.Enum()}); err != nil { + return err + } + s.maybeDeclareStreamDone() + return nil } diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index e3a88a3349..44787c9093 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -477,7 +477,7 @@ func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash return result, nil } -func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, datachannel *webRTCStream, peer peer.ID, hash crypto.Hash, inbound bool) (sec.SecureConn, error) { +func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, datachannel *stream, peer peer.ID, hash crypto.Hash, inbound bool) (sec.SecureConn, error) { prologue, err := t.generateNoisePrologue(pc, hash, inbound) if err != nil { return nil, fmt.Errorf("generate prologue: %w", err) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 7a53534e6a..1ceab46b58 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -686,55 +686,6 @@ func TestTransportWebRTC_Close(t *testing.T) { }) } -func TestReceiveFlagsAfterReadClosed(t *testing.T) { - tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) - listener, err := tr.Listen(listenMultiaddr) - require.NoError(t, err) - - tr1, connectingPeer := getTransport(t) - done := make(chan struct{}) - - go func() { - lconn, err := listener.Accept() - require.NoError(t, err) - require.Equal(t, connectingPeer, lconn.RemotePeer()) - stream, err := lconn.AcceptStream() - require.NoError(t, err) - b := make([]byte, 6) - n, err := stream.Read(b) - require.NoError(t, err) - require.Equal(t, []byte("foobar"), b[:n]) - // stop reader - require.NoError(t, stream.Reset()) - done <- struct{}{} - }() - - conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) - require.NoError(t, err) - stream, err := conn.OpenStream(context.Background()) - require.NoError(t, err) - - require.NoError(t, stream.CloseRead()) - _, err = stream.Read([]byte{0}) - require.ErrorIs(t, err, io.EOF) - _, err = stream.Write([]byte("foobar")) - require.NoError(t, err) - <-done - // at some point we should receive the stream reset - start := time.Now() - for { - if _, err := stream.Write([]byte{0x42}); err != nil { - require.ErrorIs(t, err, io.ErrClosedPipe) - break - } - if time.Since(start) > 3*time.Second { - require.Fail(t, "didn't receive stream reset") - } - time.Sleep(time.Millisecond) - } -} - func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { tr, listeningPeer := getTransport(t) listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) @@ -808,7 +759,7 @@ func TestConnectionTimeoutOnListener(t *testing.T) { // TODO: return timeout errors here for { if _, err := str.Write([]byte("test")); err != nil { - require.ErrorIs(t, err, io.ErrClosedPipe) + require.True(t, os.IsTimeout(err)) break } if time.Since(start) > 5*time.Second { @@ -819,12 +770,10 @@ func TestConnectionTimeoutOnListener(t *testing.T) { } // make sure that accepting a stream also returns an error... _, err = conn.AcceptStream() - // TODO: return timeout errors here - require.Error(t, err) + require.True(t, os.IsTimeout(err)) // ... as well as opening a new stream _, err = conn.OpenStream(context.Background()) - // TODO: return timeout errors here - require.Error(t, err) + require.True(t, os.IsTimeout(err)) } func TestMaxInFlightRequests(t *testing.T) { @@ -832,9 +781,9 @@ func TestMaxInFlightRequests(t *testing.T) { tr, listeningPeer := getTransport(t, WithListenerMaxInFlightConnections(count), ) - listener, err := tr.Listen(ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP))) + ln, err := tr.Listen(ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP))) require.NoError(t, err) - defer listener.Close() + defer ln.Close() var wg sync.WaitGroup var success, fails atomic.Int32 @@ -845,7 +794,7 @@ func TestMaxInFlightRequests(t *testing.T) { dialer, _ := getTransport(t) ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() - if _, err := dialer.Dial(ctx, listener.Multiaddr(), listeningPeer); err == nil { + if _, err := dialer.Dial(ctx, ln.Multiaddr(), listeningPeer); err == nil { success.Add(1) } else { fails.Add(1) From 96a65b034ed60a5ea9f519522ef5274ce9fd4eb5 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 11:41:12 -0700 Subject: [PATCH 38/86] webrtc: remove debug identifier from connection --- p2p/transport/webrtc/connection.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index a0942c576d..68290d2f0b 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "math" - "math/rand" "net" "sync" "sync/atomic" @@ -15,7 +14,9 @@ import ( "github.com/libp2p/go-libp2p/core/peer" tpt "github.com/libp2p/go-libp2p/core/transport" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-msgio" + ma "github.com/multiformats/go-multiaddr" "github.com/pion/datachannel" "github.com/pion/webrtc/v3" @@ -40,8 +41,6 @@ type acceptStream struct { } type connection struct { - // debug identifier for the connection - dbgId int pc *webrtc.PeerConnection transport *WebRTCTransport scope network.ConnManagementScope @@ -82,7 +81,6 @@ func newConnection( ctx, cancel := context.WithCancel(context.Background()) conn := &connection{ - dbgId: rand.Intn(65536), pc: pc, transport: transport, scope: scope, @@ -229,7 +227,6 @@ func (c *connection) removeStream(id uint16) { } func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { - fmt.Printf("[%s][%d] handling peerconnection state: %s\n", c.localPeer, c.dbgId, state) if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed { // reset any streams if c.IsClosed() { From 12c03355479d5174d6df14c33afafde4df835943 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 09:50:56 -0700 Subject: [PATCH 39/86] transport tests: add WebRTC --- p2p/test/transport/transport_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 370ef9b114..2ec7ba4005 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -27,6 +27,8 @@ import ( "github.com/libp2p/go-libp2p/p2p/protocol/ping" "github.com/libp2p/go-libp2p/p2p/security/noise" tls "github.com/libp2p/go-libp2p/p2p/security/tls" + libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) @@ -134,6 +136,21 @@ var transportsToTest = []TransportTestCase{ return h }, }, + { + Name: "WebRTC", + HostGenerator: func(t *testing.T, opts TransportTestCaseOpts) host.Host { + libp2pOpts := transformOpts(opts) + libp2pOpts = append(libp2pOpts, libp2p.Transport(libp2pwebrtc.New)) + if opts.NoListen { + libp2pOpts = append(libp2pOpts, libp2p.NoListenAddrs) + } else { + libp2pOpts = append(libp2pOpts, libp2p.ListenAddrStrings("/ip4/127.0.0.1/udp/0/webrtc-direct")) + } + h, err := libp2p.New(libp2pOpts...) + require.NoError(t, err) + return h + }, + }, } func TestPing(t *testing.T) { From 06a81004abeb4e55db4ea4282fb08fc1d11ab86c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 11:44:42 -0700 Subject: [PATCH 40/86] webrtc: rename some variables and types in the connection --- p2p/transport/webrtc/connection.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 68290d2f0b..18fa176437 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -35,7 +35,7 @@ func (errConnectionTimeout) Error() string { return "connection timeout" } func (errConnectionTimeout) Timeout() bool { return true } func (errConnectionTimeout) Temporary() bool { return false } -type acceptStream struct { +type dataChannel struct { stream datachannel.ReadWriteCloser channel *webrtc.DataChannel } @@ -57,7 +57,7 @@ type connection struct { m sync.Mutex streams map[uint16]*stream - acceptQueue chan acceptStream + acceptQueue chan dataChannel idAllocator *sidAllocator ctx context.Context @@ -96,7 +96,7 @@ func newConnection( streams: make(map[uint16]*stream), idAllocator: idAllocator, - acceptQueue: make(chan acceptStream, maxAcceptQueueLen), + acceptQueue: make(chan dataChannel, maxAcceptQueueLen), } pc.OnConnectionStateChange(conn.onConnectionStateChange) @@ -111,7 +111,7 @@ func newConnection( return } select { - case conn.acceptQueue <- acceptStream{rwc, dc}: + case conn.acceptQueue <- dataChannel{rwc, dc}: default: log.Warnf("connection busy, rejecting stream") b, _ := proto.Marshal(&pb.Message{Flag: pb.Message_RESET.Enum()}) @@ -186,19 +186,19 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { select { case <-c.ctx.Done(): return nil, c.closeErr - case str := <-c.acceptQueue: - stream := newStream( - str.channel, - str.stream, + case dc := <-c.acceptQueue: + str := newStream( + dc.channel, + dc.stream, nil, nil, - func() { c.removeStream(*str.channel.ID()) }, + func() { c.removeStream(*dc.channel.ID()) }, ) - if err := c.addStream(stream); err != nil { - stream.Close() + if err := c.addStream(str); err != nil { + str.Close() return nil, err } - return stream, nil + return str, nil } } @@ -210,13 +210,13 @@ func (c *connection) RemoteMultiaddr() ma.Multiaddr { return c.remoteMultiaddr } func (c *connection) Scope() network.ConnScope { return c.scope } func (c *connection) Transport() tpt.Transport { return c.transport } -func (c *connection) addStream(stream *stream) error { +func (c *connection) addStream(str *stream) error { c.m.Lock() defer c.m.Unlock() - if _, ok := c.streams[stream.id]; ok { + if _, ok := c.streams[str.id]; ok { return errors.New("stream ID already exists") } - c.streams[stream.id] = stream + c.streams[str.id] = str return nil } From f2dd0a21e4c91ac6ce3f55d16ca5f9cf5e24b0fc Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 12:35:45 -0700 Subject: [PATCH 41/86] webrtc: fix incorrect use of stream IDs, simplify The first stream has ID 1, not 3. --- p2p/transport/webrtc/connection.go | 72 +++++++++--------------------- 1 file changed, 21 insertions(+), 51 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 18fa176437..2bd454c075 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -54,11 +54,11 @@ type connection struct { remoteKey ic.PubKey remoteMultiaddr ma.Multiaddr - m sync.Mutex - streams map[uint16]*stream + m sync.Mutex + streams map[uint16]*stream + nextStreamID atomic.Int32 acceptQueue chan dataChannel - idAllocator *sidAllocator ctx context.Context cancel context.CancelFunc @@ -77,10 +77,8 @@ func newConnection( remoteKey ic.PubKey, remoteMultiaddr ma.Multiaddr, ) (*connection, error) { - idAllocator := newSidAllocator(direction) - ctx, cancel := context.WithCancel(context.Background()) - conn := &connection{ + c := &connection{ pc: pc, transport: transport, scope: scope, @@ -94,14 +92,20 @@ func newConnection( ctx: ctx, cancel: cancel, streams: make(map[uint16]*stream), - idAllocator: idAllocator, acceptQueue: make(chan dataChannel, maxAcceptQueueLen), } + switch direction { + case network.DirInbound: + c.nextStreamID.Store(1) + case network.DirOutbound: + // stream ID 0 is used for the Noise handshake stream + c.nextStreamID.Store(2) + } - pc.OnConnectionStateChange(conn.onConnectionStateChange) + pc.OnConnectionStateChange(c.onConnectionStateChange) pc.OnDataChannel(func(dc *webrtc.DataChannel) { - if conn.IsClosed() { + if c.IsClosed() { return } dc.OnOpen(func() { @@ -111,7 +115,7 @@ func newConnection( return } select { - case conn.acceptQueue <- dataChannel{rwc, dc}: + case c.acceptQueue <- dataChannel{rwc, dc}: default: log.Warnf("connection busy, rejecting stream") b, _ := proto.Marshal(&pb.Message{Flag: pb.Message_RESET.Enum()}) @@ -122,7 +126,7 @@ func newConnection( }) }) - return conn, nil + return c, nil } // ConnState implements transport.CapableConn @@ -156,11 +160,12 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error return nil, c.closeErr } - streamID, err := c.idAllocator.nextID() - if err != nil { - return nil, err + id := c.nextStreamID.Add(2) - 2 + if id > math.MaxUint16 { + return nil, errors.New("exhausted stream ID space") } - dc, err := c.pc.CreateDataChannel("", &webrtc.DataChannelInit{ID: streamID}) + streamID := uint16(id) + dc, err := c.pc.CreateDataChannel("", &webrtc.DataChannelInit{ID: &streamID}) if err != nil { return nil, err } @@ -173,7 +178,7 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error rwc, nil, nil, - func() { c.removeStream(*streamID) }, + func() { c.removeStream(streamID) }, ) if err := c.addStream(str); err != nil { str.Close() @@ -305,38 +310,3 @@ func (c *connection) setRemotePeer(id peer.ID) { func (c *connection) setRemotePublicKey(key ic.PubKey) { c.remoteKey = key } - -// sidAllocator is a helper struct to allocate stream IDs for datachannels. ID -// reuse is not currently implemented. This prevents streams in pion from hanging -// with `invalid DCEP message` errors. -// The id is picked using the scheme described in: -// https://datatracker.ietf.org/doc/html/draft-ietf-rtcweb-data-channel-08#section-6.5 -// By definition, the DTLS role for inbound connections is set to DTLS Server, -// and outbound connections are DTLS Client. -type sidAllocator struct { - n atomic.Uint32 -} - -func newSidAllocator(direction network.Direction) *sidAllocator { - switch direction { - case network.DirInbound: - // server will use odd values - a := new(sidAllocator) - a.n.Store(1) - return a - case network.DirOutbound: - // client will use even values - return new(sidAllocator) - default: - panic(fmt.Sprintf("create SID allocator for direction: %s", direction)) - } -} - -func (a *sidAllocator) nextID() (*uint16, error) { - nxt := a.n.Add(2) - if nxt > math.MaxUint16 { - return nil, fmt.Errorf("sid exhausted") - } - result := uint16(nxt) - return &result, nil -} From 2a76e26fe6bb7e98c414f0a3ebb075fc79b67264 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 15:31:11 -0700 Subject: [PATCH 42/86] webrtc: only send STOP_SENDING from receiving read stream state --- p2p/transport/webrtc/stream_read.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 23e317f314..2b080e4614 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -72,12 +72,18 @@ func (s *stream) readMessageFromDataChannel(msg *pb.Message) error { func (s *stream) SetReadDeadline(t time.Time) error { return s.dataChannel.SetReadDeadline(t) } func (s *stream) CloseRead() error { + s.readMu.Lock() + defer s.readMu.Unlock() + s.receiveState = receiveStateReset if s.nextMessage != nil { s.processIncomingFlag(s.nextMessage.Flag) s.nextMessage = nil } - err := s.sendControlMessage(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) - s.maybeDeclareStreamDone() - return err + if s.receiveState == receiveStateReceiving && s.closeErr == nil { + err := s.sendControlMessage(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) + s.maybeDeclareStreamDone() + return err + } + return nil } From 482b3504177c935dfee9b413046afe99e54ae30a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 18:32:09 -0700 Subject: [PATCH 43/86] webrtc: set read and write deadline on stream.SetDeadline --- p2p/transport/webrtc/stream.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 6ddae290b3..a402020e06 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -164,6 +164,7 @@ func (s *stream) LocalAddr() net.Addr { return s.laddr } func (s *stream) RemoteAddr() net.Addr { return s.raddr } func (s *stream) SetDeadline(t time.Time) error { + _ = s.SetReadDeadline(t) return s.SetWriteDeadline(t) } From d55e0e3fd48e8c61d56f2aef550cd7616dc221f9 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 3 Jul 2023 19:22:58 -0700 Subject: [PATCH 44/86] webrtc: use a single stream mutex, prevent (0, nil) Read return values --- p2p/transport/webrtc/stream.go | 20 ++--- p2p/transport/webrtc/stream_read.go | 104 +++++++++++++------------ p2p/transport/webrtc/stream_test.go | 83 +++++++++++++++++++- p2p/transport/webrtc/stream_write.go | 28 ++++--- p2p/transport/webrtc/transport_test.go | 61 --------------- 5 files changed, 161 insertions(+), 135 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index a402020e06..ef33c20c5e 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -59,10 +59,10 @@ const ( // Package pion detached data channel into a net.Conn // and then a network.MuxedStream type stream struct { + mx sync.Mutex // pbio.Reader is not thread safe, // and while our Read is not promised to be thread safe, // we ourselves internally read from multiple routines... - readMu sync.Mutex reader pbio.Reader // this buffer is limited up to a single message. Reason we need it // is because a reader might read a message midway, and so we need a @@ -72,7 +72,6 @@ type stream struct { // The public Write API is not promised to be thread safe, // but we need to be able to write control messages. - writeMu sync.Mutex writer pbio.Writer sendStateChanged chan struct{} sendState sendState @@ -118,8 +117,8 @@ func newStream( channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) channel.OnBufferedAmountLow(func() { - s.writeMu.Lock() - defer s.writeMu.Unlock() + s.mx.Lock() + defer s.mx.Unlock() // first send out queued control messages for len(s.controlMsgQueue) > 0 { msg := s.controlMsgQueue[0] @@ -171,16 +170,12 @@ func (s *stream) SetDeadline(t time.Time) error { // processIncomingFlag process the flag on an incoming message // It needs to be called with msg.Flag, not msg.GetFlag(), // otherwise we'd misinterpret the default value. +// It needs to be called while the mutex is locked. func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { if flag == nil { return } - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.readMu.Lock() - defer s.readMu.Unlock() - switch *flag { case pb.Message_FIN: if s.receiveState == receiveStateReceiving { @@ -215,9 +210,8 @@ func (s *stream) maybeDeclareStreamDone() { } func (s *stream) setCloseError(e error) { - s.writeMu.Lock() - defer s.writeMu.Unlock() - s.readMu.Lock() - defer s.readMu.Unlock() + s.mx.Lock() + defer s.mx.Unlock() + s.closeErr = e } diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 2b080e4614..43fdcee088 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -9,10 +9,14 @@ import ( "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" ) -// Read from the underlying datachannel. -// This also process SCTP control messages such as DCEP, which is handled internally by pion, -// and stream closure which is signaled by `Read` on the datachannel returning io.EOF. func (s *stream) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + + s.mx.Lock() + defer s.mx.Unlock() + if s.closeErr != nil { return 0, s.closeErr } @@ -23,67 +27,71 @@ func (s *stream) Read(b []byte) (int, error) { return 0, network.ErrReset } - if s.nextMessage == nil { - // load the next message - var msg pb.Message - if err := s.readMessageFromDataChannel(&msg); err != nil { - if err == io.EOF { - // if the channel was properly closed, return EOF - if s.receiveState == receiveStateDataRead { - return 0, io.EOF + var read int + for { + if s.nextMessage == nil { + // load the next message + s.mx.Unlock() + var msg pb.Message + if err := s.reader.ReadMsg(&msg); err != nil { + s.mx.Lock() + if err == io.EOF { + // if the channel was properly closed, return EOF + if s.receiveState == receiveStateDataRead { + return 0, io.EOF + } + // This case occurs when the remote node closes the stream without writing a FIN message + // There's little we can do here + return 0, errors.New("didn't receive final state for stream") } - // This case occurs when the remote node closes the stream without writing a FIN message - // There's little we can do here - return 0, errors.New("didn't receive final state for stream") + return 0, err } - return 0, err + s.mx.Lock() + s.nextMessage = &msg } - s.nextMessage = &msg - } - n := copy(b, s.nextMessage.Message) - s.nextMessage.Message = s.nextMessage.Message[n:] - if len(s.nextMessage.Message) > 0 { - return n, nil - } + if len(s.nextMessage.Message) > 0 { + n := copy(b, s.nextMessage.Message) + read += n + s.nextMessage.Message = s.nextMessage.Message[n:] + return read, nil + } - // process flags on the message after reading all the data - s.processIncomingFlag(s.nextMessage.Flag) - s.nextMessage = nil - if s.closeErr != nil { - return n, s.closeErr - } - switch s.receiveState { - case receiveStateDataRead: - return n, io.EOF - case receiveStateReset: - return n, network.ErrReset - default: - return n, nil + // process flags on the message after reading all the data + s.processIncomingFlag(s.nextMessage.Flag) + s.nextMessage = nil + if s.closeErr != nil { + return read, s.closeErr + } + switch s.receiveState { + case receiveStateDataRead: + return read, io.EOF + case receiveStateReset: + s.dataChannel.SetReadDeadline(time.Time{}) + return read, network.ErrReset + } } } -func (s *stream) readMessageFromDataChannel(msg *pb.Message) error { - s.readMu.Lock() - defer s.readMu.Unlock() - return s.reader.ReadMsg(msg) -} - func (s *stream) SetReadDeadline(t time.Time) error { return s.dataChannel.SetReadDeadline(t) } func (s *stream) CloseRead() error { - s.readMu.Lock() - defer s.readMu.Unlock() + s.mx.Lock() + defer s.mx.Unlock() - s.receiveState = receiveStateReset if s.nextMessage != nil { s.processIncomingFlag(s.nextMessage.Flag) s.nextMessage = nil } + var err error if s.receiveState == receiveStateReceiving && s.closeErr == nil { - err := s.sendControlMessage(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) - s.maybeDeclareStreamDone() - return err + err = s.sendControlMessage(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) } - return nil + s.receiveState = receiveStateReset + s.maybeDeclareStreamDone() + + // make any calls to Read blocking on ReadMsg return immediately + s.dataChannel.SetReadDeadline(time.Now()) + + return err } diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index 0de126aed5..02beef4edc 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-libp2p/core/network" "github.com/pion/datachannel" @@ -100,8 +102,9 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { serverStr := newStream(server.dc, server.rwc, nil, nil, func() { serverDone = true }) // send a foobar from the client - _, err := clientStr.Write([]byte("foobar")) + n, err := clientStr.Write([]byte("foobar")) require.NoError(t, err) + require.Equal(t, 6, n) require.NoError(t, clientStr.CloseWrite()) // writing after closing should error _, err = clientStr.Write([]byte("foobar")) @@ -113,7 +116,7 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("foobar"), b) // reading again should give another io.EOF - n, err := serverStr.Read(make([]byte, 10)) + n, err = serverStr.Read(make([]byte, 10)) require.Zero(t, n) require.ErrorIs(t, err, io.EOF) require.False(t, serverDone) @@ -131,6 +134,82 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { require.True(t, clientDone) } +func TestStreamPartialReads(t *testing.T) { + client, server := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) + serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + + _, err := serverStr.Write([]byte("foobar")) + require.NoError(t, err) + require.NoError(t, serverStr.CloseWrite()) + + n, err := clientStr.Read([]byte{}) // empty read + require.NoError(t, err) + require.Zero(t, n) + b := make([]byte, 3) + n, err = clientStr.Read(b) + require.NoError(t, err) + require.Equal(t, []byte("foo"), b) + b, err = io.ReadAll(clientStr) + require.NoError(t, err) + require.Equal(t, []byte("bar"), b) +} + +func TestStreamSkipEmptyFrames(t *testing.T) { + client, server := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) + serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + + for i := 0; i < 10; i++ { + require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{})) + } + require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{Message: []byte("foo")})) + for i := 0; i < 10; i++ { + require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{})) + } + require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{Message: []byte("bar")})) + for i := 0; i < 10; i++ { + require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{})) + } + require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{Flag: pb.Message_FIN.Enum()})) + + var read []byte + var count int + for i := 0; i < 100; i++ { + b := make([]byte, 10) + count++ + n, err := clientStr.Read(b) + read = append(read, b[:n]...) + if err == io.EOF { + break + } + require.NoError(t, err) + } + require.LessOrEqual(t, count, 3, "should've taken a maximum of 3 reads") + require.Equal(t, []byte("foobar"), read) +} + +func TestStreamReadReturnsOnClose(t *testing.T) { + client, _ := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) + // serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + errChan := make(chan error, 1) + go func() { + _, err := clientStr.Read([]byte{0}) + errChan <- err + }() + require.NoError(t, clientStr.Close()) + select { + case err := <-errChan: + require.ErrorIs(t, err, network.ErrReset) + case <-time.After(500 * time.Millisecond): + t.Fatal("timeout") + } +} + func TestStreamResets(t *testing.T) { client, server := getDetachedDataChannels(t) diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 2b2bfcd9eb..4578c564c1 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -16,8 +16,8 @@ var errWriteAfterClose = errors.New("write after close") const minMessageSize = 1 << 10 func (s *stream) Write(b []byte) (int, error) { - s.writeMu.Lock() - defer s.writeMu.Unlock() + s.mx.Lock() + defer s.mx.Unlock() if s.closeErr != nil { return 0, s.closeErr @@ -75,16 +75,16 @@ func (s *stream) Write(b []byte) (int, error) { availableSpace := s.availableSendSpace() if availableSpace < minMessageSize { - s.writeMu.Unlock() + s.mx.Unlock() select { case <-s.writeAvailable: case <-writeDeadlineChan: - s.writeMu.Lock() + s.mx.Lock() return n, os.ErrDeadlineExceeded case <-s.sendStateChanged: case <-s.writeDeadlineUpdated: } - s.writeMu.Lock() + s.mx.Lock() continue } end := maxMessageSize @@ -113,22 +113,25 @@ func (s *stream) spawnControlMessageReader() { s.processIncomingFlag(s.nextMessage.Flag) s.nextMessage = nil } + go func() { // no deadline needed, Read will return once there's a new message, or an error occurred _ = s.dataChannel.SetReadDeadline(time.Time{}) for { var msg pb.Message - if err := s.readMessageFromDataChannel(&msg); err != nil { + if err := s.reader.ReadMsg(&msg); err != nil { return } + s.mx.Lock() s.processIncomingFlag(msg.Flag) + s.mx.Unlock() } }() } func (s *stream) SetWriteDeadline(t time.Time) error { - s.writeMu.Lock() - defer s.writeMu.Unlock() + s.mx.Lock() + defer s.mx.Unlock() s.writeDeadline = t select { case s.writeDeadlineUpdated <- struct{}{}: @@ -149,9 +152,6 @@ func (s *stream) availableSendSpace() int { const controlMsgSize = 100 // TODO: use actual message size func (s *stream) sendControlMessage(msg *pb.Message) error { - s.writeMu.Lock() - defer s.writeMu.Unlock() - available := s.availableSendSpace() if controlMsgSize < available { return s.writer.WriteMsg(msg) @@ -161,6 +161,9 @@ func (s *stream) sendControlMessage(msg *pb.Message) error { } func (s *stream) cancelWrite() error { + s.mx.Lock() + defer s.mx.Unlock() + if s.sendState != sendStateSending { return nil } @@ -177,6 +180,9 @@ func (s *stream) cancelWrite() error { } func (s *stream) CloseWrite() error { + s.mx.Lock() + defer s.mx.Unlock() + if s.sendState != sendStateSending { return nil } diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 1ceab46b58..59af634901 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -456,67 +456,6 @@ func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { require.ErrorIs(t, <-errC, os.ErrDeadlineExceeded) } -func TestTransportWebRTC_Read(t *testing.T) { - tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) - listener, err := tr.Listen(listenMultiaddr) - require.NoError(t, err) - - tr1, connectingPeer := getTransport(t) - - createListener := func() { - lconn, err := listener.Accept() - require.NoError(t, err) - require.Equal(t, connectingPeer, lconn.RemotePeer()) - stream, err := lconn.AcceptStream() - require.NoError(t, err) - _, _ = stream.Write(make([]byte, 2*1024)) - } - - t.Run("read partial message", func(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - createListener() - }() - - conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) - require.NoError(t, err) - stream, err := conn.OpenStream(context.Background()) - require.NoError(t, err) - - buf := make([]byte, 10) - stream.SetReadDeadline(time.Now().Add(10 * time.Second)) - n, err := stream.Read(buf) - require.NoError(t, err) - require.Equal(t, n, 10) - - wg.Wait() - }) - - t.Run("read zero bytes", func(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - createListener() - }() - - conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) - require.NoError(t, err) - stream, err := conn.OpenStream(context.Background()) - require.NoError(t, err) - - stream.SetReadDeadline(time.Now().Add(10 * time.Second)) - n, err := stream.Read([]byte{}) - require.NoError(t, err) - require.Equal(t, n, 0) - - wg.Wait() - }) -} - func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { tr, listeningPeer := getTransport(t) listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) From 72ca0f9dbd3577843e7ee9febacbd31c5c4efbe8 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 31 Jul 2023 11:34:05 -0400 Subject: [PATCH 45/86] tests: use errors.As for error assertion in TestStreamReadDeadline --- p2p/test/transport/transport_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 2ec7ba4005..73389aabd7 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -613,8 +613,8 @@ func TestStreamReadDeadline(t *testing.T) { _, err = s.Read([]byte{0}) require.Error(t, err) require.Contains(t, err.Error(), "deadline") - nerr, ok := err.(net.Error) - require.True(t, ok, "expected a net.Error") + var nerr net.Error + require.True(t, errors.As(err, &nerr), "expected a net.Error") require.True(t, nerr.Timeout(), "expected net.Error.Timeout() == true") // now test that the stream is still usable s.SetReadDeadline(time.Time{}) From 6e687cba0abe2f2bd773c8b5e352f459fd9cc444 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 31 Jul 2023 11:55:33 -0400 Subject: [PATCH 46/86] tests: close hosts in gating tests --- p2p/test/transport/gating_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/p2p/test/transport/gating_test.go b/p2p/test/transport/gating_test.go index 3079007bee..ba5d13cd07 100644 --- a/p2p/test/transport/gating_test.go +++ b/p2p/test/transport/gating_test.go @@ -88,6 +88,8 @@ func TestInterceptSecuredOutgoing(t *testing.T) { h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true, ConnGater: connGater}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{}) + defer h1.Close() + defer h2.Close() require.Len(t, h2.Addrs(), 1) require.Len(t, h2.Addrs(), 1) @@ -120,6 +122,8 @@ func TestInterceptUpgradedOutgoing(t *testing.T) { h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true, ConnGater: connGater}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{}) + defer h1.Close() + defer h2.Close() require.Len(t, h2.Addrs(), 1) require.Len(t, h2.Addrs(), 1) @@ -154,6 +158,8 @@ func TestInterceptAccept(t *testing.T) { h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{ConnGater: connGater}) + defer h1.Close() + defer h2.Close() require.Len(t, h2.Addrs(), 1) ctx, cancel := context.WithTimeout(context.Background(), time.Second) @@ -187,6 +193,8 @@ func TestInterceptSecuredIncoming(t *testing.T) { h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{ConnGater: connGater}) + defer h1.Close() + defer h2.Close() require.Len(t, h2.Addrs(), 1) ctx, cancel := context.WithTimeout(context.Background(), time.Second) @@ -218,6 +226,8 @@ func TestInterceptUpgradedIncoming(t *testing.T) { h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{ConnGater: connGater}) + defer h1.Close() + defer h2.Close() require.Len(t, h2.Addrs(), 1) ctx, cancel := context.WithTimeout(context.Background(), time.Second) From aaee47f50fd25ac38182a2fff572c93687f1811e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 19 Aug 2023 09:58:35 +0700 Subject: [PATCH 47/86] webrtc: disable TestManyStreams --- p2p/test/transport/transport_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 73389aabd7..7b049acd91 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -9,6 +9,7 @@ import ( "io" "net" "runtime" + "strings" "sync" "sync/atomic" "testing" @@ -313,6 +314,9 @@ func TestManyStreams(t *testing.T) { const streamCount = 128 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { + if strings.Contains(tc.Name, "WebRTC") { + t.Skip("Pion doesn't correctly handle large queues of streams.") + } h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoRcmgr: true}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true, NoRcmgr: true}) defer h1.Close() From f0335bb09a4117a211baa3781f6f580a40019416 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 19 Aug 2023 10:00:34 +0700 Subject: [PATCH 48/86] webrtc: fix staticcheck --- p2p/transport/webrtc/stream_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index 02beef4edc..474200ec12 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -149,6 +149,7 @@ func TestStreamPartialReads(t *testing.T) { require.Zero(t, n) b := make([]byte, 3) n, err = clientStr.Read(b) + require.Equal(t, 3, n) require.NoError(t, err) require.Equal(t, []byte("foo"), b) b, err = io.ReadAll(clientStr) From 43a60419a76ece143c620c760cdb9429431c82ba Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 19 Aug 2023 10:25:01 +0700 Subject: [PATCH 49/86] transport tests: fix deadline error assertion --- p2p/test/transport/deadline_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/test/transport/deadline_test.go b/p2p/test/transport/deadline_test.go index 5c8bcf18eb..55fa7a4fbc 100644 --- a/p2p/test/transport/deadline_test.go +++ b/p2p/test/transport/deadline_test.go @@ -43,7 +43,9 @@ func TestReadWriteDeadlines(t *testing.T) { buf := make([]byte, 1) _, err = s.Read(buf) require.Error(t, err) - require.True(t, err.(net.Error).Timeout()) + var nerr net.Error + require.ErrorAs(t, err, &nerr) + require.True(t, nerr.Timeout()) require.Less(t, time.Since(start), 1*time.Second) }) @@ -80,7 +82,9 @@ func TestReadWriteDeadlines(t *testing.T) { _, err = s.Write(sendBuf) } require.Error(t, err) - require.True(t, err.(net.Error).Timeout()) + var nerr net.Error + require.ErrorAs(t, err, &nerr) + require.True(t, nerr.Timeout()) require.Less(t, time.Since(start), 1*time.Second) }) } From cee6a48ce4c93a17cebc17068ef235282a3c6de7 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 20 Aug 2023 07:22:45 +0700 Subject: [PATCH 50/86] webrtc: fix flaky stream deadline test --- p2p/transport/webrtc/stream_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index 474200ec12..d25913ca3d 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -83,13 +83,13 @@ func getDetachedDataChannels(t *testing.T) (detachedChan, detachedChan) { // Now, create an offer offer, err := offerPC.CreateOffer(nil) require.NoError(t, err) - require.NoError(t, offerPC.SetLocalDescription(offer)) require.NoError(t, answerPC.SetRemoteDescription(offer)) + require.NoError(t, offerPC.SetLocalDescription(offer)) answer, err := answerPC.CreateAnswer(nil) require.NoError(t, err) - require.NoError(t, answerPC.SetLocalDescription(answer)) require.NoError(t, offerPC.SetRemoteDescription(answer)) + require.NoError(t, answerPC.SetLocalDescription(answer)) return <-answerChan, detachedChan{rwc: <-offerRWCChan, dc: offerDC} } From f70e53fb4128a640a5590f2a0e30fa842a391c2d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 20 Aug 2023 08:30:37 +0700 Subject: [PATCH 51/86] transport tests: fix panic when opening stream fails --- p2p/test/transport/transport_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 7b049acd91..59bccd42aa 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -476,6 +476,8 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { time.Sleep(50 * time.Millisecond) continue } + t.Logf("opening stream failed: %v", err) + return } err = func(s network.Stream) error { defer s.Close() From d02bc44656d4ec4964ab1877800fb5ff0e0dbefd Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 20 Aug 2023 08:48:59 +0700 Subject: [PATCH 52/86] webrtc: fix race condition when starting the UDP muxer --- p2p/transport/webrtc/listener.go | 4 ++- p2p/transport/webrtc/udpmux/mux.go | 33 ++++++++++--------- p2p/transport/webrtc/udpmux/mux_test.go | 2 +- .../webrtc/udpmux/muxed_connection.go | 4 +-- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 1caa5592c8..408698c985 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -111,7 +111,7 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack } l.ctx, l.cancel = context.WithCancel(context.Background()) - l.mux = udpmux.NewUDPMux(socket, func(ufrag string, addr net.Addr) bool { + mux := udpmux.NewUDPMux(socket, func(ufrag string, addr net.Addr) bool { select { case <-inFlightQueueCh: // we have space to accept, Yihaa @@ -149,6 +149,8 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack return true }) + l.mux = mux + mux.Start() return l, err } diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 08b105a704..cfbfe26638 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -18,7 +18,7 @@ var log = logging.Logger("webrtc-udpmux") const ReceiveMTU = 1500 -// udpMux multiplexes multiple ICE connections over a single net.PacketConn, +// UDPMux multiplexes multiple ICE connections over a single net.PacketConn, // generally a UDP socket. // // The connections are indexed by (ufrag, IP address family) @@ -33,7 +33,7 @@ const ReceiveMTU = 1500 // is a connection associated with the (ufrag, IP address family) pair. If found // we add the association to the address map. If not found, it is a previously // unseen IP address and the `unknownUfragCallback` callback is invoked. -type udpMux struct { +type UDPMux struct { socket net.PacketConn unknownUfragCallback func(string, net.Addr) bool @@ -45,11 +45,11 @@ type udpMux struct { cancel context.CancelFunc } -var _ ice.UDPMux = &udpMux{} +var _ ice.UDPMux = &UDPMux{} -func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) bool) *udpMux { +func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) bool) *UDPMux { ctx, cancel := context.WithCancel(context.Background()) - mux := &udpMux{ + mux := &UDPMux{ ctx: ctx, cancel: cancel, socket: socket, @@ -57,16 +57,19 @@ func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr storage: newUDPMuxStorage(), } + return mux +} + +func (mux *UDPMux) Start() { mux.wg.Add(1) go func() { defer mux.wg.Done() mux.readLoop() }() - return mux } // GetListenAddresses implements ice.UDPMux -func (mux *udpMux) GetListenAddresses() []net.Addr { +func (mux *UDPMux) GetListenAddresses() []net.Addr { return []net.Addr{mux.socket.LocalAddr()} } @@ -76,7 +79,7 @@ func (mux *udpMux) GetListenAddresses() []net.Addr { // as a remote is capable of being reachable through multiple different // UDP addresses of the same IP address family (eg. Server-reflexive addresses // and peer-reflexive addresses). -func (mux *udpMux) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) { +func (mux *UDPMux) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) { a, ok := addr.(*net.UDPAddr) if !ok && addr != nil { return nil, fmt.Errorf("unexpected address type: %T", addr) @@ -86,7 +89,7 @@ func (mux *udpMux) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) } // Close implements ice.UDPMux -func (mux *udpMux) Close() error { +func (mux *UDPMux) Close() error { select { case <-mux.ctx.Done(): return nil @@ -99,13 +102,13 @@ func (mux *udpMux) Close() error { } // RemoveConnByUfrag implements ice.UDPMux -func (mux *udpMux) RemoveConnByUfrag(ufrag string) { +func (mux *UDPMux) RemoveConnByUfrag(ufrag string) { if ufrag != "" { mux.storage.RemoveConnByUfrag(ufrag) } } -func (mux *udpMux) getOrCreateConn(ufrag string, isIPv6 bool, addr net.Addr) (net.PacketConn, error) { +func (mux *UDPMux) getOrCreateConn(ufrag string, isIPv6 bool, addr net.Addr) (net.PacketConn, error) { select { case <-mux.ctx.Done(): return nil, io.ErrClosedPipe @@ -116,11 +119,11 @@ func (mux *udpMux) getOrCreateConn(ufrag string, isIPv6 bool, addr net.Addr) (ne } // writeTo writes a packet to the underlying net.PacketConn -func (mux *udpMux) writeTo(buf []byte, addr net.Addr) (int, error) { +func (mux *UDPMux) writeTo(buf []byte, addr net.Addr) (int, error) { return mux.socket.WriteTo(buf, addr) } -func (mux *udpMux) readLoop() { +func (mux *UDPMux) readLoop() { for { select { case <-mux.ctx.Done(): @@ -144,7 +147,7 @@ func (mux *udpMux) readLoop() { } } -func (mux *udpMux) processPacket(buf []byte, addr net.Addr) (processed bool) { +func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { udpAddr, ok := addr.(*net.UDPAddr) if !ok { log.Errorf("received a non-UDP address: %s", addr) @@ -250,7 +253,7 @@ func (s *udpMuxStorage) removeConnByUfrag(ufrag string, closeConn bool) { } } -func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *udpMux, addr net.Addr) (created bool, _ *muxedConnection) { +func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *UDPMux, addr net.Addr) (created bool, _ *muxedConnection) { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} s.Lock() diff --git a/p2p/transport/webrtc/udpmux/mux_test.go b/p2p/transport/webrtc/udpmux/mux_test.go index f6dc0e2aa3..498be61dcc 100644 --- a/p2p/transport/webrtc/udpmux/mux_test.go +++ b/p2p/transport/webrtc/udpmux/mux_test.go @@ -47,7 +47,7 @@ func (dummyPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) { return 0, nil } -func hasConn(m *udpMux, ufrag string, isIPv6 bool) bool { +func hasConn(m *UDPMux, ufrag string, isIPv6 bool) bool { m.storage.Lock() _, ok := m.storage.ufragMap[ufragConnKey{ufrag: ufrag, isIPv6: isIPv6}] m.storage.Unlock() diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index dbfc5d6b75..d51e846b2c 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -18,12 +18,12 @@ type muxedConnection struct { onClose func() pq *packetQueue addr net.Addr - mux *udpMux + mux *UDPMux } var _ net.PacketConn = (*muxedConnection)(nil) -func newMuxedConnection(mux *udpMux, onClose func(), addr net.Addr) *muxedConnection { +func newMuxedConnection(mux *UDPMux, onClose func(), addr net.Addr) *muxedConnection { ctx, cancel := context.WithCancel(mux.ctx) return &muxedConnection{ ctx: ctx, From da4a4f5e662a24d844a834002b4c624a8bcceb95 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 20 Aug 2023 09:16:21 +0700 Subject: [PATCH 53/86] transport tests: skip TestMoreStreamsThanOurLimits on WebRTC --- p2p/test/transport/transport_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 59bccd42aa..067bc3a935 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -382,6 +382,9 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { const streamCount = 1024 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { + if strings.Contains(tc.Name, "WebRTC") { + t.Skip("This test potentially exhausts the uint16 WebRTC stream ID space.") + } listenerLimits := rcmgr.PartialLimitConfig{ PeerDefault: rcmgr.ResourceLimits{ Streams: 32, From 2e5f5e9b482dc8a9b87965819264bd218df84b9a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 20 Aug 2023 09:39:33 +0700 Subject: [PATCH 54/86] metrics: enable metrics for /webrtc-direct multiaddrs --- p2p/metricshelper/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/metricshelper/conn.go b/p2p/metricshelper/conn.go index ef367ac9b1..b07016ce84 100644 --- a/p2p/metricshelper/conn.go +++ b/p2p/metricshelper/conn.go @@ -2,7 +2,7 @@ package metricshelper import ma "github.com/multiformats/go-multiaddr" -var transports = [...]int{ma.P_CIRCUIT, ma.P_WEBRTC, ma.P_WEBTRANSPORT, ma.P_QUIC, ma.P_QUIC_V1, ma.P_WSS, ma.P_WS, ma.P_TCP} +var transports = [...]int{ma.P_CIRCUIT, ma.P_WEBRTC, ma.P_WEBRTC_DIRECT, ma.P_WEBTRANSPORT, ma.P_QUIC, ma.P_QUIC_V1, ma.P_WSS, ma.P_WS, ma.P_TCP} func GetTransport(a ma.Multiaddr) string { for _, t := range transports { From b19f11f3bab69d7f131108292ae07c8e5f422f76 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 20 Aug 2023 10:22:38 +0700 Subject: [PATCH 55/86] webrtc: fix error return value of stream.Read when stream is reset --- p2p/transport/webrtc/stream_read.go | 3 +++ p2p/transport/webrtc/stream_test.go | 2 +- p2p/transport/webrtc/transport_test.go | 32 -------------------------- 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 43fdcee088..e064c8558b 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -44,6 +44,9 @@ func (s *stream) Read(b []byte) (int, error) { // There's little we can do here return 0, errors.New("didn't receive final state for stream") } + if s.receiveState == receiveStateReset { + return 0, network.ErrReset + } return 0, err } s.mx.Lock() diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index d25913ca3d..b0af1d769c 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -196,12 +196,12 @@ func TestStreamReadReturnsOnClose(t *testing.T) { client, _ := getDetachedDataChannels(t) clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) - // serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) errChan := make(chan error, 1) go func() { _, err := clientStr.Read([]byte{0}) errChan <- err }() + time.Sleep(50 * time.Millisecond) // give the Read call some time to hit the loop require.NoError(t, clientStr.Close()) select { case err := <-errChan: diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 59af634901..75a47b7faf 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -562,38 +562,6 @@ func TestTransportWebRTC_Close(t *testing.T) { tr1, connectingPeer := getTransport(t) - t.Run("StreamCanCloseWhenReadActive", func(t *testing.T) { - done := make(chan struct{}) - - go func() { - lconn, err := listener.Accept() - require.NoError(t, err) - t.Logf("listener accepted connection") - require.Equal(t, connectingPeer, lconn.RemotePeer()) - done <- struct{}{} - }() - - conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) - require.NoError(t, err) - t.Logf("dialer opened connection") - stream, err := conn.OpenStream(context.Background()) - require.NoError(t, err) - - time.AfterFunc(100*time.Millisecond, func() { - err := stream.Close() - require.NoError(t, err) - }) - - _, err = stream.Read(make([]byte, 19)) - require.ErrorIs(t, err, os.ErrDeadlineExceeded) - - select { - case <-done: - case <-time.After(10 * time.Second): - t.Fatal("timed out") - } - }) - t.Run("RemoteClosesStream", func(t *testing.T) { var wg sync.WaitGroup wg.Add(1) From 5647b27cc3e7904cae6f1a62696fd24bb490af28 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 10:23:20 +0700 Subject: [PATCH 56/86] webrtc: remove unused function parameter in udpMuxStorage.RemoveConnByUfrag --- p2p/transport/webrtc/udpmux/mux.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index cfbfe26638..1ad53817f9 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -237,10 +237,6 @@ func newUDPMuxStorage() *udpMuxStorage { } func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { - s.removeConnByUfrag(ufrag, true) -} - -func (s *udpMuxStorage) removeConnByUfrag(ufrag string, closeConn bool) { s.Lock() defer s.Unlock() @@ -263,7 +259,7 @@ func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *UDPMux, return false, conn } - conn := newMuxedConnection(mux, func() { s.removeConnByUfrag(ufrag, false) }, addr) + conn := newMuxedConnection(mux, func() { s.RemoveConnByUfrag(ufrag) }, addr) s.ufragMap[key] = conn s.addrMap[addr.String()] = conn From 6a221746e3f4500023510785ab17354983bdabaa Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 14:47:19 +0700 Subject: [PATCH 57/86] webrtc: use the same log level for the pion logger --- go.mod | 2 +- p2p/transport/webrtc/listener.go | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index e42b65f804..905b161277 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/fx v1.20.0 go.uber.org/goleak v1.2.0 + go.uber.org/zap v1.25.0 golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 @@ -123,7 +124,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.25.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/text v0.12.0 // indirect diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 408698c985..0a39b0fca8 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -13,9 +13,8 @@ import ( "time" "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/p2p/transport/webrtc/udpmux" - tpt "github.com/libp2p/go-libp2p/core/transport" + "github.com/libp2p/go-libp2p/p2p/transport/webrtc/udpmux" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multibase" @@ -24,6 +23,7 @@ import ( "github.com/pion/ice/v2" pionlogger "github.com/pion/logging" "github.com/pion/webrtc/v3" + "go.uber.org/zap/zapcore" ) type connMultiaddrs struct { @@ -202,9 +202,20 @@ func (l *listener) setupConnection( settingEngine := webrtc.SettingEngine{} - // suppress pion logs loggerFactory := pionlogger.NewDefaultLoggerFactory() - loggerFactory.DefaultLogLevel = pionlogger.LogLevelWarn + pionLogLevel := pionlogger.LogLevelDisabled + switch log.Level() { + case zapcore.DebugLevel: + pionLogLevel = pionlogger.LogLevelDebug + case zapcore.InfoLevel: + pionLogLevel = pionlogger.LogLevelInfo + case zapcore.WarnLevel: + pionLogLevel = pionlogger.LogLevelWarn + case zapcore.ErrorLevel: + pionLogLevel = pionlogger.LogLevelError + } + fmt.Println("log level", pionLogLevel) + loggerFactory.DefaultLogLevel = pionLogLevel settingEngine.LoggerFactory = loggerFactory settingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleServer) From 305ef47dc613b89723e879bf727530addd880859 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 14:50:09 +0700 Subject: [PATCH 58/86] webrtc: shorten package comment, move it to transport.go --- p2p/transport/webrtc/transport.go | 5 +++++ p2p/transport/webrtc/webrtc.go | 15 --------------- 2 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 p2p/transport/webrtc/webrtc.go diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 44787c9093..0d8207088c 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -1,3 +1,8 @@ +// Package libp2pwebrtc implements the WebRTC transport for go-libp2p, +// as officially described in https://github.com/libp2p/specs/tree/master/webrtc. +// +// The udpmux subpackage contains the logic for multiplexing multiple WebRTC (ICE) +// connections over a single UDP socket. package libp2pwebrtc import ( diff --git a/p2p/transport/webrtc/webrtc.go b/p2p/transport/webrtc/webrtc.go deleted file mode 100644 index 35bfcbc7d0..0000000000 --- a/p2p/transport/webrtc/webrtc.go +++ /dev/null @@ -1,15 +0,0 @@ -// libp2pwebrtc implements the WebRTC transport for go-libp2p, -// as officially described in https://github.com/libp2p/specs/tree/cfcf0230b2f5f11ed6dd060f97305faa973abed2/webrtc. -// -// Benchmarks on how this transport compares to other transports can be found in -// https://github.com/libp2p/libp2p-go-webrtc-benchmarks. -// -// Entrypoint for the logic of this Transport can be found in `transport.go`, where the WebRTC transport is implemented, -// used both by the client for Dialing as well as the server for Listening. Starting from there you should be able to follow -// the logic from start to finish. -// -// In the udpmux subpackage you can find the logic for multiplexing multiple WebRTC (ICE) connections over a single UDP socket. -// -// The pb subpackage contains the protobuf definitions for the signaling protocol used by this transport, -// which is taken verbatim from the "Multiplexing" chapter of the WebRTC spec. -package libp2pwebrtc From 665ed52c510c2d07aca256805c2cf06b613af1a4 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 15:01:59 +0700 Subject: [PATCH 59/86] webrtc: use a uniform random distribution for ufrag generation --- p2p/transport/webrtc/transport.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 0d8207088c..5abafa020b 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -12,11 +12,14 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/x509" + "encoding/binary" "errors" "fmt" "net" "time" + mrand "golang.org/x/exp/rand" + "github.com/libp2p/go-libp2p/core/connmgr" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" @@ -419,11 +422,12 @@ func genUfrag() string { uFragLength = uFragIdOffset + uFragIdLength ) + seed := [8]byte{} + rand.Read(seed[:]) + r := mrand.New(mrand.NewSource(binary.BigEndian.Uint64(seed[:]))) b := make([]byte, uFragLength) - copy(b[:], uFragPrefix[:]) - rand.Read(b[uFragIdOffset:]) for i := uFragIdOffset; i < uFragLength; i++ { - b[i] = uFragAlphabet[int(b[i])%len(uFragAlphabet)] + b[i] = uFragAlphabet[r.Intn(len(uFragAlphabet))] } return string(b) } From d53c36d1406fb79f7b9817c79ff00dd0fe87bef5 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 15:11:10 +0700 Subject: [PATCH 60/86] webrtc: improve documentation of UDPMux --- p2p/transport/webrtc/udpmux/mux.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 1ad53817f9..b0de70ac57 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -21,18 +21,19 @@ const ReceiveMTU = 1500 // UDPMux multiplexes multiple ICE connections over a single net.PacketConn, // generally a UDP socket. // -// The connections are indexed by (ufrag, IP address family) -// and by remote address from which the connection has received valid STUN/RTC -// packets. +// The connections are indexed by (ufrag, IP address family) and by remote +// address from which the connection has received valid STUN/RTC packets. // // When a new packet is received on the underlying net.PacketConn, we // first check the address map to see if there is a connection associated with the -// remote address. If found we forward the packet to the connection. If an associated -// connection is not found, we check to see if the packet is a STUN packet. We then -// fetch the ufrag of the remote from the STUN packet and use it to check if there -// is a connection associated with the (ufrag, IP address family) pair. If found -// we add the association to the address map. If not found, it is a previously -// unseen IP address and the `unknownUfragCallback` callback is invoked. +// remote address: +// If found, we pass the packet to that connection. +// Otherwise, we check to see if the packet is a STUN packet. +// If it is, we read the ufrag from the STUN packet and use it to check if there +// is a connection associated with the (ufrag, IP address family) pair. +// If found we add the association to the address map. +// Otherwise, this is a previously unseen IP address and the unknownUfragCallback +// callback is called. type UDPMux struct { socket net.PacketConn unknownUfragCallback func(string, net.Addr) bool From e1bd40d9d66ad5d0ad4795f923defd4d261656db Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 15:16:19 +0700 Subject: [PATCH 61/86] webrtc: use an error return value on the unknownUfragCallback --- p2p/transport/webrtc/listener.go | 7 +++---- p2p/transport/webrtc/udpmux/mux.go | 13 ++++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 0a39b0fca8..fb4abc0cc3 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -111,13 +111,12 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack } l.ctx, l.cancel = context.WithCancel(context.Background()) - mux := udpmux.NewUDPMux(socket, func(ufrag string, addr net.Addr) bool { + mux := udpmux.NewUDPMux(socket, func(ufrag string, addr net.Addr) error { select { case <-inFlightQueueCh: // we have space to accept, Yihaa default: - log.Debug("candidate chan full, dropping incoming candidate") - return false + return errors.New("candidate chan full, dropping incoming candidate") } go func() { @@ -147,7 +146,7 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack } }() - return true + return nil }) l.mux = mux mux.Start() diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index b0de70ac57..96e1fc0155 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -36,7 +36,7 @@ const ReceiveMTU = 1500 // callback is called. type UDPMux struct { socket net.PacketConn - unknownUfragCallback func(string, net.Addr) bool + unknownUfragCallback func(string, net.Addr) error storage *udpMuxStorage @@ -48,7 +48,7 @@ type UDPMux struct { var _ ice.UDPMux = &UDPMux{} -func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) bool) *UDPMux { +func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) error) *UDPMux { ctx, cancel := context.WithCancel(context.Background()) mux := &UDPMux{ ctx: ctx, @@ -185,9 +185,12 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { } connCreated, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) - if connCreated && !mux.unknownUfragCallback(ufrag, udpAddr) { - conn.Close() - return false + if connCreated { + if err := mux.unknownUfragCallback(ufrag, udpAddr); err != nil { + log.Debugf("creating connection failed: %w", err) + conn.Close() + return false + } } if err := conn.Push(buf); err != nil { From edf501cb7a1584bfdcd3162382f233a0884aacdd Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 15:26:21 +0700 Subject: [PATCH 62/86] webrtc: improve error logging in UDPMux packet handling --- p2p/transport/webrtc/udpmux/mux.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 96e1fc0155..a1ff512fdb 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -161,7 +161,7 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { // with it. If yes, we push the received packet to the connection if conn, ok := mux.storage.GetConnByAddr(udpAddr); ok { if err := conn.Push(buf); err != nil { - log.Errorf("could not push packet: %v", err) + log.Debugf("could not push packet: %v", err) return false } return true @@ -173,28 +173,32 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { } msg := &stun.Message{Raw: buf} - if err := msg.Decode(); err != nil || msg.Type != stun.BindingRequest { - log.Debug("incoming message should be a STUN binding request") + if err := msg.Decode(); err != nil { + log.Debugf("failed to decode STUN message: %s", err) + return false + } + if msg.Type != stun.BindingRequest { + log.Debugf("incoming message should be a STUN binding request, got %s", msg.Type) return false } ufrag, err := ufragFromSTUNMessage(msg) if err != nil { - log.Debug("could not find STUN username: %w", err) + log.Debugf("could not find STUN username: %s", err) return false } connCreated, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) if connCreated { if err := mux.unknownUfragCallback(ufrag, udpAddr); err != nil { - log.Debugf("creating connection failed: %w", err) + log.Debugf("creating connection failed: %s", err) conn.Close() return false } } if err := conn.Push(buf); err != nil { - log.Errorf("could not push packet: %v", err) + log.Debugf("could not push packet: %v", err) return false } return true From 340b882e6fd8722800d4404160d98e5d29d64e46 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 15:36:19 +0700 Subject: [PATCH 63/86] webrtc: remove skipped TestWebrtcTransport test --- p2p/transport/webrtc/transport_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 75a47b7faf..4bf649c8ed 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -21,7 +21,6 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - ttransport "github.com/libp2p/go-libp2p/p2p/transport/testsuite" ma "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multibase" @@ -712,18 +711,3 @@ func TestMaxInFlightRequests(t *testing.T) { require.Equal(t, count, int(success.Load()), "expected exactly 3 dial successes") require.Equal(t, 1, int(fails.Load()), "expected exactly 1 dial failure") } - -// TestWebrtcTransport implements the standard go-libp2p transport test. -// It's a test that however not works for many transports, and neither does it for WebRTC. -// -// Reason it doens't work for WebRTC is that it opens too many streams too rapidly, which -// in a regular environment would be seen as an attack that we wish to block/stop. -// -// Leaving it here for now only for documentation purposes, -// and to make it explicitly clear this test doesn't work for WebRTC. -func TestWebrtcTransport(t *testing.T) { - t.Skip("This test does not work for WebRTC due to the way it is setup, see comments for more explanation") - ta, _ := getTransport(t) - tb, _ := getTransport(t) - ttransport.SubtestTransport(t, ta, tb, fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP), "peerA") -} From de0bbc9ec7fed8362f9c9c27c598228226fa0e63 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 23 Aug 2023 18:25:02 +0800 Subject: [PATCH 64/86] webrtc: fix transport name in ConnectionState --- p2p/transport/webrtc/connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 2bd454c075..c0fdf1d01b 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -131,7 +131,7 @@ func newConnection( // ConnState implements transport.CapableConn func (c *connection) ConnState() network.ConnectionState { - return network.ConnectionState{Transport: "p2p-webrtc-direct"} + return network.ConnectionState{Transport: "webrtc-direct"} } // Close closes the underlying peerconnection. From c262c6426f4f7507847a44f1b02a51e0a032c7f2 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 24 Aug 2023 11:36:12 +0800 Subject: [PATCH 65/86] webrtc: remove unused local and remote address from stream --- p2p/transport/webrtc/connection.go | 16 ++-------------- p2p/transport/webrtc/listener.go | 16 +++++++++++----- p2p/transport/webrtc/stream.go | 11 ----------- p2p/transport/webrtc/stream_test.go | 26 +++++++++++++------------- p2p/transport/webrtc/transport.go | 28 ++++++++++++++++------------ 5 files changed, 42 insertions(+), 55 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index c0fdf1d01b..53b0dff721 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -173,13 +173,7 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if err != nil { return nil, fmt.Errorf("open stream: %w", err) } - str := newStream( - dc, - rwc, - nil, - nil, - func() { c.removeStream(streamID) }, - ) + str := newStream(dc, rwc, func() { c.removeStream(streamID) }) if err := c.addStream(str); err != nil { str.Close() return nil, err @@ -192,13 +186,7 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { case <-c.ctx.Done(): return nil, c.closeErr case dc := <-c.acceptQueue: - str := newStream( - dc.channel, - dc.stream, - nil, - nil, - func() { c.removeStream(*dc.channel.ID()) }, - ) + str := newStream(dc.channel, dc.stream, func() { c.removeStream(*dc.channel.ID()) }) if err := c.addStream(str); err != nil { str.Close() return nil, err diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index fb4abc0cc3..b2e1be356b 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -12,6 +12,8 @@ import ( "sync" "time" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/network" tpt "github.com/libp2p/go-libp2p/core/transport" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/udpmux" @@ -279,7 +281,7 @@ func (l *listener) setupConnection( localMultiaddrWithoutCerthash, _ := ma.SplitFunc(l.localMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) - handshakeChannel := newStream(rawDatachannel, rwc, l.localAddr, addr.raddr, func() {}) + handshakeChannel := newStream(rawDatachannel, rwc, func() {}) // The connection is instantiated before performing the Noise handshake. This is // to handle the case where the remote is faster and attempts to initiate a stream // before the ondatachannel callback can be set. @@ -299,18 +301,22 @@ func (l *listener) setupConnection( } // we do not yet know A's peer ID so accept any inbound - secureConn, err := l.transport.noiseHandshake(ctx, pc, handshakeChannel, "", crypto.SHA256, true) + remotePubKey, err := l.transport.noiseHandshake(ctx, pc, handshakeChannel, "", crypto.SHA256, true) + if err != nil { + return nil, err + } + remotePeer, err := peer.IDFromPublicKey(remotePubKey) if err != nil { return nil, err } // earliest point where we know the remote's peerID - if err := scope.SetPeer(secureConn.RemotePeer()); err != nil { + if err := scope.SetPeer(remotePeer); err != nil { return nil, err } - conn.setRemotePeer(secureConn.RemotePeer()) - conn.setRemotePublicKey(secureConn.RemotePublicKey()) + conn.setRemotePeer(remotePeer) + conn.setRemotePublicKey(remotePubKey) return conn, err } diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index ef33c20c5e..b86128dc31 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -1,7 +1,6 @@ package libp2pwebrtc import ( - "net" "sync" "time" @@ -86,9 +85,6 @@ type stream struct { id uint16 // for logging purposes dataChannel *datachannel.DataChannel closeErr error - - laddr net.Addr - raddr net.Addr } var _ network.MuxedStream = &stream{} @@ -96,7 +92,6 @@ var _ network.MuxedStream = &stream{} func newStream( channel *webrtc.DataChannel, rwc datachannel.ReadWriteCloser, - laddr, raddr net.Addr, onDone func(), ) *stream { s := &stream{ @@ -110,9 +105,6 @@ func newStream( id: *channel.ID(), dataChannel: rwc.(*datachannel.DataChannel), onDone: onDone, - - laddr: laddr, - raddr: raddr, } channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) @@ -159,9 +151,6 @@ func (s *stream) Reset() error { return closeReadErr } -func (s *stream) LocalAddr() net.Addr { return s.laddr } -func (s *stream) RemoteAddr() net.Addr { return s.raddr } - func (s *stream) SetDeadline(t time.Time) error { _ = s.SetReadDeadline(t) return s.SetWriteDeadline(t) diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index b0af1d769c..f1442b9bfd 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -98,8 +98,8 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { client, server := getDetachedDataChannels(t) var clientDone, serverDone bool - clientStr := newStream(client.dc, client.rwc, nil, nil, func() { clientDone = true }) - serverStr := newStream(server.dc, server.rwc, nil, nil, func() { serverDone = true }) + clientStr := newStream(client.dc, client.rwc, func() { clientDone = true }) + serverStr := newStream(server.dc, server.rwc, func() { serverDone = true }) // send a foobar from the client n, err := clientStr.Write([]byte("foobar")) @@ -137,8 +137,8 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { func TestStreamPartialReads(t *testing.T) { client, server := getDetachedDataChannels(t) - clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) - serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + clientStr := newStream(client.dc, client.rwc, func() {}) + serverStr := newStream(server.dc, server.rwc, func() {}) _, err := serverStr.Write([]byte("foobar")) require.NoError(t, err) @@ -160,8 +160,8 @@ func TestStreamPartialReads(t *testing.T) { func TestStreamSkipEmptyFrames(t *testing.T) { client, server := getDetachedDataChannels(t) - clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) - serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + clientStr := newStream(client.dc, client.rwc, func() {}) + serverStr := newStream(server.dc, server.rwc, func() {}) for i := 0; i < 10; i++ { require.NoError(t, serverStr.writer.WriteMsg(&pb.Message{})) @@ -195,7 +195,7 @@ func TestStreamSkipEmptyFrames(t *testing.T) { func TestStreamReadReturnsOnClose(t *testing.T) { client, _ := getDetachedDataChannels(t) - clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) + clientStr := newStream(client.dc, client.rwc, func() {}) errChan := make(chan error, 1) go func() { _, err := clientStr.Read([]byte{0}) @@ -215,8 +215,8 @@ func TestStreamResets(t *testing.T) { client, server := getDetachedDataChannels(t) var clientDone, serverDone bool - clientStr := newStream(client.dc, client.rwc, nil, nil, func() { clientDone = true }) - serverStr := newStream(server.dc, server.rwc, nil, nil, func() { serverDone = true }) + clientStr := newStream(client.dc, client.rwc, func() { clientDone = true }) + serverStr := newStream(server.dc, server.rwc, func() { serverDone = true }) // send a foobar from the client _, err := clientStr.Write([]byte("foobar")) @@ -248,8 +248,8 @@ func TestStreamResets(t *testing.T) { func TestStreamReadDeadlineAsync(t *testing.T) { client, server := getDetachedDataChannels(t) - clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) - serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + clientStr := newStream(client.dc, client.rwc, func() {}) + serverStr := newStream(server.dc, server.rwc, func() {}) timeout := 100 * time.Millisecond if os.Getenv("CI") != "" { @@ -279,8 +279,8 @@ func TestStreamReadDeadlineAsync(t *testing.T) { func TestStreamWriteDeadlineAsync(t *testing.T) { client, server := getDetachedDataChannels(t) - clientStr := newStream(client.dc, client.rwc, nil, nil, func() {}) - serverStr := newStream(server.dc, server.rwc, nil, nil, func() {}) + clientStr := newStream(client.dc, client.rwc, func() {}) + serverStr := newStream(server.dc, server.rwc, func() {}) _ = serverStr b := make([]byte, 1024) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 5abafa020b..f8118277cf 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -371,13 +371,12 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement if err != nil { return nil, fmt.Errorf("ice connection did not have selected candidate pair: error: %w", err) } - laddr := &net.UDPAddr{IP: net.ParseIP(cp.Local.Address), Port: int(cp.Local.Port)} - channel := newStream(rawHandshakeChannel, detached, laddr, raddr, func() {}) + channel := newStream(rawHandshakeChannel, detached, func() {}) // the local address of the selected candidate pair should be the // local address for the connection, since different datachannels // are multiplexed over the same SCTP connection - localAddr, err := manet.FromNetAddr(channel.LocalAddr()) + localAddr, err := manet.FromNetAddr(&net.UDPAddr{IP: net.ParseIP(cp.Local.Address), Port: int(cp.Local.Port)}) if err != nil { return nil, err } @@ -401,15 +400,15 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, err } - secConn, err := t.noiseHandshake(ctx, pc, channel, p, remoteHashFunction, false) + remotePubKey, err := t.noiseHandshake(ctx, pc, channel, p, remoteHashFunction, false) if err != nil { return conn, err } if t.gater != nil && !t.gater.InterceptSecured(network.DirOutbound, p, conn) { - secConn.Close() + conn.Close() return nil, fmt.Errorf("secured connection gated") } - conn.setRemotePublicKey(secConn.RemotePublicKey()) + conn.setRemotePublicKey(remotePubKey) return conn, nil } @@ -486,7 +485,7 @@ func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash return result, nil } -func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, datachannel *stream, peer peer.ID, hash crypto.Hash, inbound bool) (sec.SecureConn, error) { +func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, datachannel *stream, peer peer.ID, hash crypto.Hash, inbound bool) (ic.PubKey, error) { prologue, err := t.generateNoisePrologue(pc, hash, inbound) if err != nil { return nil, fmt.Errorf("generate prologue: %w", err) @@ -500,15 +499,20 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerCon } var secureConn sec.SecureConn if inbound { - secureConn, err = sessionTransport.SecureOutbound(ctx, datachannel, peer) + secureConn, err = sessionTransport.SecureOutbound(ctx, fakeStreamConn{datachannel}, peer) if err != nil { - return secureConn, fmt.Errorf("failed to secure inbound connection: %w", err) + return nil, fmt.Errorf("failed to secure inbound connection: %w", err) } } else { - secureConn, err = sessionTransport.SecureInbound(ctx, datachannel, peer) + secureConn, err = sessionTransport.SecureInbound(ctx, fakeStreamConn{datachannel}, peer) if err != nil { - return secureConn, fmt.Errorf("failed to secure outbound connection: %w", err) + return nil, fmt.Errorf("failed to secure outbound connection: %w", err) } } - return secureConn, nil + return secureConn.RemotePublicKey(), nil } + +type fakeStreamConn struct{ *stream } + +func (fakeStreamConn) LocalAddr() net.Addr { return nil } +func (fakeStreamConn) RemoteAddr() net.Addr { return nil } From 1b47936ac6eab765cb8de4037c41afffaf61d1f4 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 24 Aug 2023 11:48:38 +0800 Subject: [PATCH 66/86] webrtc: remove nonsensical comments in the muxed connection --- .../webrtc/udpmux/muxed_connection.go | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index d51e846b2c..c3e60cd12b 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -39,7 +39,15 @@ func (c *muxedConnection) Push(buf []byte) error { return c.pq.Push(buf) } -// Close implements net.PacketConn +func (c *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { + n, err := c.pq.Pop(c.ctx, p) + return n, c.addr, err +} + +func (c *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { + return c.mux.writeTo(p, addr) +} + func (c *muxedConnection) Close() error { select { case <-c.ctx.Done(): @@ -51,40 +59,20 @@ func (c *muxedConnection) Close() error { return nil } -// LocalAddr implements net.PacketConn -func (c *muxedConnection) LocalAddr() net.Addr { - return c.mux.socket.LocalAddr() -} +func (c *muxedConnection) LocalAddr() net.Addr { return c.mux.socket.LocalAddr() } +func (c *muxedConnection) Address() net.Addr { return c.addr } -func (c *muxedConnection) Address() net.Addr { - return c.addr -} - -// ReadFrom implements net.PacketConn -func (c *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { - n, err := c.pq.Pop(c.ctx, p) - return n, c.addr, err -} - -// SetDeadline implements net.PacketConn func (*muxedConnection) SetDeadline(t time.Time) error { // no deadline is desired here return nil } -// SetReadDeadline implements net.PacketConn func (*muxedConnection) SetReadDeadline(t time.Time) error { // no read deadline is desired here return nil } -// SetWriteDeadline implements net.PacketConn func (*muxedConnection) SetWriteDeadline(t time.Time) error { // no write deadline is desired here return nil } - -// WriteTo implements net.PacketConn -func (c *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { - return c.mux.writeTo(p, addr) -} From 71d381859073e1be31b1be7aee91d61e72972e0a Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 24 Aug 2023 12:04:38 +0800 Subject: [PATCH 67/86] webrtc: fix ambiguous naming of address function in muxed connection --- p2p/transport/webrtc/udpmux/mux.go | 2 +- p2p/transport/webrtc/udpmux/muxed_connection.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index a1ff512fdb..9e5a8af5ae 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -252,7 +252,7 @@ func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} if conn, ok := s.ufragMap[key]; ok { delete(s.ufragMap, key) - delete(s.addrMap, conn.Address().String()) + delete(s.addrMap, conn.RemoteAddr().String()) } } } diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index c3e60cd12b..7c3eee26aa 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -17,20 +17,20 @@ type muxedConnection struct { cancel context.CancelFunc onClose func() pq *packetQueue - addr net.Addr + remote net.Addr mux *UDPMux } -var _ net.PacketConn = (*muxedConnection)(nil) +var _ net.PacketConn = &muxedConnection{} -func newMuxedConnection(mux *UDPMux, onClose func(), addr net.Addr) *muxedConnection { +func newMuxedConnection(mux *UDPMux, onClose func(), remote net.Addr) *muxedConnection { ctx, cancel := context.WithCancel(mux.ctx) return &muxedConnection{ ctx: ctx, cancel: cancel, pq: newPacketQueue(), onClose: onClose, - addr: addr, + remote: remote, mux: mux, } } @@ -41,7 +41,7 @@ func (c *muxedConnection) Push(buf []byte) error { func (c *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { n, err := c.pq.Pop(c.ctx, p) - return n, c.addr, err + return n, c.remote, err } func (c *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { @@ -59,8 +59,8 @@ func (c *muxedConnection) Close() error { return nil } -func (c *muxedConnection) LocalAddr() net.Addr { return c.mux.socket.LocalAddr() } -func (c *muxedConnection) Address() net.Addr { return c.addr } +func (c *muxedConnection) LocalAddr() net.Addr { return c.mux.socket.LocalAddr() } +func (c *muxedConnection) RemoteAddr() net.Addr { return c.remote } func (*muxedConnection) SetDeadline(t time.Time) error { // no deadline is desired here From 351cadc174fb15e43420892f2d0a2a8930a1db99 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 24 Aug 2023 12:18:48 +0800 Subject: [PATCH 68/86] webrtc: remove stray fmt.Println in listener --- p2p/transport/webrtc/listener.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index b2e1be356b..14459b9d9c 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -201,8 +201,6 @@ func (l *listener) setupConnection( } }() - settingEngine := webrtc.SettingEngine{} - loggerFactory := pionlogger.NewDefaultLoggerFactory() pionLogLevel := pionlogger.LogLevelDisabled switch log.Level() { @@ -215,10 +213,9 @@ func (l *listener) setupConnection( case zapcore.ErrorLevel: pionLogLevel = pionlogger.LogLevelError } - fmt.Println("log level", pionLogLevel) loggerFactory.DefaultLogLevel = pionLogLevel - settingEngine.LoggerFactory = loggerFactory + settingEngine := webrtc.SettingEngine{LoggerFactory: loggerFactory} settingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleServer) settingEngine.SetICECredentials(addr.ufrag, addr.ufrag) settingEngine.SetLite(true) @@ -233,7 +230,6 @@ func (l *listener) setupConnection( settingEngine.DetachDataChannels() api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) - pc, err = api.NewPeerConnection(l.config) if err != nil { return nil, err From 256118c2dd43a5005a08bb7d4b75a19d0eb2f10b Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 24 Aug 2023 12:19:39 +0800 Subject: [PATCH 69/86] webrtc: only handle net.UDPAddrs in the UDPMux --- p2p/transport/webrtc/udpmux/mux.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 9e5a8af5ae..c1ad8d1fee 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -75,14 +75,12 @@ func (mux *UDPMux) GetListenAddresses() []net.Addr { } // GetConn implements ice.UDPMux -// It creates a net.PacketConn for a given ufrag if an existing -// one cannot be found. We differentiate IPv4 and IPv6 addresses -// as a remote is capable of being reachable through multiple different -// UDP addresses of the same IP address family (eg. Server-reflexive addresses -// and peer-reflexive addresses). +// It creates a net.PacketConn for a given ufrag if an existing one cannot be found. +// We differentiate IPv4 and IPv6 addresses, since a remote is can be reachable at multiple different +// UDP addresses of the same IP address family (eg. server-reflexive addresses and peer-reflexive addresses). func (mux *UDPMux) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) { a, ok := addr.(*net.UDPAddr) - if !ok && addr != nil { + if !ok { return nil, fmt.Errorf("unexpected address type: %T", addr) } isIPv6 := ok && a.IP.To4() == nil From cc05d439942a58119298e52fdade630251103348 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 24 Aug 2023 13:18:39 +0700 Subject: [PATCH 70/86] webrtc: rename awaitPeerConnectionOpen to something sensible, add docs --- p2p/transport/webrtc/listener.go | 18 +++++++++++------- p2p/transport/webrtc/transport.go | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 14459b9d9c..700510b41c 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -12,16 +12,15 @@ import ( "sync" "time" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" tpt "github.com/libp2p/go-libp2p/core/transport" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/udpmux" + ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" - "github.com/pion/ice/v2" pionlogger "github.com/pion/logging" "github.com/pion/webrtc/v3" @@ -244,7 +243,7 @@ func (l *listener) setupConnection( return nil, err } - errC := awaitPeerConnectionOpen(addr.ufrag, pc) + errC := addOnConnectionStateChangeCallback(pc) // Infer the client SDP from the incoming STUN message by setting the ice-ufrag. if err := pc.SetRemoteDescription(webrtc.SessionDescription{ SDP: createClientSDP(addr.raddr, addr.ufrag), @@ -265,7 +264,7 @@ func (l *listener) setupConnection( return nil, ctx.Err() case err := <-errC: if err != nil { - return nil, fmt.Errorf("peer connection error: %w", err) + return nil, fmt.Errorf("peer connection failed for ufrag: %s", addr.ufrag) } } @@ -343,7 +342,12 @@ func (l *listener) Multiaddr() ma.Multiaddr { return l.localMultiaddr } -func awaitPeerConnectionOpen(ufrag string, pc *webrtc.PeerConnection) <-chan error { +// addOnConnectionStateChangeCallback adds the OnConnectionStateChange to the PeerConnection. +// The channel returned here: +// * is closed when the state changes to Connection +// * receives an error when the state changes to Failed +// * doesn't receive anything (nor is closed) when the state changes to Disconnected +func addOnConnectionStateChangeCallback(pc *webrtc.PeerConnection) <-chan error { errC := make(chan error, 1) var once sync.Once pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { @@ -352,7 +356,7 @@ func awaitPeerConnectionOpen(ufrag string, pc *webrtc.PeerConnection) <-chan err once.Do(func() { close(errC) }) case webrtc.PeerConnectionStateFailed: once.Do(func() { - errC <- fmt.Errorf("peerconnection failed: %s", ufrag) + errC <- errors.New("peerconnection failed") close(errC) }) case webrtc.PeerConnectionStateDisconnected: diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index f8118277cf..656ac57422 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -315,7 +315,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, fmt.Errorf("instantiate peerconnection: %w", err) } - errC := awaitPeerConnectionOpen(ufrag, pc) + errC := addOnConnectionStateChangeCallback(pc) // We need to set negotiated = true for this channel on both // the client and server to avoid DCEP errors. negotiated, id := handshakeChannelNegotiated, handshakeChannelID From 94fb35117eebc229faecfe262999906bb205091c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 10:28:29 +0700 Subject: [PATCH 71/86] webrtc: test it on multidim interop --- test-plans/ping-version.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-plans/ping-version.json b/test-plans/ping-version.json index 311cb58e5b..705934f2b7 100644 --- a/test-plans/ping-version.json +++ b/test-plans/ping-version.json @@ -6,7 +6,8 @@ "ws", "wss", "quic-v1", - "webtransport" + "webtransport", + "webrtc-direct" ], "secureChannels": [ "tls", From 9a8983867ae0f23f10e31023703f501e60571a27 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 11:18:29 +0700 Subject: [PATCH 72/86] webrtc: return the correct error for a closed listener --- p2p/transport/webrtc/listener.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 700510b41c..afb962f37d 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "net" - "os" "strings" "sync" "time" @@ -319,7 +318,7 @@ func (l *listener) setupConnection( func (l *listener) Accept() (tpt.CapableConn, error) { select { case <-l.ctx.Done(): - return nil, os.ErrClosed + return nil, tpt.ErrListenerClosed case conn := <-l.acceptQueue: return conn, nil } From dadb8770e236313780941f4a9ece2ff86e572d15 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 11:29:25 +0700 Subject: [PATCH 73/86] transport test: refactor TestDiscoverPeerIDFromSecurityNegotiation to work with WebRTC --- p2p/test/transport/transport_test.go | 69 +++++++++++----------------- 1 file changed, 26 insertions(+), 43 deletions(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 067bc3a935..75bf306d13 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -654,58 +654,41 @@ func TestDiscoverPeerIDFromSecurityNegotiation(t *testing.T) { return "", inputErr } - // runs a test to verify we can extract the peer ID from a target with just its address - runTest := func(t *testing.T, h host.Host) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Use a bogus peer ID so that when we connect to the target we get an error telling - // us the targets real peer ID - bogusPeerId, err := peer.Decode("QmadAdJ3f63JyNs65X7HHzqDwV53ynvCcKtNFvdNaz3nhk") - if err != nil { - t.Fatal("the hard coded bogus peerID is invalid") - } + for _, tc := range transportsToTest { + t.Run(tc.Name, func(t *testing.T) { + h1 := tc.HostGenerator(t, TransportTestCaseOpts{}) + h2 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true}) + defer h1.Close() + defer h2.Close() - ai := &peer.AddrInfo{ - ID: bogusPeerId, - Addrs: []multiaddr.Multiaddr{h.Addrs()[0]}, - } + // runs a test to verify we can extract the peer ID from a target with just its address + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - testHost, err := libp2p.New() - if err != nil { - t.Fatal(err) - } + // Use a bogus peer ID so that when we connect to the target we get an error telling + // us the targets real peer ID + bogusPeerId, err := peer.Decode("QmadAdJ3f63JyNs65X7HHzqDwV53ynvCcKtNFvdNaz3nhk") + require.NoError(t, err, "the hard coded bogus peerID is invalid") + + ai := &peer.AddrInfo{ + ID: bogusPeerId, + Addrs: []multiaddr.Multiaddr{h1.Addrs()[0]}, + } + + // Try connecting with the bogus peer ID + err = h2.Connect(ctx, *ai) + require.Error(t, err, "somehow we successfully connected to a bogus peerID!") - // Try connecting with the bogus peer ID - if err := testHost.Connect(ctx, *ai); err != nil { // Extract the actual peer ID from the error newPeerId, err := extractPeerIDFromError(err) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ai.ID = newPeerId - // Make sure the new ID is what we expected - if ai.ID != h.ID() { - t.Fatalf("peerID mismatch: expected %s, got %s", h.ID(), ai.ID) - } + require.Equal(t, h1.ID(), ai.ID) // and just to double-check try connecting again to make sure it works - if err := testHost.Connect(ctx, *ai); err != nil { - t.Fatal(err) - } - } else { - t.Fatal("somehow we successfully connected to a bogus peerID!") - } - } - - for _, tc := range transportsToTest { - t.Run(tc.Name, func(t *testing.T) { - h := tc.HostGenerator(t, TransportTestCaseOpts{}) - defer h.Close() - - runTest(t, h) + require.NoError(t, h2.Connect(ctx, *ai)) }) } } From 294437773e051f6f3190b8b80877eca4a6312680 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 11:34:55 +0700 Subject: [PATCH 74/86] webrtc: fix peer ID validation when dialing --- p2p/transport/webrtc/transport.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 656ac57422..26700c1763 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -490,12 +490,14 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerCon if err != nil { return nil, fmt.Errorf("generate prologue: %w", err) } - sessionTransport, err := t.noiseTpt.WithSessionOptions( - noise.Prologue(prologue), - noise.DisablePeerIDCheck(), - ) + opts := make([]noise.SessionOption, 0, 2) + opts = append(opts, noise.Prologue(prologue)) + if peer == "" { + opts = append(opts, noise.DisablePeerIDCheck()) + } + sessionTransport, err := t.noiseTpt.WithSessionOptions(opts...) if err != nil { - return nil, fmt.Errorf("instantiate transport: %w", err) + return nil, fmt.Errorf("failed to instantiate Noise transport: %w", err) } var secureConn sec.SecureConn if inbound { From 2377b247b96bd14b405f9229f1e28b7c1e116c8f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 11:38:16 +0700 Subject: [PATCH 75/86] webrtc: refactor to get rid of the messy unknownUfragCallback --- p2p/transport/webrtc/listener.go | 95 ++++++++++++------------- p2p/transport/webrtc/udpmux/mux.go | 40 +++++++---- p2p/transport/webrtc/udpmux/mux_test.go | 4 +- 3 files changed, 75 insertions(+), 64 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index afb962f37d..cdc7030d36 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -20,7 +20,6 @@ import ( manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" - "github.com/pion/ice/v2" pionlogger "github.com/pion/logging" "github.com/pion/webrtc/v3" "go.uber.org/zap/zapcore" @@ -40,15 +39,10 @@ const ( DefaultMaxInFlightConnections = 10 ) -type candidateAddr struct { - ufrag string - raddr *net.UDPAddr -} - type listener struct { transport *WebRTCTransport - mux ice.UDPMux + mux *udpmux.UDPMux config webrtc.Configuration localFingerprint webrtc.DTLSFingerprint @@ -60,14 +54,6 @@ type listener struct { // buffered incoming connections acceptQueue chan tpt.CapableConn - // Accepting a connection requires instantiating a peerconnection - // and a noise connection which is expensive. We therefore limit - // the number of in-flight connection requests. A connection - // is considered to be in flight from the instant it is handled - // until it is dequeued by a call to Accept, or errors out in some - // way. - inFlightInputQueue chan struct{} - // used to control the lifecycle of the listener ctx context.Context cancel context.CancelFunc @@ -94,11 +80,6 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack return nil, err } - inFlightQueueCh := make(chan struct{}, transport.maxInFlightConnections) - for i := uint32(0); i < transport.maxInFlightConnections; i++ { - inFlightQueueCh <- struct{}{} - } - l := &listener{ transport: transport, config: config, @@ -107,31 +88,54 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack localMultiaddr: laddr, localAddr: socket.LocalAddr(), acceptQueue: make(chan tpt.CapableConn), - inFlightInputQueue: inFlightQueueCh, } l.ctx, l.cancel = context.WithCancel(context.Background()) - mux := udpmux.NewUDPMux(socket, func(ufrag string, addr net.Addr) error { + mux := udpmux.NewUDPMux(socket) + l.mux = mux + mux.Start() + + go l.listen() + + return l, err +} + +func (l *listener) listen() { + // Accepting a connection requires instantiating a peerconnection + // and a noise connection which is expensive. We therefore limit + // the number of in-flight connection requests. A connection + // is considered to be in flight from the instant it is handled + // until it is dequeued by a call to Accept, or errors out in some + // way. + inFlightQueueCh := make(chan struct{}, l.transport.maxInFlightConnections) + for i := uint32(0); i < l.transport.maxInFlightConnections; i++ { + inFlightQueueCh <- struct{}{} + } + + for { select { case <-inFlightQueueCh: - // we have space to accept, Yihaa - default: - return errors.New("candidate chan full, dropping incoming candidate") + case <-l.ctx.Done(): + return + } + + candidate, err := l.mux.Accept(l.ctx) + if err != nil { + if l.ctx.Err() == nil { + log.Debugf("accepting candidate failed: %s", err) + } + return } go func() { - defer func() { - // free this spot once again - inFlightQueueCh <- struct{}{} - }() + defer func() { inFlightQueueCh <- struct{}{} }() // free this spot once again ctx, cancel := context.WithTimeout(l.ctx, candidateSetupTimeout) defer cancel() - candidateAddr := candidateAddr{ufrag: ufrag, raddr: addr.(*net.UDPAddr)} - conn, err := l.handleCandidate(ctx, &candidateAddr) + conn, err := l.handleCandidate(ctx, candidate) if err != nil { - log.Debugf("could not accept connection: %s: %v", ufrag, err) + log.Debugf("could not accept connection: %s: %v", candidate.Ufrag, err) return } @@ -139,23 +143,15 @@ func newListener(transport *WebRTCTransport, laddr ma.Multiaddr, socket net.Pack case <-ctx.Done(): log.Warn("could not push connection: ctx done") conn.Close() - case l.acceptQueue <- conn: - // block until the connection is accepted, - // or until we are done, this effectively blocks our in flight from continuing to progress + // acceptQueue is an unbuffered channel, so this block until the connection is accepted. } }() - - return nil - }) - l.mux = mux - mux.Start() - - return l, err + } } -func (l *listener) handleCandidate(ctx context.Context, addr *candidateAddr) (tpt.CapableConn, error) { - remoteMultiaddr, err := manet.FromNetAddr(addr.raddr) +func (l *listener) handleCandidate(ctx context.Context, candidate udpmux.Candidate) (tpt.CapableConn, error) { + remoteMultiaddr, err := manet.FromNetAddr(candidate.Addr) if err != nil { return nil, err } @@ -171,7 +167,7 @@ func (l *listener) handleCandidate(ctx context.Context, addr *candidateAddr) (tp if err != nil { return nil, err } - conn, err := l.setupConnection(ctx, scope, remoteMultiaddr, addr) + conn, err := l.setupConnection(ctx, scope, remoteMultiaddr, candidate) if err != nil { scope.Done() return nil, err @@ -185,7 +181,7 @@ func (l *listener) handleCandidate(ctx context.Context, addr *candidateAddr) (tp func (l *listener) setupConnection( ctx context.Context, scope network.ConnManagementScope, - remoteMultiaddr ma.Multiaddr, addr *candidateAddr, + remoteMultiaddr ma.Multiaddr, candidate udpmux.Candidate, ) (tConn tpt.CapableConn, err error) { var pc *webrtc.PeerConnection defer func() { @@ -215,7 +211,7 @@ func (l *listener) setupConnection( settingEngine := webrtc.SettingEngine{LoggerFactory: loggerFactory} settingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleServer) - settingEngine.SetICECredentials(addr.ufrag, addr.ufrag) + settingEngine.SetICECredentials(candidate.Ufrag, candidate.Ufrag) settingEngine.SetLite(true) settingEngine.SetICEUDPMux(l.mux) settingEngine.SetIncludeLoopbackCandidate(true) @@ -245,7 +241,7 @@ func (l *listener) setupConnection( errC := addOnConnectionStateChangeCallback(pc) // Infer the client SDP from the incoming STUN message by setting the ice-ufrag. if err := pc.SetRemoteDescription(webrtc.SessionDescription{ - SDP: createClientSDP(addr.raddr, addr.ufrag), + SDP: createClientSDP(candidate.Addr, candidate.Ufrag), Type: webrtc.SDPTypeOffer, }); err != nil { return nil, err @@ -263,9 +259,8 @@ func (l *listener) setupConnection( return nil, ctx.Err() case err := <-errC: if err != nil { - return nil, fmt.Errorf("peer connection failed for ufrag: %s", addr.ufrag) + return nil, fmt.Errorf("peer connection failed for ufrag: %s", candidate.Ufrag) } - } rwc, err := getDetachedChannel(ctx, rawDatachannel) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index c1ad8d1fee..64b9eb5212 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -18,6 +18,11 @@ var log = logging.Logger("webrtc-udpmux") const ReceiveMTU = 1500 +type Candidate struct { + Ufrag string + Addr *net.UDPAddr +} + // UDPMux multiplexes multiple ICE connections over a single net.PacketConn, // generally a UDP socket. // @@ -32,13 +37,11 @@ const ReceiveMTU = 1500 // If it is, we read the ufrag from the STUN packet and use it to check if there // is a connection associated with the (ufrag, IP address family) pair. // If found we add the association to the address map. -// Otherwise, this is a previously unseen IP address and the unknownUfragCallback -// callback is called. type UDPMux struct { - socket net.PacketConn - unknownUfragCallback func(string, net.Addr) error + socket net.PacketConn storage *udpMuxStorage + queue chan Candidate // the context controls the lifecycle of the mux wg sync.WaitGroup @@ -48,14 +51,14 @@ type UDPMux struct { var _ ice.UDPMux = &UDPMux{} -func NewUDPMux(socket net.PacketConn, unknownUfragCallback func(string, net.Addr) error) *UDPMux { +func NewUDPMux(socket net.PacketConn) *UDPMux { ctx, cancel := context.WithCancel(context.Background()) mux := &UDPMux{ - ctx: ctx, - cancel: cancel, - socket: socket, - unknownUfragCallback: unknownUfragCallback, - storage: newUDPMuxStorage(), + ctx: ctx, + cancel: cancel, + socket: socket, + storage: newUDPMuxStorage(), + queue: make(chan Candidate, 32), } return mux @@ -188,8 +191,10 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { connCreated, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) if connCreated { - if err := mux.unknownUfragCallback(ufrag, udpAddr); err != nil { - log.Debugf("creating connection failed: %s", err) + select { + case mux.queue <- Candidate{Addr: udpAddr, Ufrag: ufrag}: + default: + log.Debugw("queue full, dropping incoming candidate", "ufrag", ufrag, "addr", udpAddr) conn.Close() return false } @@ -202,6 +207,17 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { return true } +func (mux *UDPMux) Accept(ctx context.Context) (Candidate, error) { + select { + case c := <-mux.queue: + return c, nil + case <-ctx.Done(): + return Candidate{}, ctx.Err() + case <-mux.ctx.Done(): + return Candidate{}, mux.ctx.Err() + } +} + type ufragConnKey struct { ufrag string isIPv6 bool diff --git a/p2p/transport/webrtc/udpmux/mux_test.go b/p2p/transport/webrtc/udpmux/mux_test.go index 498be61dcc..e190ae5f00 100644 --- a/p2p/transport/webrtc/udpmux/mux_test.go +++ b/p2p/transport/webrtc/udpmux/mux_test.go @@ -60,7 +60,7 @@ var ( ) func TestUDPMux_GetConn(t *testing.T) { - m := NewUDPMux(dummyPacketConn{}, nil) + m := NewUDPMux(dummyPacketConn{}) require.False(t, hasConn(m, "test", false)) conn, err := m.GetConn("test", &addrV4) require.NoError(t, err) @@ -75,7 +75,7 @@ func TestUDPMux_GetConn(t *testing.T) { } func TestUDPMux_RemoveConnectionOnClose(t *testing.T) { - mux := NewUDPMux(dummyPacketConn{}, nil) + mux := NewUDPMux(dummyPacketConn{}) conn, err := mux.GetConn("test", &addrV4) require.NoError(t, err) require.NotNil(t, conn) From c38865351904de4349d973b46efa2825337634da Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 14:17:39 +0700 Subject: [PATCH 76/86] interop: enable the WebRTC transport --- test-plans/cmd/ping/main.go | 9 ++- test-plans/go.mod | 21 ++++++ test-plans/go.sum | 134 ++++++++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 3 deletions(-) diff --git a/test-plans/cmd/ping/main.go b/test-plans/cmd/ping/main.go index df90a4bf08..c836a72e72 100644 --- a/test-plans/cmd/ping/main.go +++ b/test-plans/cmd/ping/main.go @@ -15,6 +15,8 @@ import ( "strconv" "time" + libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc" + "github.com/go-redis/redis/v8" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" @@ -103,6 +105,9 @@ func main() { case "webtransport": options = append(options, libp2p.Transport(libp2pwebtransport.New)) listenAddr = fmt.Sprintf("/ip4/%s/udp/0/quic-v1/webtransport", ip) + case "webrtc-direct": + options = append(options, libp2p.Transport(libp2pwebrtc.New)) + listenAddr = fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", ip) default: log.Fatalf("Unsupported transport: %s", transport) } @@ -112,13 +117,11 @@ func main() { var skipMuxer bool var skipSecureChannel bool switch transport { - case "quic": - fallthrough case "quic-v1": fallthrough case "webtransport": fallthrough - case "webrtc": + case "webrtc-direct": skipMuxer = true skipSecureChannel = true } diff --git a/test-plans/go.mod b/test-plans/go.mod index 93554a2356..4b08c78e9c 100644 --- a/test-plans/go.mod +++ b/test-plans/go.mod @@ -26,6 +26,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -40,6 +41,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect @@ -78,7 +80,24 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pion/datachannel v1.5.5 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect + github.com/pion/ice/v2 v2.3.6 // indirect + github.com/pion/interceptor v0.1.17 // indirect + github.com/pion/logging v0.2.2 // indirect + github.com/pion/mdns v0.0.7 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.10 // indirect + github.com/pion/rtp v1.7.13 // indirect + github.com/pion/sctp v1.8.7 // indirect + github.com/pion/sdp/v3 v3.0.6 // indirect + github.com/pion/srtp/v2 v2.0.15 // indirect + github.com/pion/stun v0.6.0 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/turn/v2 v2.1.0 // indirect + github.com/pion/webrtc/v3 v3.2.9 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect @@ -89,10 +108,12 @@ require ( github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/test-plans/go.sum b/test-plans/go.sum index f14d5e5b9f..f0ef3a53c1 100644 --- a/test-plans/go.sum +++ b/test-plans/go.sum @@ -54,6 +54,7 @@ github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwU github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -61,6 +62,7 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -79,11 +81,21 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= @@ -95,6 +107,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -102,6 +116,7 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -207,10 +222,19 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= @@ -218,6 +242,45 @@ github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= +github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= +github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= +github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= +github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= +github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= +github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= +github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= +github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= +github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= +github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= +github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= +github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= +github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= +github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= +github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= +github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= +github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= +github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= +github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= +github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= +github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= +github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= +github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= +github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -248,6 +311,7 @@ github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtD github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -278,12 +342,21 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= @@ -291,6 +364,7 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -316,6 +390,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -330,6 +407,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -344,10 +423,19 @@ golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -363,6 +451,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -372,23 +462,51 @@ golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -403,9 +521,12 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -428,22 +549,35 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 0be0e6579620ff2190c8f926686176386b70053c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 25 Aug 2023 15:22:59 +0700 Subject: [PATCH 77/86] webrtc: move the udpMuxStorage into the UDPMux There was never a good reason to have this as a separate struct to begin with. --- p2p/transport/webrtc/udpmux/mux.go | 99 ++++++++++--------------- p2p/transport/webrtc/udpmux/mux_test.go | 6 +- 2 files changed, 42 insertions(+), 63 deletions(-) diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index 64b9eb5212..4dd0bf78c2 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -40,8 +40,11 @@ type Candidate struct { type UDPMux struct { socket net.PacketConn - storage *udpMuxStorage - queue chan Candidate + queue chan Candidate + + mx sync.Mutex + ufragMap map[ufragConnKey]*muxedConnection + addrMap map[string]*muxedConnection // the context controls the lifecycle of the mux wg sync.WaitGroup @@ -54,11 +57,12 @@ var _ ice.UDPMux = &UDPMux{} func NewUDPMux(socket net.PacketConn) *UDPMux { ctx, cancel := context.WithCancel(context.Background()) mux := &UDPMux{ - ctx: ctx, - cancel: cancel, - socket: socket, - storage: newUDPMuxStorage(), - queue: make(chan Candidate, 32), + ctx: ctx, + cancel: cancel, + socket: socket, + ufragMap: make(map[ufragConnKey]*muxedConnection), + addrMap: make(map[string]*muxedConnection), + queue: make(chan Candidate, 32), } return mux @@ -86,8 +90,14 @@ func (mux *UDPMux) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) if !ok { return nil, fmt.Errorf("unexpected address type: %T", addr) } - isIPv6 := ok && a.IP.To4() == nil - return mux.getOrCreateConn(ufrag, isIPv6, addr) + select { + case <-mux.ctx.Done(): + return nil, io.ErrClosedPipe + default: + isIPv6 := ok && a.IP.To4() == nil + _, conn := mux.getOrCreateConn(ufrag, isIPv6, mux, addr) + return conn, nil + } } // Close implements ice.UDPMux @@ -103,23 +113,6 @@ func (mux *UDPMux) Close() error { return nil } -// RemoveConnByUfrag implements ice.UDPMux -func (mux *UDPMux) RemoveConnByUfrag(ufrag string) { - if ufrag != "" { - mux.storage.RemoveConnByUfrag(ufrag) - } -} - -func (mux *UDPMux) getOrCreateConn(ufrag string, isIPv6 bool, addr net.Addr) (net.PacketConn, error) { - select { - case <-mux.ctx.Done(): - return nil, io.ErrClosedPipe - default: - _, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, addr) - return conn, nil - } -} - // writeTo writes a packet to the underlying net.PacketConn func (mux *UDPMux) writeTo(buf []byte, addr net.Addr) (int, error) { return mux.socket.WriteTo(buf, addr) @@ -160,7 +153,10 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { // Connections are indexed by remote address. We first // check if the remote address has a connection associated // with it. If yes, we push the received packet to the connection - if conn, ok := mux.storage.GetConnByAddr(udpAddr); ok { + mux.mx.Lock() + conn, ok := mux.addrMap[addr.String()] + mux.mx.Unlock() + if ok { if err := conn.Push(buf); err != nil { log.Debugf("could not push packet: %v", err) return false @@ -189,7 +185,7 @@ func (mux *UDPMux) processPacket(buf []byte, addr net.Addr) (processed bool) { return false } - connCreated, conn := mux.storage.GetOrCreateConn(ufrag, isIPv6, mux, udpAddr) + connCreated, conn := mux.getOrCreateConn(ufrag, isIPv6, mux, udpAddr) if connCreated { select { case mux.queue <- Candidate{Addr: udpAddr, Ufrag: ufrag}: @@ -244,53 +240,36 @@ func ufragFromSTUNMessage(msg *stun.Message) (string, error) { return string(attr[index+1:]), nil } -type udpMuxStorage struct { - sync.Mutex - - ufragMap map[ufragConnKey]*muxedConnection - addrMap map[string]*muxedConnection -} - -func newUDPMuxStorage() *udpMuxStorage { - return &udpMuxStorage{ - ufragMap: make(map[ufragConnKey]*muxedConnection), - addrMap: make(map[string]*muxedConnection), +func (mux *UDPMux) RemoveConnByUfrag(ufrag string) { + if ufrag == "" { + return } -} -func (s *udpMuxStorage) RemoveConnByUfrag(ufrag string) { - s.Lock() - defer s.Unlock() + mux.mx.Lock() + defer mux.mx.Unlock() for _, isIPv6 := range [...]bool{true, false} { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} - if conn, ok := s.ufragMap[key]; ok { - delete(s.ufragMap, key) - delete(s.addrMap, conn.RemoteAddr().String()) + if conn, ok := mux.ufragMap[key]; ok { + delete(mux.ufragMap, key) + delete(mux.addrMap, conn.RemoteAddr().String()) } } } -func (s *udpMuxStorage) GetOrCreateConn(ufrag string, isIPv6 bool, mux *UDPMux, addr net.Addr) (created bool, _ *muxedConnection) { +func (mux *UDPMux) getOrCreateConn(ufrag string, isIPv6 bool, _ *UDPMux, addr net.Addr) (created bool, _ *muxedConnection) { key := ufragConnKey{ufrag: ufrag, isIPv6: isIPv6} - s.Lock() - defer s.Unlock() + mux.mx.Lock() + defer mux.mx.Unlock() - if conn, ok := s.ufragMap[key]; ok { + if conn, ok := mux.ufragMap[key]; ok { return false, conn } - conn := newMuxedConnection(mux, func() { s.RemoveConnByUfrag(ufrag) }, addr) - s.ufragMap[key] = conn - s.addrMap[addr.String()] = conn + conn := newMuxedConnection(mux, func() { mux.RemoveConnByUfrag(ufrag) }, addr) + mux.ufragMap[key] = conn + mux.addrMap[addr.String()] = conn return true, conn } - -func (s *udpMuxStorage) GetConnByAddr(addr *net.UDPAddr) (*muxedConnection, bool) { - s.Lock() - conn, ok := s.addrMap[addr.String()] - s.Unlock() - return conn, ok -} diff --git a/p2p/transport/webrtc/udpmux/mux_test.go b/p2p/transport/webrtc/udpmux/mux_test.go index e190ae5f00..4121b6fdf5 100644 --- a/p2p/transport/webrtc/udpmux/mux_test.go +++ b/p2p/transport/webrtc/udpmux/mux_test.go @@ -48,9 +48,9 @@ func (dummyPacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) { } func hasConn(m *UDPMux, ufrag string, isIPv6 bool) bool { - m.storage.Lock() - _, ok := m.storage.ufragMap[ufragConnKey{ufrag: ufrag, isIPv6: isIPv6}] - m.storage.Unlock() + m.mx.Lock() + _, ok := m.ufragMap[ufragConnKey{ufrag: ufrag, isIPv6: isIPv6}] + m.mx.Unlock() return ok } From f674480964a44e78ea18e61d7575cf429274537d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 26 Aug 2023 10:38:15 +0700 Subject: [PATCH 78/86] webrtc: mark the transport as experimental (#2525) * webrtc: mark the transport as experimental * Update p2p/transport/webrtc/transport.go Co-authored-by: Prithvi Shahi --------- Co-authored-by: Marco Munizaga Co-authored-by: Prithvi Shahi --- p2p/transport/webrtc/transport.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 26700c1763..dd4028d1f2 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -1,5 +1,12 @@ // Package libp2pwebrtc implements the WebRTC transport for go-libp2p, -// as officially described in https://github.com/libp2p/specs/tree/master/webrtc. +// as described in https://github.com/libp2p/specs/tree/master/webrtc. +// +// At this point, this package is EXPERIMENTAL, and the WebRTC transport is not enabled by default. +// While we're fairly confident that the implementation correctly implements the specification, +// we're not making any guarantees regarding its security (especially regarding resource exhaustion attacks). +// Fixes, even for security-related issues, will be conducted in the open. +// +// Experimentation is encouraged. Please open an issue if you encounter any problems with this transport. // // The udpmux subpackage contains the logic for multiplexing multiple WebRTC (ICE) // connections over a single UDP socket. From 0875a06f2ce5afd7a7334db0c634b92de1a21091 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 26 Aug 2023 11:04:28 +0700 Subject: [PATCH 79/86] webrtc: fix race condition when closing --- p2p/transport/webrtc/connection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 53b0dff721..550c6a8cdc 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -140,6 +140,8 @@ func (c *connection) Close() error { return nil } + c.m.Lock() + defer c.m.Unlock() c.scope.Done() c.closeErr = errors.New("connection closed") c.cancel() From 9032331b2a75aa213e1b2ae17fa6273264d573d3 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 26 Aug 2023 11:05:57 +0700 Subject: [PATCH 80/86] webrtc: add comment for controlMsgSize hack --- p2p/transport/webrtc/stream_write.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 4578c564c1..c7eb3bf7a4 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -149,7 +149,9 @@ func (s *stream) availableSendSpace() int { return availableSpace } -const controlMsgSize = 100 // TODO: use actual message size +// There's no way to determine the size of a Protobuf message in the pbio package. +// Setting the size to 100 works as long as the control messages (incl. the varint prefix) are smaller than that value. +const controlMsgSize = 100 func (s *stream) sendControlMessage(msg *pb.Message) error { available := s.availableSendSpace() From dd0e159556f81998ba41139cbcb3ac7d8815ac8c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 26 Aug 2023 11:20:54 +0700 Subject: [PATCH 81/86] webrtc: use a channel instead of a custom packet queue implementation --- .../webrtc/udpmux/muxed_connection.go | 26 ++++-- p2p/transport/webrtc/udpmux/packetqueue.go | 80 ------------------- .../webrtc/udpmux/packetqueue_bench_test.go | 40 ---------- .../webrtc/udpmux/packetqueue_test.go | 50 ------------ 4 files changed, 21 insertions(+), 175 deletions(-) delete mode 100644 p2p/transport/webrtc/udpmux/packetqueue.go delete mode 100644 p2p/transport/webrtc/udpmux/packetqueue_bench_test.go delete mode 100644 p2p/transport/webrtc/udpmux/packetqueue_test.go diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index 7c3eee26aa..95c916bd0a 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -2,12 +2,15 @@ package udpmux import ( "context" + "errors" "net" "time" ) var _ net.PacketConn = &muxedConnection{} +const queueLen = 128 + // muxedConnection provides a net.PacketConn abstraction // over packetQueue and adds the ability to store addresses // from which this connection (indexed by ufrag) received @@ -16,7 +19,7 @@ type muxedConnection struct { ctx context.Context cancel context.CancelFunc onClose func() - pq *packetQueue + queue chan []byte remote net.Addr mux *UDPMux } @@ -28,7 +31,7 @@ func newMuxedConnection(mux *UDPMux, onClose func(), remote net.Addr) *muxedConn return &muxedConnection{ ctx: ctx, cancel: cancel, - pq: newPacketQueue(), + queue: make(chan []byte, queueLen), onClose: onClose, remote: remote, mux: mux, @@ -36,12 +39,25 @@ func newMuxedConnection(mux *UDPMux, onClose func(), remote net.Addr) *muxedConn } func (c *muxedConnection) Push(buf []byte) error { - return c.pq.Push(buf) + select { + case c.queue <- buf: + return nil + default: + return errors.New("queue full") + } } func (c *muxedConnection) ReadFrom(p []byte) (int, net.Addr, error) { - n, err := c.pq.Pop(c.ctx, p) - return n, c.remote, err + select { + case buf := <-c.queue: + n := copy(p, buf) // This might discard parts of the packet, if p is too short + if n < len(buf) { + log.Debugf("short read, had %d, read %d", len(buf), n) + } + return n, c.remote, nil + case <-c.ctx.Done(): + return 0, nil, c.ctx.Err() + } } func (c *muxedConnection) WriteTo(p []byte, addr net.Addr) (n int, err error) { diff --git a/p2p/transport/webrtc/udpmux/packetqueue.go b/p2p/transport/webrtc/udpmux/packetqueue.go deleted file mode 100644 index 054419818a..0000000000 --- a/p2p/transport/webrtc/udpmux/packetqueue.go +++ /dev/null @@ -1,80 +0,0 @@ -package udpmux - -import ( - "context" - "errors" - "sync" - - pool "github.com/libp2p/go-buffer-pool" -) - -type packet struct { - buf []byte -} - -var errTooManyPackets = errors.New("too many packets in queue; dropping") - -const maxPacketsInQueue = 128 - -type packetQueue struct { - packetsMux sync.Mutex - packetsCh chan struct{} - packets []packet -} - -func newPacketQueue() *packetQueue { - return &packetQueue{ - packetsCh: make(chan struct{}, 1), - } -} - -// Pop reads a packet from the packetQueue. -func (pq *packetQueue) Pop(ctx context.Context, buf []byte) (int, error) { -start: - select { - case <-pq.packetsCh: - pq.packetsMux.Lock() - - if len(pq.packets) == 0 { - pq.packetsMux.Unlock() - goto start - } - - defer pq.packetsMux.Unlock() - p := pq.packets[0] - n := copy(buf, p.buf) - if n < len(p.buf) { - log.Debugf("short read, had %d, read %d", len(p.buf), n) - } - pq.packets = pq.packets[1:] - pool.Put(p.buf) - if len(pq.packets) > 0 { - // to make sure a next pop call will work - pq.notify() - } - return n, nil - case <-ctx.Done(): - return 0, ctx.Err() - } -} - -// Push adds a packet to the packetQueue -func (pq *packetQueue) Push(buf []byte) error { - pq.packetsMux.Lock() - defer pq.packetsMux.Unlock() - - if len(pq.packets) >= maxPacketsInQueue { - return errTooManyPackets - } - - pq.packets = append(pq.packets, packet{buf}) - pq.notify() - return nil -} - -func (pq *packetQueue) notify() { - select { - case pq.packetsCh <- struct{}{}: - default: - } -} diff --git a/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go b/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go deleted file mode 100644 index aaf8688d85..0000000000 --- a/p2p/transport/webrtc/udpmux/packetqueue_bench_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package udpmux - -import ( - "context" - "fmt" - "testing" - - pool "github.com/libp2p/go-buffer-pool" -) - -var sizes = []int{ - 1, - 10, - 100, - maxPacketsInQueue, -} - -func BenchmarkQueue(b *testing.B) { - for _, dequeue := range [...]bool{true, false} { - for _, input := range sizes { - testCase := fmt.Sprintf("enqueue_%d", input) - if dequeue { - testCase = testCase + "_dequeue" - } - b.Run(testCase, func(b *testing.B) { - pq := newPacketQueue() - buf := make([]byte, 256) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for k := 0; k < input; k++ { - pq.Push(pool.Get(255)) - } - for k := 0; k < input; k++ { - pq.Pop(context.Background(), buf) - } - } - }) - } - } -} diff --git a/p2p/transport/webrtc/udpmux/packetqueue_test.go b/p2p/transport/webrtc/udpmux/packetqueue_test.go deleted file mode 100644 index 74cfa6b00a..0000000000 --- a/p2p/transport/webrtc/udpmux/packetqueue_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package udpmux - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestPacketQueue_QueuePacketsForRead(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - pq := newPacketQueue() - pq.Push([]byte{1, 2, 3}) - pq.Push([]byte{5, 6, 7, 8}) - - size, err := pq.Pop(ctx, make([]byte, 6)) - require.NoError(t, err) - require.Equal(t, size, 3) - - size, err = pq.Pop(ctx, make([]byte, 6)) - require.NoError(t, err) - require.Equal(t, size, 4) -} - -func TestPacketQueue_WaitsForData(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - pq := newPacketQueue() - - timer := time.AfterFunc(200*time.Millisecond, func() { - pq.Push([]byte("foobar")) - }) - - defer timer.Stop() - size, err := pq.Pop(ctx, make([]byte, 6)) - require.NoError(t, err) - require.Equal(t, size, 6) -} - -func TestPacketQueue_DropsPacketsWhenQueueIsFull(t *testing.T) { - pq := newPacketQueue() - for i := 0; i < maxPacketsInQueue; i++ { - require.NoError(t, pq.Push(make([]byte, 10))) - } - require.ErrorIs(t, pq.Push(make([]byte, 10)), errTooManyPackets) -} From 88ea49d372646d513e1632a8a9c4a8447a4f8277 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 26 Aug 2023 11:34:02 +0700 Subject: [PATCH 82/86] webrtc: simplify super messy IP logic in tests --- p2p/transport/webrtc/fetch_ip_linux_test.go | 10 ------ p2p/transport/webrtc/fetch_ip_test.go | 40 --------------------- p2p/transport/webrtc/transport_test.go | 38 +++++++++----------- 3 files changed, 16 insertions(+), 72 deletions(-) delete mode 100644 p2p/transport/webrtc/fetch_ip_linux_test.go delete mode 100644 p2p/transport/webrtc/fetch_ip_test.go diff --git a/p2p/transport/webrtc/fetch_ip_linux_test.go b/p2p/transport/webrtc/fetch_ip_linux_test.go deleted file mode 100644 index 8bfe033e13..0000000000 --- a/p2p/transport/webrtc/fetch_ip_linux_test.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build linux -// +build linux - -package libp2pwebrtc - -import "net" - -func getListenerAndDialerIP() (net.IP, net.IP) { - return net.IPv4(0, 0, 0, 0), net.IPv4(127, 0, 0, 1) -} diff --git a/p2p/transport/webrtc/fetch_ip_test.go b/p2p/transport/webrtc/fetch_ip_test.go deleted file mode 100644 index 62dde5f434..0000000000 --- a/p2p/transport/webrtc/fetch_ip_test.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build !linux -// +build !linux - -package libp2pwebrtc - -import "net" - -// non-linux builds need to bind to a non-loopback interface -// to accept incoming connections. 0.0.0.0 does not work since -// Pion will bind to a local interface which is not loopback -// and there may not be a route from, say 192.168.0.0/16 to 0.0.0.0. - -func getListenerAndDialerIP() (listenerIp net.IP, dialerIp net.IP) { - listenerIp = net.IPv4(0, 0, 0, 0) - dialerIp = net.IPv4(0, 0, 0, 0) - ifaces, err := net.Interfaces() - if err != nil { - return - } - for _, iface := range ifaces { - log.Debugf("checking interface: %s", iface.Name) - if iface.Flags&net.FlagUp == 0 { - continue - } - addrs, err := iface.Addrs() - if err != nil { - return - } - for _, addr := range addrs { - if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.IsPrivate() { - if ipnet.IP.To4() != nil { - listenerIp = ipnet.IP.To4() - dialerIp = listenerIp - return - } - } - } - } - return -} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 4bf649c8ed..19d39ba282 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -42,12 +42,6 @@ func getTransport(t *testing.T, opts ...Option) (*WebRTCTransport, peer.ID) { return transport, peerID } -var listenerIP, dialerIP net.IP - -func init() { - listenerIP, dialerIP = getListenerAndDialerIP() -} - func TestTransportWebRTC_CanDial(t *testing.T) { tr, _ := getTransport(t) invalid := []string{ @@ -110,7 +104,7 @@ func TestTransportWebRTC_DialFailsOnUnsupportedHashFunction(t *testing.T) { func TestTransportWebRTC_CanListenSingle(t *testing.T) { tr, listeningPeer := getTransport(t) tr1, connectingPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -151,7 +145,7 @@ func TestTransportWebRTC_CanListenMultiple(t *testing.T) { count := 3 tr, listeningPeer := getTransport(t, WithListenerMaxInFlightConnections(uint32(count))) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -194,7 +188,7 @@ func TestTransportWebRTC_CanListenMultiple(t *testing.T) { func TestTransportWebRTC_CanCreateSuccessiveConnections(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) count := 2 @@ -217,7 +211,7 @@ func TestTransportWebRTC_CanCreateSuccessiveConnections(t *testing.T) { func TestTransportWebRTC_ListenerCanCreateStreams(t *testing.T) { tr, listeningPeer := getTransport(t) tr1, connectingPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -259,7 +253,7 @@ func TestTransportWebRTC_ListenerCanCreateStreams(t *testing.T) { func TestTransportWebRTC_DialerCanCreateStreams(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -302,7 +296,7 @@ func TestTransportWebRTC_DialerCanCreateStreams(t *testing.T) { func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { count := 5 tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -364,7 +358,7 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { func TestTransportWebRTC_Deadline(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) tr1, connectingPeer := getTransport(t) @@ -417,7 +411,7 @@ func TestTransportWebRTC_Deadline(t *testing.T) { func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -457,7 +451,7 @@ func TestTransportWebRTC_StreamWriteBufferContention(t *testing.T) { func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -507,7 +501,7 @@ func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { func TestTransportWebRTC_RemoteReadsAfterClose2(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -555,7 +549,7 @@ func TestTransportWebRTC_RemoteReadsAfterClose2(t *testing.T) { func TestTransportWebRTC_Close(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) require.NoError(t, err) @@ -594,7 +588,7 @@ func TestTransportWebRTC_Close(t *testing.T) { func TestTransportWebRTC_PeerConnectionDTLSFailed(t *testing.T) { tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") ln, err := tr.Listen(listenMultiaddr) require.NoError(t, err) defer ln.Close() @@ -623,14 +617,14 @@ func TestConnectionTimeoutOnListener(t *testing.T) { tr.peerConnectionTimeouts.Failed = 150 * time.Millisecond tr.peerConnectionTimeouts.Keepalive = 50 * time.Millisecond - listenMultiaddr := ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP)) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") ln, err := tr.Listen(listenMultiaddr) require.NoError(t, err) defer ln.Close() var drop atomic.Bool - proxy, err := quicproxy.NewQuicProxy(fmt.Sprintf("%s:0", listenerIP), &quicproxy.Opts{ - RemoteAddr: fmt.Sprintf("%s:%d", listenerIP, ln.Addr().(*net.UDPAddr).Port), + proxy, err := quicproxy.NewQuicProxy("127.0.0.1:0", &quicproxy.Opts{ + RemoteAddr: fmt.Sprintf("127.0.0.1:%d", ln.Addr().(*net.UDPAddr).Port), DropPacket: func(quicproxy.Direction, []byte) bool { return drop.Load() }, }) require.NoError(t, err) @@ -687,7 +681,7 @@ func TestMaxInFlightRequests(t *testing.T) { tr, listeningPeer := getTransport(t, WithListenerMaxInFlightConnections(count), ) - ln, err := tr.Listen(ma.StringCast(fmt.Sprintf("/ip4/%s/udp/0/webrtc-direct", listenerIP))) + ln, err := tr.Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct")) require.NoError(t, err) defer ln.Close() From f15181e7fdf8a2c4e17195e7d623ad4dce11b945 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 29 Aug 2023 12:13:43 +0700 Subject: [PATCH 83/86] webrtc: remove connections from UDP mux if connection setup fails --- p2p/transport/webrtc/listener.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index cdc7030d36..0b29bf655d 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -135,6 +135,7 @@ func (l *listener) listen() { conn, err := l.handleCandidate(ctx, candidate) if err != nil { + l.mux.RemoveConnByUfrag(candidate.Ufrag) log.Debugf("could not accept connection: %s: %v", candidate.Ufrag, err) return } From a1644b11d048c274ab8b9ca6b871e27e7439020e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 29 Aug 2023 13:06:04 +0700 Subject: [PATCH 84/86] webrtc: drain the packet queue when the connection is closed --- p2p/transport/webrtc/udpmux/muxed_connection.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/udpmux/muxed_connection.go b/p2p/transport/webrtc/udpmux/muxed_connection.go index 95c916bd0a..202e0cae2c 100644 --- a/p2p/transport/webrtc/udpmux/muxed_connection.go +++ b/p2p/transport/webrtc/udpmux/muxed_connection.go @@ -39,6 +39,11 @@ func newMuxedConnection(mux *UDPMux, onClose func(), remote net.Addr) *muxedConn } func (c *muxedConnection) Push(buf []byte) error { + select { + case <-c.ctx.Done(): + return errors.New("closed") + default: + } select { case c.queue <- buf: return nil @@ -72,7 +77,14 @@ func (c *muxedConnection) Close() error { } c.onClose() c.cancel() - return nil + // drain the packet queue + for { + select { + case <-c.queue: + default: + return nil + } + } } func (c *muxedConnection) LocalAddr() net.Addr { return c.mux.socket.LocalAddr() } From 2f62254a2ffda66686f0dc25653aefd4d0a4c922 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 29 Aug 2023 22:12:22 +0700 Subject: [PATCH 85/86] transport tests: relax InterceptAccept counter for WebRTC --- p2p/test/transport/gating_test.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/p2p/test/transport/gating_test.go b/p2p/test/transport/gating_test.go index ba5d13cd07..65d732ba84 100644 --- a/p2p/test/transport/gating_test.go +++ b/p2p/test/transport/gating_test.go @@ -2,6 +2,7 @@ package transport_integration import ( "context" + "strings" "testing" "time" @@ -165,10 +166,20 @@ func TestInterceptAccept(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() // The basic host dials the first connection. - connGater.EXPECT().InterceptAccept(gomock.Any()).Do(func(addrs network.ConnMultiaddrs) { - // remove the certhash component from WebTransport addresses - require.Equal(t, stripCertHash(h2.Addrs()[0]), addrs.LocalMultiaddr()) - }) + if strings.Contains(tc.Name, "WebRTC") { + // In WebRTC, retransmissions of the STUN packet might cause us to create multiple connections, + // if the first connection attempt is rejected. + connGater.EXPECT().InterceptAccept(gomock.Any()).Do(func(addrs network.ConnMultiaddrs) { + // remove the certhash component from WebTransport addresses + require.Equal(t, stripCertHash(h2.Addrs()[0]), addrs.LocalMultiaddr()) + }).AnyTimes() + } else { + connGater.EXPECT().InterceptAccept(gomock.Any()).Do(func(addrs network.ConnMultiaddrs) { + // remove the certhash component from WebTransport addresses + require.Equal(t, stripCertHash(h2.Addrs()[0]), addrs.LocalMultiaddr()) + }) + } + h1.Peerstore().AddAddrs(h2.ID(), h2.Addrs(), time.Hour) _, err := h1.NewStream(ctx, h2.ID(), protocol.TestingID) require.Error(t, err) From 4a6e72637b1fbac6a49d566156ce2d32024958d9 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 12 Sep 2023 12:47:27 +0700 Subject: [PATCH 86/86] webrtc: don't close stream, but limit to 512 streams per direction --- p2p/test/transport/transport_test.go | 2 +- p2p/transport/webrtc/connection.go | 15 +++++++++++++++ p2p/transport/webrtc/stream.go | 4 +++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 75bf306d13..a7e98a0d85 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -247,7 +247,7 @@ func TestLotsOfDataManyStreams(t *testing.T) { // 64k buffer const bufSize = 64 << 10 sendBuf := [bufSize]byte{} - const totalStreams = 512 + const totalStreams = 500 const parallel = 8 // Total sends are > 20MiB require.Greater(t, len(sendBuf)*totalStreams, 20<<20) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 550c6a8cdc..fd31f8351a 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -27,6 +27,8 @@ var _ tpt.CapableConn = &connection{} const maxAcceptQueueLen = 10 +const maxDataChannelID = 1 << 10 + type errConnectionTimeout struct{} var _ net.Error = &errConnectionTimeout{} @@ -108,6 +110,12 @@ func newConnection( if c.IsClosed() { return } + // Limit the number of streams, since we're not able to actually properly close them. + // See https://github.com/libp2p/specs/issues/575 for details. + if *dc.ID() > maxDataChannelID { + c.Close() + return + } dc.OnOpen(func() { rwc, err := dc.Detach() if err != nil { @@ -166,6 +174,13 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if id > math.MaxUint16 { return nil, errors.New("exhausted stream ID space") } + // Limit the number of streams, since we're not able to actually properly close them. + // See https://github.com/libp2p/specs/issues/575 for details. + if id > maxDataChannelID { + c.Close() + return c.OpenStream(ctx) + } + streamID := uint16(id) dc, err := c.pc.CreateDataChannel("", &webrtc.DataChannelInit{ID: &streamID}) if err != nil { diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index b86128dc31..7e873f5634 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -192,7 +192,9 @@ func (s *stream) maybeDeclareStreamDone() { (s.receiveState == receiveStateReset || s.receiveState == receiveStateDataRead) && len(s.controlMsgQueue) == 0 { _ = s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) // pion ignores zero times - _ = s.dataChannel.Close() + // TODO: we should be closing the underlying datachannel, but this resets the stream + // See https://github.com/libp2p/specs/issues/575 for details. + // _ = s.dataChannel.Close() // TODO: write for the spawned reader to return s.onDone() }