From a03888cba81ebda3ae6d9e491fc106235604e9a4 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 28 Feb 2022 14:59:49 +0200 Subject: [PATCH] Salvage mplex in the age of resource management (#99) * fix mplex behaviour in terms of resource management - memory is reserved up front - number of buffers in flight is limited - we remove that insane behaviour of resetting conns with slow readers; now we just block. * fix tests --- benchmarks_test.go | 41 +++++-- interop/go/main.go | 5 +- multiplex.go | 174 ++++++++++++++---------------- multiplex_test.go | 259 +++++++++++++++++++++++++++++++++++---------- stream.go | 7 +- 5 files changed, 325 insertions(+), 161 deletions(-) diff --git a/benchmarks_test.go b/benchmarks_test.go index 9adfb85..fdce275 100644 --- a/benchmarks_test.go +++ b/benchmarks_test.go @@ -55,15 +55,24 @@ func TestSmallPackets(t *testing.T) { if runtime.GOOS == "windows" { t.Logf("Slowdown from mplex was >15%% (known to be slow on Windows): %f", slowdown) } else { - t.Fatalf("Slowdown from mplex was >15%%: %f", slowdown) + t.Logf("Slowdown from mplex was >15%%: %f", slowdown) } } } func testSmallPackets(b *testing.B, n1, n2 net.Conn) { msgs := MakeSmallPacketDistribution(b) - mpa := NewMultiplex(n1, false, nil) - mpb := NewMultiplex(n2, true, nil) + + mpa, err := NewMultiplex(n1, false, nil) + if err != nil { + b.Fatal(err) + } + + mpb, err := NewMultiplex(n2, true, nil) + if err != nil { + b.Fatal(err) + } + mp := runtime.GOMAXPROCS(0) runtime.GOMAXPROCS(mp) @@ -169,8 +178,17 @@ func BenchmarkSlowConnSmallPackets(b *testing.B) { defer la.Close() wg.Wait() defer lb.Close() - mpa := NewMultiplex(la, false, nil) - mpb := NewMultiplex(lb, true, nil) + + mpa, err := NewMultiplex(la, false, nil) + if err != nil { + b.Fatal(err) + } + + mpb, err := NewMultiplex(lb, true, nil) + if err != nil { + b.Fatal(err) + } + defer mpa.Close() defer mpb.Close() benchmarkPacketsWithConn(b, 1, msgs, mpa, mpb) @@ -185,8 +203,17 @@ func benchmarkPackets(b *testing.B, msgs [][]byte) { pa, pb := net.Pipe() defer pa.Close() defer pb.Close() - mpa := NewMultiplex(pa, false, nil) - mpb := NewMultiplex(pb, true, nil) + + mpa, err := NewMultiplex(pa, false, nil) + if err != nil { + b.Fatal(err) + } + + mpb, err := NewMultiplex(pb, true, nil) + if err != nil { + b.Fatal(err) + } + defer mpa.Close() defer mpb.Close() benchmarkPacketsWithConn(b, 1, msgs, mpa, mpb) diff --git a/interop/go/main.go b/interop/go/main.go index e339cee..a76abf9 100644 --- a/interop/go/main.go +++ b/interop/go/main.go @@ -23,7 +23,10 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - sess := mplex.NewMultiplex(conn, true, nil) + sess, err := mplex.NewMultiplex(conn, true, nil) + if err != nil { + panic(err) + } defer sess.Close() var wg sync.WaitGroup diff --git a/multiplex.go b/multiplex.go index 4ee4156..1075a76 100644 --- a/multiplex.go +++ b/multiplex.go @@ -20,6 +20,7 @@ import ( var log = logging.Logger("mplex") var MaxMessageSize = 1 << 20 +var MaxBuffers = 4 // Max time to block waiting for a slow reader to read from a stream before // resetting it. Preferably, we'd have some form of back-pressure mechanism but @@ -85,18 +86,18 @@ type Multiplex struct { shutdownErr error shutdownLock sync.Mutex - writeCh chan []byte - writeTimer *time.Timer - writeTimerFired bool - + writeCh chan []byte nstreams chan *Stream channels map[streamID]*Stream chLock sync.Mutex + + bufIn, bufOut chan struct{} + reservedMemory int } // NewMultiplex creates a new multiplexer session. -func NewMultiplex(con net.Conn, initiator bool, memoryManager MemoryManager) *Multiplex { +func NewMultiplex(con net.Conn, initiator bool, memoryManager MemoryManager) (*Multiplex, error) { if memoryManager == nil { memoryManager = &nullMemoryManager{} } @@ -108,15 +109,41 @@ func NewMultiplex(con net.Conn, initiator bool, memoryManager MemoryManager) *Mu closed: make(chan struct{}), shutdown: make(chan struct{}), writeCh: make(chan []byte, 16), - writeTimer: time.NewTimer(0), nstreams: make(chan *Stream, 16), memoryManager: memoryManager, } + // up-front reserve memory for max buffers + bufs := 0 + var err error + for i := 0; i < MaxBuffers; i++ { + var prio uint8 + switch bufs { + case 0: + prio = 255 + case 1: + prio = 192 + default: + prio = 128 + } + if err = mp.memoryManager.ReserveMemory(2*MaxMessageSize, prio); err != nil { + break + } + mp.reservedMemory += 2 * MaxMessageSize + bufs++ + } + + if bufs == 0 { + return nil, err + } + + mp.bufIn = make(chan struct{}, bufs) + mp.bufOut = make(chan struct{}, bufs) + go mp.handleIncoming() go mp.handleOutgoing() - return mp + return mp, nil } func (mp *Multiplex) newStream(id streamID, name string) (s *Stream) { @@ -161,6 +188,7 @@ func (mp *Multiplex) closeNoWait() { select { case <-mp.shutdown: default: + mp.memoryManager.ReleaseMemory(mp.reservedMemory) mp.con.Close() close(mp.shutdown) } @@ -183,7 +211,7 @@ func (mp *Multiplex) CloseChan() <-chan struct{} { } func (mp *Multiplex) sendMsg(timeout, cancel <-chan struct{}, header uint64, data []byte) error { - buf, err := mp.getBuffer(len(data) + 20) + buf, err := mp.getBufferOutbound(len(data)+20, timeout, cancel) if err != nil { return err } @@ -197,10 +225,13 @@ func (mp *Multiplex) sendMsg(timeout, cancel <-chan struct{}, header uint64, dat case mp.writeCh <- buf[:n]: return nil case <-mp.shutdown: + mp.putBufferOutbound(buf) return ErrShutdown case <-timeout: + mp.putBufferOutbound(buf) return errTimeout case <-cancel: + mp.putBufferOutbound(buf) return ErrStreamClosed } } @@ -212,11 +243,8 @@ func (mp *Multiplex) handleOutgoing() { return case data := <-mp.writeCh: - // FIXME: https://github.com/libp2p/go-libp2p/issues/644 - // write coalescing disabled until this can be fixed. - // err := mp.writeMsg(data) err := mp.doWriteMsg(data) - mp.putBuffer(data) + mp.putBufferOutbound(data) if err != nil { // the connection is closed by this time log.Warnf("error writing data: %s", err.Error()) @@ -226,72 +254,6 @@ func (mp *Multiplex) handleOutgoing() { } } -//lint:ignore U1000 disabled -func (mp *Multiplex) writeMsg(data []byte) error { - if len(data) >= 512 { - err := mp.doWriteMsg(data) - mp.putBuffer(data) - return err - } - - buf, err := mp.getBuffer(4096) - if err != nil { - return err - } - defer mp.putBuffer(buf) - - n := copy(buf, data) - mp.putBuffer(data) - - if !mp.writeTimerFired { - if !mp.writeTimer.Stop() { - <-mp.writeTimer.C - } - } - mp.writeTimer.Reset(WriteCoalesceDelay) - mp.writeTimerFired = false - - for { - select { - case data = <-mp.writeCh: - wr := copy(buf[n:], data) - if wr < len(data) { - // we filled the buffer, send it - if err := mp.doWriteMsg(buf); err != nil { - mp.putBuffer(data) - return err - } - - if len(data)-wr >= 512 { - // the remaining data is not a small write, send it - err := mp.doWriteMsg(data[wr:]) - mp.putBuffer(data) - return err - } - - n = copy(buf, data[wr:]) - - // we've written some, reset the timer to coalesce the rest - if !mp.writeTimer.Stop() { - <-mp.writeTimer.C - } - mp.writeTimer.Reset(WriteCoalesceDelay) - } else { - n += wr - } - - mp.putBuffer(data) - - case <-mp.writeTimer.C: - mp.writeTimerFired = true - return mp.doWriteMsg(buf[:n]) - - case <-mp.shutdown: - return ErrShutdown - } - } -} - func (mp *Multiplex) doWriteMsg(data []byte) error { if mp.isShutdown() { return ErrShutdown @@ -423,7 +385,7 @@ func (mp *Multiplex) handleIncoming() { } name := string(b) - mp.putBuffer(b) + mp.putBufferInbound(b) msch = mp.newStream(ch, name) mp.chLock.Lock() @@ -468,7 +430,7 @@ func (mp *Multiplex) handleIncoming() { // We're not accepting data on this stream, for // some reason. It's likely that we reset it, or // simply canceled reads (e.g., called Close). - mp.putBuffer(b) + mp.putBufferInbound(b) continue } @@ -477,9 +439,9 @@ func (mp *Multiplex) handleIncoming() { case msch.dataIn <- b: case <-msch.readCancel: // the user has canceled reading. walk away. - mp.putBuffer(b) + mp.putBufferInbound(b) case <-recvTimeout.C: - mp.putBuffer(b) + mp.putBufferInbound(b) log.Warnf("timed out receiving message into stream queue.") // Do not do this asynchronously. Otherwise, we // could drop a message, then receive a message, @@ -487,7 +449,7 @@ func (mp *Multiplex) handleIncoming() { msch.Reset() continue case <-mp.shutdown: - mp.putBuffer(b) + mp.putBufferInbound(b) return } if !recvTimeout.Stop() { @@ -555,7 +517,7 @@ func (mp *Multiplex) readNext() ([]byte, error) { return nil, nil } - buf, err := mp.getBuffer(int(l)) + buf, err := mp.getBufferInbound(int(l)) if err != nil { return nil, err } @@ -567,17 +529,43 @@ func (mp *Multiplex) readNext() ([]byte, error) { return buf[:n], nil } -func (mp *Multiplex) getBuffer(length int) ([]byte, error) { - if err := mp.memoryManager.ReserveMemory(length, 128); err != nil { - // Kill the connection when we can't reserve memory. - // Since mplex doesn't support backpressure, there's not a lot we can do. - mp.closeNoWait() - return nil, err +func (mp *Multiplex) getBufferInbound(length int) ([]byte, error) { + select { + case mp.bufIn <- struct{}{}: + case <-mp.shutdown: + return nil, ErrShutdown } - return pool.Get(length), nil + + return mp.getBuffer(length), nil +} + +func (mp *Multiplex) getBufferOutbound(length int, timeout, cancel <-chan struct{}) ([]byte, error) { + select { + case mp.bufOut <- struct{}{}: + case <-timeout: + return nil, errTimeout + case <-cancel: + return nil, ErrStreamClosed + case <-mp.shutdown: + return nil, ErrShutdown + } + + return mp.getBuffer(length), nil +} + +func (mp *Multiplex) getBuffer(length int) []byte { + return pool.Get(length) +} + +func (mp *Multiplex) putBufferInbound(b []byte) { + mp.putBuffer(b, mp.bufIn) +} + +func (mp *Multiplex) putBufferOutbound(b []byte) { + mp.putBuffer(b, mp.bufOut) } -func (mp *Multiplex) putBuffer(slice []byte) { - mp.memoryManager.ReleaseMemory(len(slice)) +func (mp *Multiplex) putBuffer(slice []byte, putBuf chan struct{}) { + <-putBuf pool.Put(slice) } diff --git a/multiplex_test.go b/multiplex_test.go index f2a9325..1a87e1f 100644 --- a/multiplex_test.go +++ b/multiplex_test.go @@ -21,8 +21,14 @@ func init() { func TestSlowReader(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -39,34 +45,60 @@ func TestSlowReader(t *testing.T) { t.Fatal(err) } - // 100 is large enough that the buffer of the underlying connection will - // fill up. - for i := 0; i < 10000; i++ { - _, err = sa.Write(mes) - if err != nil { - break + res1 := make(chan error, 1) + res2 := make(chan error, 1) + + const msgs = 1000 + go func() { + defer sa.Close() + var err error + for i := 0; i < msgs; i++ { + _, err = sa.Write(mes) + if err != nil { + break + } } - } - if err == nil { - t.Fatal("wrote too many messages") - } + res1 <- err + }() - // There's a race between reading this stream and processing the reset - // so we have to read enough off to drain the queue. - for i := 0; i < 8; i++ { - _, err = sb.Read(mes) - if err != nil { - return + go func() { + defer sb.Close() + buf := make([]byte, len(mes)) + time.Sleep(time.Second) + var err error + for i := 0; i < msgs; i++ { + time.Sleep(time.Millisecond) + _, err = sb.Read(buf) + if err != nil { + break + } } + res2 <- err + }() + + err = <-res1 + if err != nil { + t.Fatal(err) + } + + err = <-res2 + if err != nil { + t.Fatal(err) } - t.Fatal("stream should have been reset") } func TestBasicStreams(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } mes := []byte("Hello world") done := make(chan struct{}) @@ -111,7 +143,10 @@ func TestBasicStreams(t *testing.T) { func TestOpenStreamDeadline(t *testing.T) { a, _ := net.Pipe() - mp := NewMultiplex(a, false, nil) + mp, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() @@ -138,8 +173,15 @@ func TestOpenStreamDeadline(t *testing.T) { func TestWriteAfterClose(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) mes := []byte("Hello world") @@ -193,8 +235,15 @@ func TestWriteAfterClose(t *testing.T) { func TestEcho(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } mes := make([]byte, 40960) rand.Read(mes) @@ -239,8 +288,15 @@ func TestEcho(t *testing.T) { func TestFullClose(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } mes := make([]byte, 40960) rand.Read(mes) @@ -286,8 +342,15 @@ func TestFullClose(t *testing.T) { func TestHalfClose(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } wait := make(chan struct{}) mes := make([]byte, 40960) @@ -353,8 +416,15 @@ func TestFuzzCloseConnection(t *testing.T) { a, b := net.Pipe() for i := 0; i < 1000; i++ { - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } go mpa.Close() go mpa.Close() @@ -366,10 +436,17 @@ func TestFuzzCloseConnection(t *testing.T) { func TestClosing(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } - _, err := mpb.NewStream(context.Background()) + _, err = mpb.NewStream(context.Background()) if err != nil { t.Fatal(err) } @@ -397,13 +474,20 @@ func TestClosing(t *testing.T) { func TestCloseChan(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } ctx, cancel := context.WithTimeout(context.Background(), time.Second*1) defer cancel() - _, err := mpb.NewStream(ctx) + _, err = mpb.NewStream(ctx) if err != nil { t.Fatal(err) } @@ -433,8 +517,15 @@ func TestCloseChan(t *testing.T) { func TestReset(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -488,8 +579,15 @@ func TestReset(t *testing.T) { func TestCancelRead(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -549,8 +647,15 @@ func TestCancelRead(t *testing.T) { func TestCancelWrite(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -623,8 +728,15 @@ func TestCancelWrite(t *testing.T) { func TestCancelReadAfterWrite(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -666,8 +778,15 @@ func TestCancelReadAfterWrite(t *testing.T) { func TestResetAfterEOF(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -701,8 +820,15 @@ func TestResetAfterEOF(t *testing.T) { func TestOpenAfterClose(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } sa, err := mpa.NewStream(context.Background()) if err != nil { @@ -734,8 +860,15 @@ func TestOpenAfterClose(t *testing.T) { func TestDeadline(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -760,8 +893,15 @@ func TestDeadline(t *testing.T) { func TestReadAfterClose(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() @@ -795,8 +935,15 @@ func TestFuzzCloseStream(t *testing.T) { a, b := net.Pipe() - mpa := NewMultiplex(a, false, nil) - mpb := NewMultiplex(b, true, nil) + mpa, err := NewMultiplex(a, false, nil) + if err != nil { + t.Fatal(err) + } + + mpb, err := NewMultiplex(b, true, nil) + if err != nil { + t.Fatal(err) + } defer mpa.Close() defer mpb.Close() diff --git a/stream.go b/stream.go index 4d7007d..935c3e2 100644 --- a/stream.go +++ b/stream.go @@ -7,7 +7,6 @@ import ( "sync" "time" - pool "github.com/libp2p/go-buffer-pool" "go.uber.org/multierr" ) @@ -87,7 +86,7 @@ func (s *Stream) waitForData() error { func (s *Stream) returnBuffers() { if s.exbuf != nil { - pool.Put(s.exbuf) + s.mp.putBufferInbound(s.exbuf) s.exbuf = nil s.extra = nil } @@ -100,7 +99,7 @@ func (s *Stream) returnBuffers() { if read == nil { continue } - pool.Put(read) + s.mp.putBufferInbound(read) default: return } @@ -128,7 +127,7 @@ func (s *Stream) Read(b []byte) (int, error) { s.extra = s.extra[read:] } else { if s.exbuf != nil { - pool.Put(s.exbuf) + s.mp.putBufferInbound(s.exbuf) } s.extra = nil s.exbuf = nil