diff --git a/exchange/bitswap/bitswap.go b/exchange/bitswap/bitswap.go
index e1fb20de4fd..60988fc839c 100644
--- a/exchange/bitswap/bitswap.go
+++ b/exchange/bitswap/bitswap.go
@@ -393,6 +393,7 @@ func (bs *Bitswap) updateReceiveCounters(b blocks.Block) error {
 // Connected/Disconnected warns bitswap about peer connections
 func (bs *Bitswap) PeerConnected(p peer.ID) {
 	bs.wm.Connected(p)
+	bs.engine.PeerConnected(p)
 }
 
 // Connected/Disconnected warns bitswap about peer connections
diff --git a/exchange/bitswap/decision/engine.go b/exchange/bitswap/decision/engine.go
index d494554d053..99e37082a70 100644
--- a/exchange/bitswap/decision/engine.go
+++ b/exchange/bitswap/decision/engine.go
@@ -295,8 +295,32 @@ func (e *Engine) MessageSent(p peer.ID, m bsmsg.BitSwapMessage) error {
 	return nil
 }
 
+func (e *Engine) PeerConnected(p peer.ID) {
+	e.lock.Lock()
+	l, ok := e.ledgerMap[p]
+	if !ok {
+		l = newLedger(p)
+		e.ledgerMap[p] = l
+	}
+	l.lk.Lock()
+	l.ref++
+	l.lk.Unlock()
+	e.lock.Unlock()
+}
+
 func (e *Engine) PeerDisconnected(p peer.ID) {
-	// TODO: release ledger
+	e.lock.Lock()
+	defer e.lock.Unlock()
+	l, ok := e.ledgerMap[p]
+	if !ok {
+		return
+	}
+	l.lk.Lock()
+	l.ref--
+	if l.ref <= 0 {
+		delete(e.ledgerMap, p)
+	}
+	l.lk.Unlock()
 }
 
 func (e *Engine) numBytesSentTo(p peer.ID) uint64 {
diff --git a/exchange/bitswap/decision/ledger.go b/exchange/bitswap/decision/ledger.go
index db1f242877a..202daba86bb 100644
--- a/exchange/bitswap/decision/ledger.go
+++ b/exchange/bitswap/decision/ledger.go
@@ -43,6 +43,10 @@ type ledger struct {
 	// to a given peer
 	sentToPeer map[string]time.Time
 
+	// ref is the reference count for this ledger, its used to ensure we
+	// don't drop the reference to this ledger in multi-connection scenarios
+	ref int
+
 	lk sync.Mutex
 }