Merge pull request #91 from lightninglabs/limbo-balance

lndmon: additional monitoring for closing channel balances

collectors/channels_collector.go

@@ -13,8 +13,10 @@ import (
 
 // ChannelsCollector is a collector that keeps track of channel information.
 type ChannelsCollector struct {
-	channelBalanceDesc        *prometheus.Desc
-	pendingChannelBalanceDesc *prometheus.Desc
+	channelBalanceDesc           *prometheus.Desc
+	pendingChannelBalanceDesc    *prometheus.Desc
+	pendingForceCloseBalanceDesc *prometheus.Desc
+	waitingCloseBalanceDesc      *prometheus.Desc
 
 	incomingChanSatDesc *prometheus.Desc
 	outgoingChanSatDesc *prometheus.Desc
@@ -70,7 +72,16 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
 			"total balance of all pending channels in satoshis",
 			nil, nil,
 		),
-
+		pendingForceCloseBalanceDesc: prometheus.NewDesc(
+			"lnd_channels_pending_force_close_balance_sat",
+			"force closed channel balances in satoshis",
+			append(labels, "blocks_until_maturity"), nil,
+		),
+		waitingCloseBalanceDesc: prometheus.NewDesc(
+			"lnd_channels_waiting_close_balance_sat",
+			"waiting to close channel balances in satoshis",
+			labels, nil,
+		),
 		incomingChanSatDesc: prometheus.NewDesc(
 			"lnd_channels_bandwidth_incoming_sat",
 			"total available incoming channel bandwidth within this channel",
@@ -177,6 +188,8 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
 func (c *ChannelsCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.channelBalanceDesc
 	ch <- c.pendingChannelBalanceDesc
+	ch <- c.pendingForceCloseBalanceDesc
+	ch <- c.waitingCloseBalanceDesc
 
 	ch <- c.incomingChanSatDesc
 	ch <- c.outgoingChanSatDesc
@@ -204,6 +217,22 @@ func (c *ChannelsCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.inboundFee
 }
 
+func anchorStateToString(state lndclient.ForceCloseAnchorState) string {
+	switch state {
+	case lndclient.ForceCloseAnchorStateLimbo:
+		return "Limbo"
+
+	case lndclient.ForceCloseAnchorStateRecovered:
+		return "Recovered"
+
+	case lndclient.ForceCloseAnchorStateLost:
+		return "Lost"
+
+	default:
+		return "Unknown"
+	}
+}
+
 // Collect is called by the Prometheus registry when collecting metrics.
 //
 // NOTE: Part of the prometheus.Collector interface.
@@ -379,6 +408,33 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
 		"waiting_close",
 	)
 
+	for _, forceClose := range pendingChannelsResp.PendingForceClose {
+		// Labels are: "chan_id", "status", "initiator", "peer",
+		// "blocks_until_maturity". We'll use status to hold the anchor
+		// state.
+		ch <- prometheus.MustNewConstMetric(
+			c.pendingForceCloseBalanceDesc, prometheus.GaugeValue,
+			float64(forceClose.RecoveredBalance),
+			forceClose.ChannelPoint.String(),
+			anchorStateToString(forceClose.AnchorState),
+			forceClose.ChannelInitiator.String(),
+			forceClose.PubKeyBytes.String(),
+			fmt.Sprintf("%d", forceClose.BlocksUntilMaturity),
+		)
+	}
+
+	for _, waitingClose := range pendingChannelsResp.WaitingClose {
+		// Labels are: "chan_id", "status", "initiator", "peer".
+		ch <- prometheus.MustNewConstMetric(
+			c.waitingCloseBalanceDesc, prometheus.GaugeValue,
+			float64(waitingClose.LocalBalance),
+			waitingClose.ChannelPoint.String(),
+			waitingClose.ChanStatusFlags,
+			waitingClose.ChannelInitiator.String(),
+			waitingClose.PubKeyBytes.String(),
+		)
+	}
+
 	// Get the list of closed channels.
 	closedChannelsResp, err := c.lnd.ClosedChannels(context.Background())
 	if err != nil {

go.mod

@@ -5,8 +5,8 @@ require (
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/jrick/logrotate v1.0.0
-	github.com/lightninglabs/lndclient v0.16.0-1
-	github.com/lightningnetwork/lnd v0.15.0-beta.rc6.0.20220825081330-cf9a9864cf25
+	github.com/lightninglabs/lndclient v0.16.0-5
+	github.com/lightningnetwork/lnd v0.15.0-beta.rc6.0.20221104092723-22fec76339a7
 	github.com/prometheus/client_golang v1.11.0
 	github.com/stretchr/testify v1.7.1
 )
@@ -16,14 +16,14 @@ require (
 	github.com/aead/siphash v1.0.1 // indirect
 	github.com/andybalholm/brotli v1.0.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btcd v0.23.1 // indirect
+	github.com/btcsuite/btcd v0.23.3 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
 	github.com/btcsuite/btcd/btcutil/psbt v1.1.5 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
-	github.com/btcsuite/btcwallet v0.15.2-0.20220804001213-5aafe4789850 // indirect
-	github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3 // indirect
+	github.com/btcsuite/btcwallet v0.16.1 // indirect
+	github.com/btcsuite/btcwallet/wallet/txauthor v1.3.2 // indirect
 	github.com/btcsuite/btcwallet/wallet/txrules v1.2.0 // indirect
-	github.com/btcsuite/btcwallet/wallet/txsizes v1.1.0 // indirect
+	github.com/btcsuite/btcwallet/wallet/txsizes v1.2.3 // indirect
 	github.com/btcsuite/btcwallet/walletdb v1.4.0 // indirect
 	github.com/btcsuite/btcwallet/wtxmgr v1.5.0 // indirect
 	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
@@ -76,7 +76,7 @@ require (
 	github.com/lightningnetwork/lnd/queue v1.1.0 // indirect
 	github.com/lightningnetwork/lnd/ticker v1.1.0 // indirect
 	github.com/lightningnetwork/lnd/tlv v1.0.3 // indirect
-	github.com/lightningnetwork/lnd/tor v1.0.2 // indirect
+	github.com/lightningnetwork/lnd/tor v1.1.0 // indirect
 	github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mholt/archiver/v3 v3.5.0 // indirect

go.sum

@@ -75,8 +75,9 @@ github.com/btcsuite/btcd v0.22.0-beta.0.20220204213055-eaf0459ff879/go.mod h1:os
 github.com/btcsuite/btcd v0.22.0-beta.0.20220207191057-4dc4ff7963b4/go.mod h1:7alexyj/lHlOtr2PJK7L/+HDJZpcGDn/pAU98r7DY08=
 github.com/btcsuite/btcd v0.22.0-beta.0.20220316175102-8d5c75c28923/go.mod h1:taIcYprAW2g6Z9S0gGUxyR+zDwimyDMK5ePOX+iJ2ds=
 github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=
-github.com/btcsuite/btcd v0.23.1 h1:IB8cVQcC2X5mHbnfirLG5IZnkWYNTPlLZVrxUYSotbE=
 github.com/btcsuite/btcd v0.23.1/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=
+github.com/btcsuite/btcd v0.23.3 h1:4KH/JKy9WiCd+iUS9Mu0Zp7Dnj17TGdKrg9xc/FGj24=
+github.com/btcsuite/btcd v0.23.3/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=
 github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA=
 github.com/btcsuite/btcd/btcec/v2 v2.1.1/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
 github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
@@ -96,14 +97,17 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtyd
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcwallet v0.15.2-0.20220804001213-5aafe4789850 h1:yhtY53u3I6qT0yMJg35Yfa+yCVhvoqWWKTB2D2GBNJE=
-github.com/btcsuite/btcwallet v0.15.2-0.20220804001213-5aafe4789850/go.mod h1:7OFsQ8ypiRwmr67hE0z98uXgJgXGAihE79jCib9x6ag=
-github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3 h1:M2yr5UlULvpqtxUqpMxTME/pA92Z9cpqeyvAFk9lAg0=
+github.com/btcsuite/btcwallet v0.16.1 h1:nD8qXJeAU8c7a0Jlx5jwI2ufbf/9ouy29XGapRLTmos=
+github.com/btcsuite/btcwallet v0.16.1/go.mod h1:NCO8+5rIcbUm5CtVNSQM0xrtK4iYprlyuvpGzhkejaM=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3/go.mod h1:T2xSiKGpUkSLCh68aF+FMXmKK9mFqNdHl9VaqOr+JjU=
+github.com/btcsuite/btcwallet/wallet/txauthor v1.3.2 h1:etuLgGEojecsDOYTII8rYiGHjGyV5xTqsXi+ZQ715UU=
+github.com/btcsuite/btcwallet/wallet/txauthor v1.3.2/go.mod h1:Zpk/LOb2sKqwP2lmHjaZT9AdaKsHPSbNLm2Uql5IQ/0=
 github.com/btcsuite/btcwallet/wallet/txrules v1.2.0 h1:BtEN5Empw62/RVnZ0VcJaVtVlBijnLlJY+dwjAye2Bg=
 github.com/btcsuite/btcwallet/wallet/txrules v1.2.0/go.mod h1:AtkqiL7ccKWxuLYtZm8Bu8G6q82w4yIZdgq6riy60z0=
-github.com/btcsuite/btcwallet/wallet/txsizes v1.1.0 h1:wZnOolEAeNOHzHTnznw/wQv+j35ftCIokNrnOTOU5o8=
 github.com/btcsuite/btcwallet/wallet/txsizes v1.1.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs=
+github.com/btcsuite/btcwallet/wallet/txsizes v1.2.2/go.mod h1:q08Rms52VyWyXcp5zDc4tdFRKkFgNsMQrv3/LvE1448=
+github.com/btcsuite/btcwallet/wallet/txsizes v1.2.3 h1:PszOub7iXVYbtGybym5TGCp9Dv1h1iX4rIC3HICZGLg=
+github.com/btcsuite/btcwallet/wallet/txsizes v1.2.3/go.mod h1:q08Rms52VyWyXcp5zDc4tdFRKkFgNsMQrv3/LvE1448=
 github.com/btcsuite/btcwallet/walletdb v1.3.5/go.mod h1:oJDxAEUHVtnmIIBaa22wSBPTVcs6hUp5NKWmI8xDwwU=
 github.com/btcsuite/btcwallet/walletdb v1.3.6-0.20210803004036-eebed51155ec/go.mod h1:oJDxAEUHVtnmIIBaa22wSBPTVcs6hUp5NKWmI8xDwwU=
 github.com/btcsuite/btcwallet/walletdb v1.4.0 h1:/C5JRF+dTuE2CNMCO/or5N8epsrhmSM4710uBQoYPTQ=
@@ -495,15 +499,15 @@ github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg=
 github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf h1:HZKvJUHlcXI/f/O0Avg7t8sqkPo78HFzjmeYFl6DPnc=
 github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf/go.mod h1:vxmQPeIQxPf6Jf9rM8R+B4rKBqLA2AjttNxkFBL2Plk=
-github.com/lightninglabs/lndclient v0.16.0-1 h1:DRnCRjYrGrCmE9HTVcW9aruqeO/GkPg8QV+ClMFJMDE=
-github.com/lightninglabs/lndclient v0.16.0-1/go.mod h1:G4D4/rcEnbdl8I0cTUO7oaJgEGa8WrplMv2IuPy1xu0=
+github.com/lightninglabs/lndclient v0.16.0-5 h1:Davob0ypwZQVOaNAfjmSxR7/e/6QYV/QZHdEObYm6E8=
+github.com/lightninglabs/lndclient v0.16.0-5/go.mod h1:W5vhVyBd8Qr+fjY/lYwZzQo1ZlybVkrZLkXLVdguoKs=
 github.com/lightninglabs/neutrino v0.14.2 h1:yrnZUCYMZ5ECtXhgDrzqPq2oX8awoAN2D/cgCewJcCo=
 github.com/lightninglabs/neutrino v0.14.2/go.mod h1:OICUeTCn+4Tu27YRJIpWvvqySxx4oH4vgdP33Sw9RDc=
 github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display/go.mod h1:2oKOBU042GKFHrdbgGiKax4xVrFiZu51lhacUZQ9MnE=
 github.com/lightningnetwork/lightning-onion v1.0.2-0.20220211021909-bb84a1ccb0c5 h1:TkKwqFcQTGYoI+VEqyxA8rxpCin8qDaYX0AfVRinT3k=
 github.com/lightningnetwork/lightning-onion v1.0.2-0.20220211021909-bb84a1ccb0c5/go.mod h1:7dDx73ApjEZA0kcknI799m2O5kkpfg4/gr7N092ojNo=
-github.com/lightningnetwork/lnd v0.15.0-beta.rc6.0.20220825081330-cf9a9864cf25 h1:2nMMQRjS8Oi/0VbJL3Vn8aDeQC5WLDbfZYzsMxIWNcY=
-github.com/lightningnetwork/lnd v0.15.0-beta.rc6.0.20220825081330-cf9a9864cf25/go.mod h1:3C62CV5fL8F6DK8VHZJJSgNGFzXxb3OPcyCz+McrjSU=
+github.com/lightningnetwork/lnd v0.15.0-beta.rc6.0.20221104092723-22fec76339a7 h1:zeswbfu7P7/77vU1HZDpRn5BxDhcXnvT1rUbaHci9GA=
+github.com/lightningnetwork/lnd v0.15.0-beta.rc6.0.20221104092723-22fec76339a7/go.mod h1:V43l3Kqyzqe7jmDVTGnuFK7I1ajXSkzV72OjYInUrB8=
 github.com/lightningnetwork/lnd/cert v1.1.1/go.mod h1:1P46svkkd73oSoeI4zjkVKgZNwGq8bkGuPR8z+5vQUs=
 github.com/lightningnetwork/lnd/clock v1.0.1/go.mod h1:KnQudQ6w0IAMZi1SgvecLZQZ43ra2vpDNj7H/aasemg=
 github.com/lightningnetwork/lnd/clock v1.1.0 h1:/yfVAwtPmdx45aQBoXQImeY7sOIEr7IXlImRMBOZ7GQ=
@@ -523,8 +527,8 @@ github.com/lightningnetwork/lnd/tlv v1.0.2/go.mod h1:fICAfsqk1IOsC1J7G9IdsWX1EqW
 github.com/lightningnetwork/lnd/tlv v1.0.3 h1:0xBZcPuXagP6f7TY/RnLNR4igE21ov6qUdTr5NyvhhI=
 github.com/lightningnetwork/lnd/tlv v1.0.3/go.mod h1:dzR/aZetBri+ZY/fHbwV06fNn/3UID6htQzbHfREFdo=
 github.com/lightningnetwork/lnd/tor v1.0.0/go.mod h1:RDtaAdwfAm+ONuPYwUhNIH1RAvKPv+75lHPOegUcz64=
-github.com/lightningnetwork/lnd/tor v1.0.2 h1:GlumRkKdzXCX0AIvIi2UXKpeY1Q4RT7Lz/CfGpKSLrU=
-github.com/lightningnetwork/lnd/tor v1.0.2/go.mod h1:RDtaAdwfAm+ONuPYwUhNIH1RAvKPv+75lHPOegUcz64=
+github.com/lightningnetwork/lnd/tor v1.1.0 h1:iXO7fSzjxTI+p88KmtpbuyuRJeNfgtpl9QeaAliILXE=
+github.com/lightningnetwork/lnd/tor v1.1.0/go.mod h1:RDtaAdwfAm+ONuPYwUhNIH1RAvKPv+75lHPOegUcz64=
 github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 h1:sjOGyegMIhvgfq5oaue6Td+hxZuf3tDC8lAPrFldqFw=
 github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796/go.mod h1:3p7ZTf9V1sNPI5H8P3NkTFF4LuwMdPl2DodF60qAKqY=
 github.com/ltcsuite/ltcutil v0.0.0-20181217130922-17f3b04680b6/go.mod h1:8Vg/LTOO0KYa/vlHWJ6XZAevPQThGH5sufO0Hrou/lA=