From fbdb5a3b30a71aab84adf2ed7efb142e35b00da0 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Thu, 2 Jul 2020 12:10:08 +0200 Subject: [PATCH] collectors: add inbound fee metric --- .gitignore | 2 + collectors/channel_collector_test.go | 88 +++++++++++++ collectors/channels_collector.go | 180 +++++++++++++++++++++++++++ go.mod | 1 + go.sum | 3 + 5 files changed, 274 insertions(+) create mode 100644 collectors/channel_collector_test.go diff --git a/.gitignore b/.gitignore index b72be3c..ecaccff 100755 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ user-config/* !user-config/.gitkeep nginx/etc/ssl/* nginx/etc/.htpasswd + +cmd/lndmon/lndmon \ No newline at end of file diff --git a/collectors/channel_collector_test.go b/collectors/channel_collector_test.go new file mode 100644 index 0000000..54040a8 --- /dev/null +++ b/collectors/channel_collector_test.go @@ -0,0 +1,88 @@ +package collectors + +import ( + "testing" + + "github.com/btcsuite/btcutil" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/stretchr/testify/require" +) + +var ( + remotePolicies = map[uint64]*lnrpc.RoutingPolicy{ + 1: { + FeeBaseMsat: 20000, + FeeRateMilliMsat: 10000, + }, + 2: { + FeeBaseMsat: 250000, + FeeRateMilliMsat: 6000, + }, + } + + remoteBalances = map[uint64]btcutil.Amount{ + 1: 10000, + 2: 10000, + } +) + +// TestGetInboundFee tests the specific-fee based inbound fee calculation. +func TestGetInboundFee(t *testing.T) { + testCases := []struct { + name string + amt btcutil.Amount + expectedFee btcutil.Amount + expectNoLiquidity bool + }{ + { + name: "single channel use all", + amt: 10000, + expectedFee: 120, + }, + { + name: "single channel partially used", + amt: 5000, + expectedFee: 70, + }, + { + name: "not enough", + amt: 25000, + expectNoLiquidity: true, + }, + { + name: "two channels use all", + amt: 20000, + expectedFee: 120 + 310, + }, + { + name: "two channels partially used", + amt: 15000, + expectedFee: 120 + 280, + }, + } + + for _, test := range testCases { + test := test + + t.Run(test.name, func(t *testing.T) { + testGetInboundFee( + t, test.amt, test.expectedFee, + test.expectNoLiquidity, + ) + }) + } +} + +func testGetInboundFee(t *testing.T, amt, expectedFee btcutil.Amount, + expectNoLiquidity bool) { + + fee := approximateInboundFee(amt, remotePolicies, remoteBalances) + + if expectNoLiquidity { + require.Nil(t, fee, "expected no liquidity") + return + } + + require.NotNil(t, fee, "expected routing to be possible") + require.Equal(t, expectedFee, *fee) +} diff --git a/collectors/channels_collector.go b/collectors/channels_collector.go index 7734b13..0d83358 100644 --- a/collectors/channels_collector.go +++ b/collectors/channels_collector.go @@ -2,8 +2,10 @@ package collectors import ( "context" + "fmt" "strconv" + "github.com/btcsuite/btcutil" "github.com/lightningnetwork/lnd/lnrpc" "github.com/prometheus/client_golang/prometheus" ) @@ -34,6 +36,10 @@ type ChannelsCollector struct { commitWeightDesc *prometheus.Desc commitFeeDesc *prometheus.Desc + // inboundFee is a metric that reflects the fee paid by senders on the + // last hop towards this node. + inboundFee *prometheus.Desc + lnd lnrpc.LightningClient } @@ -135,6 +141,13 @@ func NewChannelsCollector(lnd lnrpc.LightningClient) *ChannelsCollector { labels, nil, ), + // Use labels for the inbound fee for various amounts. + inboundFee: prometheus.NewDesc( + "inbound_fee", + "fee charged for forwarding to this node", + []string{"amount"}, nil, + ), + lnd: lnd, } } @@ -170,6 +183,8 @@ func (c *ChannelsCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.feePerKwDesc ch <- c.commitWeightDesc ch <- c.commitFeeDesc + + ch <- c.inboundFee } // Collect is called by the Prometheus registry when collecting metrics. @@ -248,10 +263,18 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { return "false" } + remoteBalances := make(map[uint64]btcutil.Amount) for _, channel := range listChannelsResp.Channels { status := statusLabel(channel) initiator := initiatorLabel(channel) + // Only record balances for channels that are usable. + if channel.Active { + remoteBalances[channel.ChanId] = btcutil.Amount( + channel.RemoteBalance, + ) + } + ch <- prometheus.MustNewConstMetric( c.incomingChanSatDesc, prometheus.GaugeValue, float64(channel.RemoteBalance), @@ -317,6 +340,163 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { ) } } + + // Get all remote policies + remotePolicies, err := c.getRemotePolicies(getInfoResp.IdentityPubkey) + if err != nil { + channelLogger.Error(err) + return + } + + // Export the inbound fee metric for a series of amounts. + var receiveAmt btcutil.Amount = 100000 + for { + // For each fee amount, we'll approximate the total routing fee + // that needs to be paid to pay us. + inboundFee := approximateInboundFee( + receiveAmt, remotePolicies, remoteBalances, + ) + if inboundFee == nil { + break + } + + // Calculate the fee proportional to the amount to receive. + proportionalFee := float64(*inboundFee) / float64(receiveAmt) + + ch <- prometheus.MustNewConstMetric( + c.inboundFee, prometheus.GaugeValue, + proportionalFee, + receiveAmt.String(), + ) + + // Continue the series with double the amount. + receiveAmt *= 2 + } +} + +// approximateInboundFee calculates to forward fee for a specific amount charged by the +// last hop before this node. +func approximateInboundFee(amt btcutil.Amount, remotePolicies map[uint64]*lnrpc.RoutingPolicy, + remoteBalances map[uint64]btcutil.Amount) *btcutil.Amount { + + var fee btcutil.Amount + + // Copy the remote balances so they can be decreased as we find shards. + remainingBalances := make(map[uint64]btcutil.Amount) + for ch, balance := range remoteBalances { + remainingBalances[ch] = balance + } + + // Assume a perfect mpp splitting algorithm that knows exactly how much + // can be sent through each channel. This is a simplification, because + // in reality senders need to trial and error to find a shard amount + // that works. + // + // We'll keep iterating through all channels until we've covered the + // total amount. Each iteration, the best channel for that shard is + // selected based on the specific fee. + amountRemaining := amt + for amountRemaining > 0 { + var ( + bestChan uint64 + bestSpecificFee float64 + bestAmount btcutil.Amount + bestFee btcutil.Amount + ) + + // Find the best channel to send the amount or a part of the + // amount. + for ch, balance := range remainingBalances { + // Skip channels without remote balance. + if balance == 0 { + continue + } + + policy, ok := remotePolicies[ch] + if !ok { + continue + } + + // Cap at the maximum receive amount for this channel. + amountToSend := amountRemaining + if amountToSend > balance { + amountToSend = balance + } + + // Calculate fee for this amount to send. + fee := btcutil.Amount( + policy.FeeBaseMsat/1000 + + int64(amountToSend)*policy.FeeRateMilliMsat/1000000, + ) + + // Calculate the specific fee for this amount, being the + // fee per sat sent. + specificFee := float64(fee) / float64(amountToSend) + + // Select the best channel for this shard based on the + // lowest specific fee. + if bestChan == 0 || bestSpecificFee > specificFee { + bestChan = ch + bestSpecificFee = specificFee + bestAmount = amountToSend + bestFee = fee + } + } + + // No liquidity to send the full amount, break. + if bestChan == 0 { + return nil + } + + amountRemaining -= bestAmount + fee += bestFee + remainingBalances[bestChan] -= bestAmount + } + + return &fee +} + +// getRemotePolicies gets all the remote policies for enabled channels of this +// node's peers. +func (c *ChannelsCollector) getRemotePolicies(pubkey string) ( + map[uint64]*lnrpc.RoutingPolicy, error) { + + nodeInfoResp, err := c.lnd.GetNodeInfo( + context.Background(), &lnrpc.NodeInfoRequest{ + IncludeChannels: true, + PubKey: pubkey, + }, + ) + if err != nil { + return nil, err + } + + policies := make(map[uint64]*lnrpc.RoutingPolicy) + for _, i := range nodeInfoResp.Channels { + var policy *lnrpc.RoutingPolicy + switch { + case i.Node1Pub == pubkey: + if i.Node2Policy != nil { + policy = i.Node2Policy + } + + case i.Node2Pub == pubkey: + if i.Node1Policy != nil { + policy = i.Node1Policy + } + + default: + return nil, fmt.Errorf("pubkey not in node info channels") + } + + // Only record policies for peers that have this channel + // enabled. + if !policy.Disabled { + policies[i.ChannelId] = policy + } + } + + return policies, nil } func init() { diff --git a/go.mod b/go.mod index e4edf2b..8e9567f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/lightninglabs/loop v0.2.4-alpha.0.20191116024025-539d6ed9e3e8 github.com/lightningnetwork/lnd v0.8.0-beta-rc3.0.20191115230031-4d7a151b4763 github.com/prometheus/client_golang v0.9.3 + github.com/stretchr/testify v1.2.2 ) go 1.13 diff --git a/go.sum b/go.sum index 854fee6..22ec0c7 100644 --- a/go.sum +++ b/go.sum @@ -171,6 +171,7 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= @@ -192,7 +193,9 @@ github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02 h1:tcJ6OjwOMvExLlzrAVZute09ocAGa7KqOON60++Gz4E= github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02/go.mod h1:tHlrkM198S068ZqfrO6S8HsoJq2bF3ETfTL+kt4tInY=