Ability to filter reported channels

README.md

@@ -26,18 +26,16 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
+   --allowedentropy value      allowed entropy in bits for channel balances (default: 64)
    --apikey value              api key
-   --rpcserver value           host:port of ln daemon (default: "localhost:10009")
+   --interval value            interval to poll - 10s, 1m, 10m or 1h (default: "10s")
    --lnddir value              path to lnd's base directory (default: "/home/user/.lnd")
-   --tlscertpath value         path to TLS certificate (default: "/home/user/.lnd/tls.cert")
-   --chain value, -c value     the chain lnd is running on e.g. bitcoin (default: "bitcoin")
-   --network value, -n value   the network lnd is running on e.g. mainnet, testnet, etc. (default: "mainnet")
    --macaroonpath value        path to macaroon file
-   --allowedentropy value      allowed entropy in bits for channel balances (default: 64)
-   --interval value            interval to poll - 10s, 1m, 10m or 1h (default: "10s")
    --private                   report private data as well (default: false)
    --preferipv4                If you have the choice between IPv6 and IPv4 prefer IPv4 (default: false)
-   --channels-whitelist value  Path to file containing a whitelist of channels
+   --rpcserver value           host:port of ln daemon (default: "localhost:10009")
+   --tlscertpath value         path to TLS certificate (default: "/home/user/.lnd/tls.cert")
+   --channel-whitelist value   Path to file containing a whitelist of channels
    --verbosity value           log level for V logs (default: 0)
    --help, -h                  show help
    --version, -v               print the version
@@ -132,8 +130,13 @@ docker run -v /tmp:/tmp -e API_KEY=changeme ghcr.io/bolt-observer/agent:v0.0.35
 
 ## Filtering on agent side
 
-You can limit what channnels are reported using `--channels-whitelist` option. It specifies a path of a local file to be used as a whitelist of what channels to report.
-`--channels-whitelist` implies `--private` (and specifying both is a mistake). The format of the file is simple:
+You can limit what channnels are reported using `--channel-whitelist` option. It specifies a local file path to be used as a whitelist of what channels to report.
+When adding `--private` to `--channel-whitelist` this means every private channel AND whatever is listed in the file. There is also `--public` to allow all public channels.
+Using the options without `--channel-whitelist` makes no sense since by default all public channels are reported however it has to be explicit with `--channel-whitelist` in order
+to automatically allow all public channels (beside what is allowed through the file).
+If you set `--private` and `--public` then no matter what you add to the `--channel-whitelist` file everything will be reported.
+
+The format of the file is simple:
 
 ```
 # Comments start with a # character

channelchecker/channelchecker.go

@@ -198,7 +198,7 @@ func (c *ChannelChecker) getChannelList(
 			continue
 		}
 
-		if !filter.AllowChanId(channel.ChanId) && !filter.AllowPubKey(channel.RemotePubkey) {
+		if !filter.AllowChanId(channel.ChanId) && !filter.AllowPubKey(channel.RemotePubkey) || !filter.AllowSpecial(channel.Private) {
 			glog.V(3).Infof("Filtering channel %v", channel.ChanId)
 			continue
 		}

cmd/balance-agent/main.go

@@ -39,6 +39,7 @@ const (
 	defaultChainSubDir      = "chain"
 	defaultTLSCertFilename  = "tls.cert"
 	defaultMacaroonFilename = "readonly.macaroon"
+	whitelist               = "channel-whitelist"
 )
 
 var (
@@ -207,59 +208,58 @@ func getApp() *cli.App {
 	app.Version = GitRevision
 
 	app.Flags = []cli.Flag{
+		&cli.IntFlag{
+			Name:  "allowedentropy",
+			Usage: "allowed entropy in bits for channel balances",
+			Value: 64,
+		},
 		&cli.StringFlag{
 			Name:  "apikey",
 			Value: "",
 			Usage: "api key",
 		},
 		&cli.StringFlag{
-			Name:  "rpcserver",
-			Value: defaultRPCHostPort,
-			Usage: "host:port of ln daemon",
+			Name:  "interval",
+			Usage: "interval to poll - 10s, 1m, 10m or 1h",
+			Value: "10s",
 		},
 		&cli.StringFlag{
 			Name:  "lnddir",
 			Value: defaultLndDir,
 			Usage: "path to lnd's base directory",
 		},
-		&cli.StringFlag{
-			Name:  "tlscertpath",
-			Value: defaultTLSCertPath,
-			Usage: "path to TLS certificate",
-		},
-		&cli.StringFlag{
-			Name:  "chain, c",
-			Usage: "the chain lnd is running on e.g. bitcoin",
-			Value: "bitcoin",
-		},
-		&cli.StringFlag{
-			Name: "network, n",
-			Usage: "the network lnd is running on e.g. mainnet, " +
-				"testnet, etc.",
-			Value: "mainnet",
-		},
 		&cli.StringFlag{
 			Name:  "macaroonpath",
 			Usage: "path to macaroon file",
 		},
-		&cli.IntFlag{
-			Name:  "allowedentropy",
-			Usage: "allowed entropy in bits for channel balances",
-			Value: 64,
-		},
-		&cli.StringFlag{
-			Name:  "interval",
-			Usage: "interval to poll - 10s, 1m, 10m or 1h",
-			Value: "10s",
-		},
 		&cli.BoolFlag{
 			Name:  "private",
 			Usage: "report private data as well (default: false)",
 		},
+		&cli.BoolFlag{
+			Name:   "public",
+			Usage:  fmt.Sprintf("report public data - useful with %s (default: false)", whitelist),
+			Hidden: true,
+		},
 		&cli.BoolFlag{
 			Name:  "preferipv4",
 			Usage: "If you have the choice between IPv6 and IPv4 prefer IPv4 (default: false)",
 		},
+		&cli.StringFlag{
+			Name:  "rpcserver",
+			Value: defaultRPCHostPort,
+			Usage: "host:port of ln daemon",
+		},
+		&cli.StringFlag{
+			Name:  "tlscertpath",
+			Value: defaultTLSCertPath,
+			Usage: "path to TLS certificate",
+		},
+		&cli.StringFlag{
+			Name:   whitelist,
+			Usage:  "Path to file containing a whitelist of channels",
+			Hidden: true,
+		},
 		&cli.BoolFlag{
 			Name:   "userest",
 			Usage:  "Use REST API when true instead of gRPC",
@@ -306,9 +306,19 @@ func getApp() *cli.App {
 			Usage:  "Ignore CLN socket",
 			Hidden: true,
 		},
+
+		&cli.StringFlag{
+			Name:   "chain, c",
+			Usage:  "the chain lnd is running on e.g. bitcoin",
+			Value:  "bitcoin",
+			Hidden: true,
+		},
 		&cli.StringFlag{
-			Name:  "channels-whitelist",
-			Usage: "Path to file containing a whitelist of channels",
+			Name: "network, n",
+			Usage: "the network lnd is running on e.g. mainnet, " +
+				"testnet, etc.",
+			Value:  "mainnet",
+			Hidden: true,
 		},
 	}
 
@@ -570,24 +580,34 @@ func checker(ctx *cli.Context) error {
 	}
 
 	if apiKey == "" && (ctx.String("url") != "" || ctx.String("nodeurl") != "") {
-		return fmt.Errorf("missing API key")
-	}
-
-	if ctx.String("channels-whitelist") != "" && ctx.Bool("private") {
-		return fmt.Errorf("channels-whitelist implies private, do not set both options")
+		// We don't return error here since we don't want glog to handle it
+		fmt.Fprintf(os.Stderr, "missing API key\n")
+		os.Exit(1)
 	}
 
 	ct := context.Background()
 
 	var err error
 
 	f, _ := filter.NewAllowAllFilter()
-	if ctx.String("channels-whitelist") != "" {
-		if _, err = os.Stat(ctx.String("channels-whitelist")); err != nil {
-			return fmt.Errorf("channels-whitelist points to non-existing file")
+	if ctx.String(whitelist) != "" {
+		if _, err = os.Stat(ctx.String(whitelist)); err != nil {
+			// We don't return error here since we don't want glog to handle it
+			fmt.Fprintf(os.Stderr, "%s points to non-existing file", whitelist)
+			os.Exit(1)
+		}
+
+		o := filter.None
+
+		if ctx.Bool("private") {
+			o |= filter.AllowAllPrivate
+		}
+
+		if ctx.Bool("public") {
+			o |= filter.AllowAllPublic
 		}
 
-		f, err = filter.NewFilterFromFile(ct, ctx.String("channels-whitelist"))
+		f, err = filter.NewFilterFromFile(ct, ctx.String(whitelist), o)
 		if err != nil {
 			return err
 		}
@@ -597,7 +617,7 @@ func checker(ctx *cli.Context) error {
 
 	url = ctx.String("url")
 	nodeurl = ctx.String("nodeurl")
-	private = ctx.Bool("private") || ctx.String("channels-whitelist") != ""
+	private = ctx.Bool("private") || ctx.String(whitelist) != ""
 
 	interval, err := getInterval(ctx, "interval")
 	if err != nil {

filter/dummy.go

@@ -16,6 +16,10 @@ func (f *AllowAllFilter) AllowChanId(id uint64) bool {
 	return true
 }
 
+func (f *AllowAllFilter) AllowSpecial(private bool) bool {
+	return true
+}
+
 type UnitTestFilter struct {
 	Filter
 }
@@ -38,3 +42,7 @@ func (u *UnitTestFilter) AddAllowPubKey(id string) {
 func (u *UnitTestFilter) AddAllowChanId(id uint64) {
 	u.chanIdWhitelist[id] = struct{}{}
 }
+
+func (f *UnitTestFilter) ChangeOptions(options Options) {
+	f.Options = options
+}

filter/filter.go

@@ -18,6 +18,7 @@ type FileFilter struct {
 	Filter
 	WhitelistFilePath string
 	Mutex             sync.Mutex
+	DefaultOptions    Options
 }
 
 func (f *FileFilter) Reload() error {
@@ -36,6 +37,7 @@ func (f *FileFilter) Reload() error {
 
 	f.nodeIdWhitelist = make(map[string]struct{})
 	f.chanIdWhitelist = make(map[uint64]struct{})
+	f.Options = f.DefaultOptions
 
 	fileScanner := bufio.NewScanner(readFile)
 	fileScanner.Split(bufio.ScanLines)
@@ -53,6 +55,10 @@ func (f *FileFilter) Reload() error {
 
 		if utils.ValidatePubkey(line) {
 			f.nodeIdWhitelist[line] = struct{}{}
+		} else if strings.ToLower(line) == "private" {
+			f.Options |= AllowAllPrivate
+		} else if strings.ToLower(line) == "public" {
+			f.Options |= AllowAllPublic
 		} else {
 			val, err := strconv.ParseUint(line, 10, 64)
 			if err != nil {
@@ -69,11 +75,13 @@ func (f *FileFilter) Reload() error {
 	return nil
 }
 
-func NewFilterFromFile(ctx context.Context, filePath string) (FilterInterface, error) {
+func NewFilterFromFile(ctx context.Context, filePath string, options Options) (FilterInterface, error) {
 	f := &FileFilter{
 		WhitelistFilePath: filePath,
 	}
 
+	f.DefaultOptions = options
+
 	err := f.Reload()
 	if err != nil {
 		return nil, err
@@ -135,3 +143,14 @@ func (f *FileFilter) AllowChanId(id uint64) bool {
 
 	return ok
 }
+
+func (f *FileFilter) AllowSpecial(private bool) bool {
+	f.Mutex.Lock()
+	defer f.Mutex.Unlock()
+
+	if private {
+		return f.Options&AllowAllPrivate == AllowAllPrivate
+	} else {
+		return f.Options&AllowAllPublic == AllowAllPublic
+	}
+}

filter/filter_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestFile(t *testing.T) {
-	f, err := os.CreateTemp("", "tmpfile-") // in Go version older than 1.17 you can use ioutil.TempFile
+	f, err := os.CreateTemp("", "tmpfile-")
 	if err != nil {
 		t.Fatalf("Got error %v\n", err)
 	}
@@ -30,7 +30,7 @@ func TestFile(t *testing.T) {
 		t.Fatalf("Got error %v\n", err)
 	}
 
-	fil, err := NewFilterFromFile(context.TODO(), f.Name())
+	fil, err := NewFilterFromFile(context.TODO(), f.Name(), None)
 	if err != nil {
 		t.Fatalf("Got error %v\n", err)
 	}
@@ -43,3 +43,31 @@ func TestFile(t *testing.T) {
 		assert.Equal(t, false, fil.AllowChanId(chanid), "Should not be allowed")
 	}
 }
+
+func TestOptions(t *testing.T) {
+	f, err := os.CreateTemp("", "tmpfile-")
+	if err != nil {
+		t.Fatalf("Got error %v\n", err)
+	}
+
+	defer f.Close()
+	defer os.Remove(f.Name())
+
+	data := []byte(`
+	###
+	# Demo file
+	###
+	public
+	`)
+	if _, err := f.Write(data); err != nil {
+		t.Fatalf("Got error %v\n", err)
+	}
+
+	fil, err := NewFilterFromFile(context.TODO(), f.Name(), AllowAllPrivate)
+	if err != nil {
+		t.Fatalf("Got error %v\n", err)
+	}
+
+	assert.Equal(t, true, fil.AllowSpecial(true), "Private channels should be allowed")
+	assert.Equal(t, true, fil.AllowSpecial(false), "Publlic channels should be allowed")
+}

filter/interface.go

@@ -1,15 +1,27 @@
 package filter
 
+type Options uint8
+
+const (
+	None Options = 1 << iota
+	AllowAllPrivate
+	AllowAllPublic
+)
+
 type FilterInterface interface {
 	AllowPubKey(id string) bool
 	AllowChanId(id uint64) bool
+	AllowSpecial(private bool) bool
 }
 
 type Filter struct {
+	Options         Options
 	chanIdWhitelist map[uint64]struct{}
 	nodeIdWhitelist map[string]struct{}
 }
 
+// TODO: yeah it would be easier if we had just Allow(chan) or sth
+
 func (f *Filter) AllowPubKey(id string) bool {
 	_, ok := f.nodeIdWhitelist[id]
 
@@ -21,3 +33,11 @@ func (f *Filter) AllowChanId(id uint64) bool {
 
 	return ok
 }
+
+func (f *Filter) AllowSpecial(private bool) bool {
+	if private {
+		return f.Options&AllowAllPrivate == AllowAllPrivate
+	} else {
+		return f.Options&AllowAllPublic == AllowAllPublic
+	}
+}

nodeinfo/nodeinfo.go

@@ -256,7 +256,7 @@ func applyFilter(info *lightning_api.NodeInfoApiExtended, filter filter.FilterIn
 		nodeAllowed := (filter.AllowPubKey(c.Node1Pub) && info.Node.PubKey != c.Node1Pub) || (filter.AllowPubKey(c.Node2Pub) && info.Node.PubKey != c.Node2Pub)
 		chanAllowed := filter.AllowChanId(c.ChannelId)
 
-		if nodeAllowed || chanAllowed {
+		if nodeAllowed || chanAllowed || filter.AllowSpecial(c.Private) {
 			ret.Channels = append(ret.Channels, c)
 		} else {
 			ret.NumChannels -= 1