add --reader-node-line-buffer-size flag, double default value

CHANGELOG.md

@@ -8,6 +8,10 @@ Operators, you should copy/paste content of this content straight to your projec
 
 If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you should copy the content between those 2 version to your own repository, replacing placeholder value `fire{chain}` with your chain's own binary.
 
+## v1.5.6
+
+* add `--reader-node-line-buffer-size` flag and bump default value from 100M to 200M to go over crazy block 278208000 on Solana
+
 ## v1.5.5
 
 * added well known type for starknet and cosmos

cmd/apps/reader_node.go

@@ -62,6 +62,7 @@ func RegisterReaderNodeApp[B firecore.Block](chain *firecore.Chain[B], rootLog *
 			cmd.Flags().Uint("reader-node-start-block-num", 0, "Blocks that were produced with smaller block number then the given block num are skipped")
 			cmd.Flags().Uint("reader-node-stop-block-num", 0, "Shutdown reader when we the following 'stop-block-num' has been reached, inclusively.")
 			cmd.Flags().Int("reader-node-blocks-chan-capacity", 100, "Capacity of the channel holding blocks read by the reader. Process will shutdown reader-node if the channel gets over 90% of that capacity to prevent horrible consequences. Raise this number when processing tiny blocks very quickly")
+			cmd.Flags().Uint64("reader-node-line-buffer-size", 209715200, "Capacity of the buffer for reading a single line out of the node, in bytes (This is a hard limit. Some future enormouse blocks may require raising this to process them).")
 			cmd.Flags().String("reader-node-one-block-suffix", "default", cli.FlagDescription(`
 				Unique identifier for reader, so that it can produce 'oneblock files' in the same store as another instance without competing
 				for writes. You should set this flag if you have multiple reader running, each one should get a unique identifier, the
@@ -132,7 +133,9 @@ func RegisterReaderNodeApp[B firecore.Block](chain *firecore.Chain[B], rootLog *
 				readinessMaxLatency,
 			)
 
-			superviser := sv.SupervisorFactory(chain.ExecutableName, nodePath, nodeArguments, appLogger)
+			lineBufferSize := viper.GetUint64("reader-node-line-buffer-size")
+
+			superviser := sv.SupervisorFactory(chain.ExecutableName, nodePath, nodeArguments, lineBufferSize, appLogger)
 			superviser.RegisterLogPlugin(sv.NewNodeLogPlugin(debugFirehose))
 
 			var bootstrapper operator.Bootstrapper

go.sum

@@ -630,8 +630,6 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt
 github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8=
 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 h1:Y15G1Z4fpEdm2b+/70owI7TLuXadlqBtGM7rk4Hxrzk=
 github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0/go.mod h1:/Rnz2TJvaShjUct0scZ9kKV2Jr9/+KBAoWy4UMYxgv4=
-github.com/streamingfast/substreams v1.9.0 h1:Jr74Lo+jSp7CmyK94mdqJ0LfCYKy1o7dvYcufDPX/rk=
-github.com/streamingfast/substreams v1.9.0/go.mod h1:XtL4RgQawes9/a9iM9d6bAABacfIuekY+jceszF7u2c=
 github.com/streamingfast/substreams v1.9.1 h1:nINVI9vA4APGzuHPz6xdQ81H8/RtiuikPjhuDJPAjQw=
 github.com/streamingfast/substreams v1.9.1/go.mod h1:XtL4RgQawes9/a9iM9d6bAABacfIuekY+jceszF7u2c=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

superviser/genericsupervisor.go

@@ -21,13 +21,11 @@ type GenericSuperviser struct {
 	name      string
 }
 
-const LIMIT_BYTE = 100 * 1024 * 1024
-
 // This is the default implementation of the Chain Supervisor. If you wish to override the implementation for
 // your given chain you can override the 'SupervisorFactory' variable
-func newGenericSupervisor(name, binary string, arguments []string, appLogger *zap.Logger) nodeManager.ChainSuperviser {
-	if overseer.DEFAULT_LINE_BUFFER_SIZE < LIMIT_BYTE {
-		overseer.DEFAULT_LINE_BUFFER_SIZE = LIMIT_BYTE
+func newGenericSupervisor(name, binary string, arguments []string, lineBufferSize uint64, appLogger *zap.Logger) nodeManager.ChainSuperviser {
+	if overseer.DEFAULT_LINE_BUFFER_SIZE < int(lineBufferSize) {
+		overseer.DEFAULT_LINE_BUFFER_SIZE = int(lineBufferSize)
 	}
 
 	return &GenericSuperviser{