dex/encode: allow data longer than 65535 bytes

dex/encode/encode.go

@@ -4,25 +4,27 @@
 package encode
 
 import (
-	"bytes"
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"time"
 )
 
 var (
-	// IntCoder is the DEX-wide integer byte-encoding order.
+	// IntCoder is the DEX-wide integer byte-encoding order. IntCoder must be
+	// BigEndian so that variable length data encodings work as intended.
 	IntCoder = binary.BigEndian
 	// A byte-slice representation of boolean false.
 	ByteFalse = []byte{0}
 	// A byte-slice representation of boolean true.
 	ByteTrue = []byte{1}
-	maxU16   = int(^uint16(0))
-	bEqual   = bytes.Equal
+	// MaxDataLen is the largest byte slice that can be stored when using
+	// (BuildyBytes).AddData.
+	MaxDataLen = 0x00fe_ffff // top two bytes in big endian stop at 254, signalling 32-bit len
 )
 
 // Uint64Bytes converts the uint16 to a length-2, big-endian encoded byte slice.
@@ -122,10 +124,20 @@ func ExtractPushes(b []byte, preAlloc ...int) ([][]byte, error) {
 		b = b[1:]
 		if l == 255 {
 			if len(b) < 2 {
-				return nil, fmt.Errorf("2 bytes not available for uint16 data length")
+				return nil, fmt.Errorf("2 bytes not available for data length")
 			}
 			l = int(IntCoder.Uint16(b[:2]))
-			b = b[2:]
+			if l < 255 {
+				// This indicates it's really a uint32 capped at 0x00fe_ffff, and
+				// we are looking at the top two bytes. Decode all four.
+				if len(b) < 4 {
+					return nil, fmt.Errorf("4 bytes not available for 32-bit data length")
+				}
+				l = int(IntCoder.Uint32(b[:4]))
+				b = b[4:]
+			} else { // includes 255
+				b = b[2:]
+			}
 		}
 		if len(b) < l {
 			return nil, fmt.Errorf("data too short for pop of %d bytes", l)
@@ -155,9 +167,9 @@ func DecodeBlob(b []byte, preAlloc ...int) (byte, [][]byte, error) {
 
 // BuildyBytes is a byte-slice with an AddData method for building linearly
 // encoded 2D byte slices. The AddData method supports chaining. The canonical
-// use case is to create "versioned blobs", where the BuildyBytes is instantated
-// with a single version byte, and then data pushes are added using the AddData
-// method. Example use:
+// use case is to create "versioned blobs", where the BuildyBytes is
+// instantiated with a single version byte, and then data pushes are added using
+// the AddData method. Example use:
 //
 //   version := 0
 //   b := BuildyBytes{version}.AddData(data1).AddData(data2)
@@ -168,16 +180,30 @@ func DecodeBlob(b []byte, preAlloc ...int) (byte, [][]byte, error) {
 type BuildyBytes []byte
 
 // AddData adds the data to the BuildyBytes, and returns the new BuildyBytes.
-// The data has hard-coded length limit of uint16_max = 65535 bytes.
+// The data has hard-coded length limit of MaxDataLen = 16711679 bytes. The
+// caller should ensure the data is not larger since AddData panics if it is.
 func (b BuildyBytes) AddData(d []byte) BuildyBytes {
 	l := len(d)
 	var lBytes []byte
-	if l > 0xff-1 {
-		if l > maxU16 {
-			panic("cannot use addData for pushes > 65535 bytes")
+	if l >= 0xff {
+		if l > MaxDataLen {
+			panic("cannot use addData for pushes > 16711679 bytes")
+		}
+		var i []byte
+		if l > math.MaxUint16 { // not >= since that is historically in 2 bytes
+			// We are retrofitting for data longer than 65535 bytes, so we
+			// cannot switch to uint32 at 65535 itself since it is possible
+			// there is data of exactly that length already stored using just
+			// two bytes to encode the length. Thus, the decoder should inspect
+			// the top two bytes (big endian), switching to uint32 if under 255.
+			// Therefore, the highest length with this scheme is 0x00fe_ffff
+			// (16,711,679 bytes).
+			i = make([]byte, 4)
+			IntCoder.PutUint32(i, uint32(l))
+		} else { // includes MaxUint16 for historical reasons
+			i = make([]byte, 2)
+			IntCoder.PutUint16(i, uint16(l))
 		}
-		i := make([]byte, 2)
-		IntCoder.PutUint16(i, uint16(l))
 		lBytes = append([]byte{0xff}, i...)
 	} else {
 		lBytes = []byte{byte(l)}

dex/encode/encode_test.go

@@ -1,10 +1,12 @@
 package encode
 
 import (
+	"bytes"
 	"testing"
 )
 
 var (
+	bEqual = bytes.Equal
 	tEmpty = []byte{}
 	tA     = []byte{0xaa}
 	tB     = []byte{0xbb, 0xbb}
@@ -46,7 +48,17 @@ func TestBuildyBytes(t *testing.T) {
 }
 
 func TestDecodeBlob(t *testing.T) {
+	almostLongBlob := RandomBytes(254)
 	longBlob := RandomBytes(255)
+	longBlob2 := RandomBytes(555)
+	longestUint16Blob := RandomBytes(65535)
+	longerBlob := RandomBytes(65536)
+	longerBlob2 := RandomBytes(65599)
+	megaBlob := RandomBytes(12_345_678)
+	almostLargestBlob := RandomBytes(MaxDataLen - 1)
+	largestBlob := RandomBytes(MaxDataLen)
+	// tooLargeBlob tested after the loop, recovering the expected panic
+
 	type test struct {
 		v       byte
 		b       []byte
@@ -64,11 +76,51 @@ func TestDecodeBlob(t *testing.T) {
 			b:   BuildyBytes{5}.AddData(tB).AddData(tC),
 			exp: [][]byte{tB, tC},
 		},
+		{
+			v:   250,
+			b:   BuildyBytes{250}.AddData(tA).AddData(almostLongBlob),
+			exp: [][]byte{tA, almostLongBlob},
+		},
 		{
 			v:   255,
 			b:   BuildyBytes{255}.AddData(tA).AddData(longBlob),
 			exp: [][]byte{tA, longBlob},
 		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(longBlob2),
+			exp: [][]byte{tA, longBlob2},
+		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(longestUint16Blob),
+			exp: [][]byte{tA, longestUint16Blob},
+		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(longerBlob),
+			exp: [][]byte{tA, longerBlob},
+		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(longerBlob2),
+			exp: [][]byte{tA, longerBlob2},
+		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(megaBlob),
+			exp: [][]byte{tA, megaBlob},
+		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(almostLargestBlob),
+			exp: [][]byte{tA, almostLargestBlob},
+		},
+		{
+			v:   255,
+			b:   BuildyBytes{255}.AddData(tA).AddData(largestBlob),
+			exp: [][]byte{tA, largestBlob},
+		},
 		{
 			b:       []byte{0x01, 0x02}, // missing two bytes
 			wantErr: true,
@@ -95,4 +147,14 @@ func TestDecodeBlob(t *testing.T) {
 			}
 		}
 	}
+
+	tooLargeBlob := RandomBytes(MaxDataLen + 1)
+	func() {
+		defer func() {
+			if r := recover(); r == nil {
+				t.Errorf("no panic encoding data that's too large")
+			}
+		}()
+		BuildyBytes{255}.AddData(tA).AddData(tooLargeBlob)
+	}()
 }