diff --git a/dex/encode/encode.go b/dex/encode/encode.go index 7726015a7d..2c3ea0ecd0 100644 --- a/dex/encode/encode.go +++ b/dex/encode/encode.go @@ -4,25 +4,27 @@ package encode import ( - "bytes" "crypto/rand" "crypto/sha256" "encoding/binary" "fmt" "io" + "math" "os" "time" ) var ( - // IntCoder is the DEX-wide integer byte-encoding order. + // IntCoder is the DEX-wide integer byte-encoding order. IntCoder must be + // BigEndian so that variable length data encodings work as intended. IntCoder = binary.BigEndian // A byte-slice representation of boolean false. ByteFalse = []byte{0} // A byte-slice representation of boolean true. ByteTrue = []byte{1} - maxU16 = int(^uint16(0)) - bEqual = bytes.Equal + // MaxDataLen is the largest byte slice that can be stored when using + // (BuildyBytes).AddData. + MaxDataLen = 0x00fe_ffff // top two bytes in big endian stop at 254, signalling 32-bit len ) // Uint64Bytes converts the uint16 to a length-2, big-endian encoded byte slice. @@ -122,10 +124,20 @@ func ExtractPushes(b []byte, preAlloc ...int) ([][]byte, error) { b = b[1:] if l == 255 { if len(b) < 2 { - return nil, fmt.Errorf("2 bytes not available for uint16 data length") + return nil, fmt.Errorf("2 bytes not available for data length") } l = int(IntCoder.Uint16(b[:2])) - b = b[2:] + if l < 255 { + // This indicates it's really a uint32 capped at 0x00fe_ffff, and + // we are looking at the top two bytes. Decode all four. + if len(b) < 4 { + return nil, fmt.Errorf("4 bytes not available for 32-bit data length") + } + l = int(IntCoder.Uint32(b[:4])) + b = b[4:] + } else { // includes 255 + b = b[2:] + } } if len(b) < l { return nil, fmt.Errorf("data too short for pop of %d bytes", l) @@ -155,9 +167,9 @@ func DecodeBlob(b []byte, preAlloc ...int) (byte, [][]byte, error) { // BuildyBytes is a byte-slice with an AddData method for building linearly // encoded 2D byte slices. The AddData method supports chaining. The canonical -// use case is to create "versioned blobs", where the BuildyBytes is instantated -// with a single version byte, and then data pushes are added using the AddData -// method. Example use: +// use case is to create "versioned blobs", where the BuildyBytes is +// instantiated with a single version byte, and then data pushes are added using +// the AddData method. Example use: // // version := 0 // b := BuildyBytes{version}.AddData(data1).AddData(data2) @@ -168,16 +180,30 @@ func DecodeBlob(b []byte, preAlloc ...int) (byte, [][]byte, error) { type BuildyBytes []byte // AddData adds the data to the BuildyBytes, and returns the new BuildyBytes. -// The data has hard-coded length limit of uint16_max = 65535 bytes. +// The data has hard-coded length limit of MaxDataLen = 16711679 bytes. The +// caller should ensure the data is not larger since AddData panics if it is. func (b BuildyBytes) AddData(d []byte) BuildyBytes { l := len(d) var lBytes []byte - if l > 0xff-1 { - if l > maxU16 { - panic("cannot use addData for pushes > 65535 bytes") + if l >= 0xff { + if l > MaxDataLen { + panic("cannot use addData for pushes > 16711679 bytes") + } + var i []byte + if l > math.MaxUint16 { // not >= since that is historically in 2 bytes + // We are retrofitting for data longer than 65535 bytes, so we + // cannot switch to uint32 at 65535 itself since it is possible + // there is data of exactly that length already stored using just + // two bytes to encode the length. Thus, the decoder should inspect + // the top two bytes (big endian), switching to uint32 if under 255. + // Therefore, the highest length with this scheme is 0x00fe_ffff + // (16,711,679 bytes). + i = make([]byte, 4) + IntCoder.PutUint32(i, uint32(l)) + } else { // includes MaxUint16 for historical reasons + i = make([]byte, 2) + IntCoder.PutUint16(i, uint16(l)) } - i := make([]byte, 2) - IntCoder.PutUint16(i, uint16(l)) lBytes = append([]byte{0xff}, i...) } else { lBytes = []byte{byte(l)} diff --git a/dex/encode/encode_test.go b/dex/encode/encode_test.go index 01b8ddcc8c..a791fdf2f0 100644 --- a/dex/encode/encode_test.go +++ b/dex/encode/encode_test.go @@ -1,10 +1,12 @@ package encode import ( + "bytes" "testing" ) var ( + bEqual = bytes.Equal tEmpty = []byte{} tA = []byte{0xaa} tB = []byte{0xbb, 0xbb} @@ -46,7 +48,17 @@ func TestBuildyBytes(t *testing.T) { } func TestDecodeBlob(t *testing.T) { + almostLongBlob := RandomBytes(254) longBlob := RandomBytes(255) + longBlob2 := RandomBytes(555) + longestUint16Blob := RandomBytes(65535) + longerBlob := RandomBytes(65536) + longerBlob2 := RandomBytes(65599) + megaBlob := RandomBytes(12_345_678) + almostLargestBlob := RandomBytes(MaxDataLen - 1) + largestBlob := RandomBytes(MaxDataLen) + // tooLargeBlob tested after the loop, recovering the expected panic + type test struct { v byte b []byte @@ -64,11 +76,51 @@ func TestDecodeBlob(t *testing.T) { b: BuildyBytes{5}.AddData(tB).AddData(tC), exp: [][]byte{tB, tC}, }, + { + v: 250, + b: BuildyBytes{250}.AddData(tA).AddData(almostLongBlob), + exp: [][]byte{tA, almostLongBlob}, + }, { v: 255, b: BuildyBytes{255}.AddData(tA).AddData(longBlob), exp: [][]byte{tA, longBlob}, }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(longBlob2), + exp: [][]byte{tA, longBlob2}, + }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(longestUint16Blob), + exp: [][]byte{tA, longestUint16Blob}, + }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(longerBlob), + exp: [][]byte{tA, longerBlob}, + }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(longerBlob2), + exp: [][]byte{tA, longerBlob2}, + }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(megaBlob), + exp: [][]byte{tA, megaBlob}, + }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(almostLargestBlob), + exp: [][]byte{tA, almostLargestBlob}, + }, + { + v: 255, + b: BuildyBytes{255}.AddData(tA).AddData(largestBlob), + exp: [][]byte{tA, largestBlob}, + }, { b: []byte{0x01, 0x02}, // missing two bytes wantErr: true, @@ -95,4 +147,14 @@ func TestDecodeBlob(t *testing.T) { } } } + + tooLargeBlob := RandomBytes(MaxDataLen + 1) + func() { + defer func() { + if r := recover(); r == nil { + t.Errorf("no panic encoding data that's too large") + } + }() + BuildyBytes{255}.AddData(tA).AddData(tooLargeBlob) + }() }