Change entry v1 data layout

linxGnu · Jul 25, 2021 · 1cfca43 · 1cfca43
1 parent 0db5866
commit 1cfca43
Show file tree

Hide file tree

Showing 7 changed files with 111 additions and 121 deletions.
diff --git a/README.md b/README.md
diff --git a/common/pkg.go b/common/pkg.go
@@ -17,13 +17,13 @@ type EntryFormat = uint32
 const (
 	// EntryV1 layout:
 	//
-	// [Length - uint32][Payload - bytes][Checksum - uint32]
+	// [Length - uint32][Checksum - uint32][Payload - bytes]
 	//
 	// Note:
 	// - `Payload` has size of `Length`
 	// - `Checksum` is crc32_IEEE(`Payload`)
 	// - `Entry` always starts with non-zero `Length` header
-	// - `Length` == 0 means ending, Payload and Checksum won't be written in this case.
+	// - `Length` == 0 means ending, Payload won't be written in this case.
 	EntryV1 EntryFormat = iota
 )
 

diff --git a/entry/entry.go b/entry/entry.go
@@ -40,17 +40,13 @@ func (e Entry) Marshal(w WriteFlusher, format common.EntryFormat) (code common.E
 	}
 }
 
-// [Length - uint32][Payload - bytes][Checksum - uint32]
+// [Length - uint32][Checksum - uint32][Payload - bytes]
 func (e Entry) marshalV1(w WriteFlusher) (code common.ErrCode, err error) {
-	var buf [4]byte
-
-	common.Endianese.PutUint32(buf[:], uint32(len(e)))
+	var buf [8]byte
+	common.Endianese.PutUint64(buf[:], uint64(len(e))<<32|uint64(crc32.ChecksumIEEE(e)))
 	if _, err = w.Write(buf[:]); err == nil {
 		if _, err = w.Write(e); err == nil {
-			common.Endianese.PutUint32(buf[:], crc32.ChecksumIEEE(e))
-			if _, err = w.Write(buf[:]); err == nil {
-				err = w.Flush()
-			}
+			err = w.Flush()
 		}
 	}
 
@@ -74,9 +70,9 @@ func (e *Entry) Unmarshal(r io.Reader, format common.EntryFormat) (common.ErrCod
 	}
 }
 
-// [Length - uint32][Payload - bytes][Checksum - uint32]
+// [Length - uint32][Checksum - uint32][Payload - bytes]
 func (e *Entry) unmarshalV1(r io.Reader) (code common.ErrCode, err error) {
-	var buffer [4]byte
+	var buffer [8]byte
 
 	// read length
 	_, err = io.ReadFull(r, buffer[:])
@@ -90,7 +86,8 @@ func (e *Entry) unmarshalV1(r io.Reader) (code common.ErrCode, err error) {
 	}
 
 	// check length
-	size := common.Endianese.Uint32(buffer[:])
+	sizeAndSum := common.Endianese.Uint64(buffer[:])
+	size := sizeAndSum >> 32
 	if size == 0 {
 		code = common.EntryZeroSize
 		return
@@ -109,15 +106,8 @@ func (e *Entry) unmarshalV1(r io.Reader) (code common.ErrCode, err error) {
 		return
 	}
 
-	// read sum
-	_, err = io.ReadFull(r, buffer[:])
-	if err != nil {
-		code = common.EntryCorrupted
-		return
-	}
-
 	// checksum
-	if common.Endianese.Uint32(buffer[:]) != crc32.ChecksumIEEE(data) {
+	if crc32.ChecksumIEEE(data) != uint32(sizeAndSum) { // downcast to get lower 32-bit
 		code, err = common.EntryCorrupted, common.ErrEntryInvalidCheckSum
 	} else {
 		*e = data

diff --git a/entry/entry_test.go b/entry/entry_test.go
@@ -31,14 +31,14 @@ func TestEntryUnmarshal(t *testing.T) {
 
 		// length zero
 		{
-			code, err := e.Unmarshal(bytes.NewBuffer([]byte{0, 0, 0, 0}), common.EntryV1)
+			code, err := e.Unmarshal(bytes.NewBuffer([]byte{0, 0, 0, 0, 0, 0, 0, 0}), common.EntryV1)
 			require.Equal(t, common.EntryZeroSize, code)
 			require.NoError(t, err)
 		}
 
 		// too big
 		{
-			code, err := e.Unmarshal(bytes.NewBuffer([]byte{255, 255, 0, 0}), common.EntryV1)
+			code, err := e.Unmarshal(bytes.NewBuffer([]byte{255, 255, 0, 0, 0, 0, 0, 0}), common.EntryV1)
 			require.Equal(t, common.EntryTooBig, code)
 			require.NoError(t, err)
 		}
@@ -68,7 +68,7 @@ func TestEntryUnmarshal(t *testing.T) {
 	t.Run("Happy", func(t *testing.T) {
 		var e Entry = make([]byte, 32)
 
-		buf := bytes.NewBuffer([]byte{0, 0, 0, 2, 19, 31, 173, 62, 94, 152})
+		buf := bytes.NewBuffer([]byte{0, 0, 0, 2, 173, 62, 94, 152, 19, 31})
 
 		code, err := e.Unmarshal(buf, common.EntryV1)
 		require.Equal(t, common.NoError, code)
@@ -136,7 +136,7 @@ func TestEntryMarshal(t *testing.T) {
 		code, err := e.Marshal(&noopFlusher{Writer: &buf}, common.EntryV1)
 		require.NoError(t, err)
 		require.Equal(t, code, common.NoError)
-		require.EqualValues(t, []byte{0, 0, 0, 4, 1, 2, 3, 4, 0xb6, 0x3c, 0xfb, 0xcd}, buf.Bytes())
+		require.EqualValues(t, []byte{0, 0, 0, 4, 0xb6, 0x3c, 0xfb, 0xcd, 1, 2, 3, 4}, buf.Bytes())
 	})
 }
 

diff --git a/segment/v1/segment_reader_test.go b/segment/v1/segment_reader_test.go
@@ -23,7 +23,7 @@ func TestSegmentReader(t *testing.T) {
 		}
 
 		{
-			w := newSegmentReader(io.NopCloser(bytes.NewBuffer([]byte{255, 255, 0, 0})), common.EntryV1)
+			w := newSegmentReader(io.NopCloser(bytes.NewBuffer([]byte{255, 255, 0, 0, 0, 0, 0, 0})), common.EntryV1)
 
 			var e entry.Entry
 			code, err := w.ReadEntry(&e)
@@ -32,7 +32,7 @@ func TestSegmentReader(t *testing.T) {
 		}
 
 		{
-			w := newSegmentReader(io.NopCloser(bytes.NewBuffer([]byte{0, 0, 0, 0})), common.EntryV1)
+			w := newSegmentReader(io.NopCloser(bytes.NewBuffer([]byte{0, 0, 0, 0, 0, 0, 0, 0})), common.EntryV1)
 
 			var e entry.Entry
 			code, err := w.ReadEntry(&e)
@@ -52,8 +52,8 @@ func TestSegmentReader(t *testing.T) {
 
 	t.Run("Happy", func(t *testing.T) {
 		underlying := io.NopCloser(bytes.NewBuffer([]byte{
-			0, 0, 0, 2, 19, 31, 173, 62, 94, 152,
-			0, 0, 0, 2, 19, 31, 173, 62, 94, 152,
+			0, 0, 0, 2, 173, 62, 94, 152, 19, 31,
+			0, 0, 0, 2, 173, 62, 94, 152, 19, 31,
 		}))
 		w := newSegmentReader(newBufferReader(underlying), common.EntryV1)
 

diff --git a/segment/v1/segment_test.go b/segment/v1/segment_test.go
@@ -162,7 +162,7 @@ func TestNewSegmentReadWrite(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, common.SegmentNoMoreReadWeak, code)
 
-			_, _ = buffer.Write([]byte{0, 0, 0, 0})
+			_, _ = buffer.Write(segmentEnding)
 			code, err = s.readEntry(&e)
 			require.NoError(t, err)
 			require.Equal(t, common.SegmentNoMoreReadStrong, code)

diff --git a/segment/v1/segment_writer.go b/segment/v1/segment_writer.go
@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	segmentEnding = []byte{0, 0, 0, 0}
+	segmentEnding = []byte{0, 0, 0, 0, 0, 0, 0, 0}
 )
 
 type segmentWriter struct {