[wip] E35

commitment/bin_patricia_hashed.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"math/bits"
+	"sort"
 
 	"github.com/holiman/uint256"
 	"github.com/ledgerwatch/log/v3"
@@ -746,7 +747,7 @@ func (bph *BinPatriciaHashed) computeBinaryCellHash(cell *BinaryCell, depth int,
 		var valBuf [128]byte
 		valLen := cell.accountForHashing(valBuf[:], storageRootHash)
 		if bph.trace {
-			fmt.Printf("accountLeafHashWithKey for [%x]=>[%x]\n", bph.hashAuxBuffer[:halfKeySize+1-depth], valBuf[:valLen])
+			fmt.Printf("accountLeafHashWithKey for [%x]=>[%x]\n", cell.downHashedKey[:halfKeySize+1-depth], rlp.RlpEncodedBytes(valBuf[:valLen]))
 		}
 		return bph.accountLeafHashWithKey(buf, cell.downHashedKey[:halfKeySize+1-depth], rlp.RlpEncodedBytes(valBuf[:valLen]))
 	}
@@ -1274,9 +1275,19 @@ func (bph *BinPatriciaHashed) RootHash() ([]byte, error) {
 	return hash[1:], nil // first byte is 128+hash_len
 }
 
-func (bph *BinPatriciaHashed) ReviewKeys(plainKeys, hashedKeys [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
+func (bph *BinPatriciaHashed) ProcessKeys(plainKeys [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
 	branchNodeUpdates = make(map[string]BranchData)
 
+	pks := make(map[string]int, len(plainKeys))
+	hashedKeys := make([][]byte, len(plainKeys))
+	for i, pk := range plainKeys {
+		hashedKeys[i] = hexToBin(pk)
+		pks[string(hashedKeys[i])] = i
+	}
+
+	sort.Slice(hashedKeys, func(i, j int) bool {
+		return bytes.Compare(hashedKeys[i], hashedKeys[j]) < 0
+	})
 	stagedBinaryCell := new(BinaryCell)
 	for i, hashedKey := range hashedKeys {
 		plainKey := plainKeys[i]
@@ -1310,7 +1321,7 @@ func (bph *BinPatriciaHashed) ReviewKeys(plainKeys, hashedKeys [][]byte) (rootHa
 				cell.setAccountFields(stagedBinaryCell.CodeHash[:], &stagedBinaryCell.Balance, stagedBinaryCell.Nonce)
 
 				if bph.trace {
-					fmt.Printf("accountFn reading key %x => balance=%v nonce=%v codeHash=%x\n", cell.apk, cell.Balance.Uint64(), cell.Nonce, cell.CodeHash)
+					fmt.Printf("accountFn reading key %x => balance=%d nonce=%v codeHash=%x\n", cell.apk, &cell.Balance, cell.Nonce, cell.CodeHash)
 				}
 			}
 		} else {
@@ -1503,12 +1514,10 @@ func (bph *BinPatriciaHashed) SetState(buf []byte) error {
 		return err
 	}
 
-	bph.currentKeyLen = int(s.CurrentKeyLen)
 	bph.rootChecked = s.RootChecked
 	bph.rootTouched = s.RootTouched
 	bph.rootPresent = s.RootPresent
 
-	copy(bph.currentKey[:], s.CurrentKey[:])
 	copy(bph.depths[:], s.Depths[:])
 	copy(bph.branchBefore[:], s.BranchBefore[:])
 	copy(bph.touchMap[:], s.TouchMap[:])
@@ -1517,44 +1526,52 @@ func (bph *BinPatriciaHashed) SetState(buf []byte) error {
 	return nil
 }
 
-func (bph *BinPatriciaHashed) ProcessUpdates(plainKeys, hashedKeys [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
+func (bph *BinPatriciaHashed) ProcessUpdates(plainKeys [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
 	branchNodeUpdates = make(map[string]BranchData)
 
+	for i, pk := range plainKeys {
+		updates[i].hashedKey = hexToBin(pk)
+		updates[i].plainKey = pk
+	}
+
+	sort.Slice(updates, func(i, j int) bool {
+		return bytes.Compare(updates[i].hashedKey, updates[j].hashedKey) < 0
+	})
+
 	for i, plainKey := range plainKeys {
-		hashedKey := hashedKeys[i]
+		update := updates[i]
 		if bph.trace {
-			fmt.Printf("plainKey=[%x], hashedKey=[%x], currentKey=[%x]\n", plainKey, hashedKey, bph.currentKey[:bph.currentKeyLen])
+			fmt.Printf("plainKey=[%x], hashedKey=[%x], currentKey=[%x]\n", update.plainKey, update.hashedKey, bph.currentKey[:bph.currentKeyLen])
 		}
 		// Keep folding until the currentKey is the prefix of the key we modify
-		for bph.needFolding(hashedKey) {
+		for bph.needFolding(update.hashedKey) {
 			if branchData, updateKey, err := bph.fold(); err != nil {
 				return nil, nil, fmt.Errorf("fold: %w", err)
 			} else if branchData != nil {
 				branchNodeUpdates[string(updateKey)] = branchData
 			}
 		}
 		// Now unfold until we step on an empty cell
-		for unfolding := bph.needUnfolding(hashedKey); unfolding > 0; unfolding = bph.needUnfolding(hashedKey) {
-			if err := bph.unfold(hashedKey, unfolding); err != nil {
+		for unfolding := bph.needUnfolding(update.hashedKey); unfolding > 0; unfolding = bph.needUnfolding(update.hashedKey) {
+			if err := bph.unfold(update.hashedKey, unfolding); err != nil {
 				return nil, nil, fmt.Errorf("unfold: %w", err)
 			}
 		}
 
-		update := updates[i]
 		// Update the cell
 		if update.Flags == DeleteUpdate {
-			bph.deleteBinaryCell(hashedKey)
+			bph.deleteBinaryCell(update.hashedKey)
 			if bph.trace {
-				fmt.Printf("key %x deleted\n", plainKey)
+				fmt.Printf("key %x deleted\n", update.plainKey)
 			}
 		} else {
-			cell := bph.updateBinaryCell(plainKey, hashedKey)
+			cell := bph.updateBinaryCell(update.plainKey, update.hashedKey)
 			if bph.trace {
 				fmt.Printf("accountFn updated key %x =>", plainKey)
 			}
 			if update.Flags&BalanceUpdate != 0 {
 				if bph.trace {
-					fmt.Printf(" balance=%d", update.Balance.Uint64())
+					fmt.Printf(" balance=%d", &update.Balance)
 				}
 				cell.Balance.Set(&update.Balance)
 			}
@@ -1603,13 +1620,13 @@ func (bph *BinPatriciaHashed) hashAndNibblizeKey2(key []byte) []byte { //nolint
 
 	bph.keccak.Reset()
 	bph.keccak.Write(key[:length.Addr])
-	copy(hashedKey[:length.Hash], bph.keccak.Sum(nil))
+	bph.keccak.Read(hashedKey[:length.Hash])
 
 	if len(key[length.Addr:]) > 0 {
 		hashedKey = append(hashedKey, make([]byte, length.Hash)...)
 		bph.keccak.Reset()
 		bph.keccak.Write(key[length.Addr:])
-		copy(hashedKey[length.Hash:], bph.keccak.Sum(nil))
+		bph.keccak.Read(hashedKey[length.Hash:])
 	}
 
 	nibblized := make([]byte, len(hashedKey)*2)

commitment/bin_patricia_hashed_test.go

@@ -20,7 +20,7 @@ func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
 	trie := NewBinPatriciaHashed(length.Addr, ms.branchFn, ms.accountFn, ms.storageFn)
 	trieBatch := NewBinPatriciaHashed(length.Addr, ms2.branchFn, ms2.accountFn, ms2.storageFn)
 
-	plainKeys, hashedKeys, updates := NewUpdateBuilder().
+	plainKeys, updates := NewUpdateBuilder().
 		Balance("e25652aaa6b9417973d325f9a1246b48ff9420bf", 12).
 		Balance("cdd0a12034e978f7eccda72bd1bd89a8142b704e", 120000).
 		Balance("5bb6abae12c87592b940458437526cb6cad60d50", 170).
@@ -43,25 +43,25 @@ func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
 	fmt.Println("1. Running sequential updates over the bin trie")
 	var seqHash []byte
 	for i := 0; i < len(updates); i++ {
-		sh, branchNodeUpdates, err := trie.ReviewKeys(plainKeys[i:i+1], hashedKeys[i:i+1])
+		sh, branchNodeUpdates, err := trie.ProcessKeys(plainKeys[i : i+1])
 		require.NoError(t, err)
 		require.Len(t, sh, length.Hash)
 		ms.applyBranchNodeUpdates(branchNodeUpdates)
 		// WARN! provided sequential branch updates are incorrect - lead to deletion of prefixes (afterMap is zero)
 		//       while root hashes are equal
-		renderUpdates(branchNodeUpdates)
+		//renderUpdates(branchNodeUpdates)
 
 		fmt.Printf("h=%x\n", sh)
 		seqHash = sh
 	}
 
 	fmt.Println("2. Running batch updates over the bin trie")
 
-	batchHash, branchBatchUpdates, err := trieBatch.ReviewKeys(plainKeys, hashedKeys)
+	batchHash, branchBatchUpdates, err := trieBatch.ProcessKeys(plainKeys)
 	require.NoError(t, err)
 	ms2.applyBranchNodeUpdates(branchBatchUpdates)
 
-	renderUpdates(branchBatchUpdates)
+	//renderUpdates(branchBatchUpdates)
 
 	require.EqualValues(t, seqHash, batchHash)
 	// require.EqualValues(t, seqHash, batchHash)
@@ -88,7 +88,7 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) {
 	ms := NewMockState(t)
 	ms2 := NewMockState(t)
 
-	plainKeys, hashedKeys, updates := NewUpdateBuilder().
+	plainKeys, updates := NewUpdateBuilder().
 		Balance("f5", 4).
 		Balance("ff", 900234).
 		Balance("04", 1233).
@@ -122,22 +122,22 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		sequentialRoot, branchNodeUpdates, err := trieOne.ReviewKeys(plainKeys[i:i+1], hashedKeys[i:i+1])
+		sequentialRoot, branchNodeUpdates, err := trieOne.ProcessKeys(plainKeys[i : i+1])
 		require.NoError(t, err)
 		roots = append(roots, sequentialRoot)
 
 		ms.applyBranchNodeUpdates(branchNodeUpdates)
-		renderUpdates(branchNodeUpdates)
+		//renderUpdates(branchNodeUpdates)
 	}
 
 	err := ms2.applyPlainUpdates(plainKeys, updates)
 	require.NoError(t, err)
 
 	fmt.Printf("\n2. Trie batch update generated following branch updates\n")
 	// batch update
-	batchRoot, branchNodeUpdatesTwo, err := trieTwo.ReviewKeys(plainKeys, hashedKeys)
+	batchRoot, branchNodeUpdatesTwo, err := trieTwo.ProcessKeys(plainKeys)
 	require.NoError(t, err)
-	renderUpdates(branchNodeUpdatesTwo)
+	//renderUpdates(branchNodeUpdatesTwo)
 
 	fmt.Printf("\n sequential roots:\n")
 	for i, rh := range roots {
@@ -154,7 +154,7 @@ func Test_BinPatriciaHashed_EmptyState(t *testing.T) {
 	ms := NewMockState(t)
 	hph := NewBinPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn)
 	hph.SetTrace(false)
-	plainKeys, hashedKeys, updates := NewUpdateBuilder().
+	plainKeys, updates := NewUpdateBuilder().
 		Balance("00", 4).
 		Balance("01", 5).
 		Balance("02", 6).
@@ -171,56 +171,56 @@ func Test_BinPatriciaHashed_EmptyState(t *testing.T) {
 	err := ms.applyPlainUpdates(plainKeys, updates)
 	require.NoError(t, err)
 
-	firstRootHash, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
+	firstRootHash, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
 	require.NoError(t, err)
 
 	t.Logf("root hash %x\n", firstRootHash)
 
 	ms.applyBranchNodeUpdates(branchNodeUpdates)
 
 	fmt.Printf("1. Generated updates\n")
-	renderUpdates(branchNodeUpdates)
+	//renderUpdates(branchNodeUpdates)
 
 	// More updates
 	hph.Reset()
 	hph.SetTrace(false)
-	plainKeys, hashedKeys, updates = NewUpdateBuilder().
+	plainKeys, updates = NewUpdateBuilder().
 		Storage("03", "58", "050505").
 		Build()
 	err = ms.applyPlainUpdates(plainKeys, updates)
 	require.NoError(t, err)
 
-	secondRootHash, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
+	secondRootHash, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
 	require.NoError(t, err)
 	require.NotEqualValues(t, firstRootHash, secondRootHash)
 
 	ms.applyBranchNodeUpdates(branchNodeUpdates)
 	fmt.Printf("2. Generated single update\n")
-	renderUpdates(branchNodeUpdates)
+	//renderUpdates(branchNodeUpdates)
 
 	// More updates
 	hph.Reset()
 	hph.SetTrace(false)
-	plainKeys, hashedKeys, updates = NewUpdateBuilder().
+	plainKeys, updates = NewUpdateBuilder().
 		Storage("03", "58", "070807").
 		Build()
 	err = ms.applyPlainUpdates(plainKeys, updates)
 	require.NoError(t, err)
 
-	thirdRootHash, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
+	thirdRootHash, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
 	require.NoError(t, err)
 	require.NotEqualValues(t, secondRootHash, thirdRootHash)
 
 	ms.applyBranchNodeUpdates(branchNodeUpdates)
 	fmt.Printf("3. Generated single update\n")
-	renderUpdates(branchNodeUpdates)
+	//renderUpdates(branchNodeUpdates)
 }
 
 func Test_BinPatriciaHashed_EmptyUpdateState(t *testing.T) {
 	ms := NewMockState(t)
 	hph := NewBinPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn)
 	hph.SetTrace(false)
-	plainKeys, hashedKeys, updates := NewUpdateBuilder().
+	plainKeys, updates := NewUpdateBuilder().
 		Balance("00", 4).
 		Nonce("00", 246462653).
 		Balance("01", 5).
@@ -233,24 +233,24 @@ func Test_BinPatriciaHashed_EmptyUpdateState(t *testing.T) {
 	err := ms.applyPlainUpdates(plainKeys, updates)
 	require.NoError(t, err)
 
-	hashBeforeEmptyUpdate, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
+	hashBeforeEmptyUpdate, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
 	require.NoError(t, err)
 	require.NotEmpty(t, hashBeforeEmptyUpdate)
 
 	ms.applyBranchNodeUpdates(branchNodeUpdates)
 
 	fmt.Println("1. Updates applied")
-	renderUpdates(branchNodeUpdates)
+	//renderUpdates(branchNodeUpdates)
 
 	// generate empty updates and do NOT reset tree
 	hph.SetTrace(true)
 
-	plainKeys, hashedKeys, updates = NewUpdateBuilder().Build()
+	plainKeys, updates = NewUpdateBuilder().Build()
 
 	err = ms.applyPlainUpdates(plainKeys, updates)
 	require.NoError(t, err)
 
-	hashAfterEmptyUpdate, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
+	hashAfterEmptyUpdate, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
 	require.NoError(t, err)
 
 	ms.applyBranchNodeUpdates(branchNodeUpdates)

commitment/commitment.go

@@ -24,9 +24,10 @@ type Trie interface {
 	// Reset Drops everything from the trie
 	Reset()
 
-	ReviewKeys(pk, hk [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)
+	// Reads updates from storage
+	ProcessKeys(pk [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)
 
-	ProcessUpdates(pk, hk [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)
+	ProcessUpdates(pk [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)
 
 	ResetFns(
 		branchFn func(prefix []byte) ([]byte, error),
@@ -70,6 +71,9 @@ const (
 type BranchData []byte
 
 func (branchData BranchData) String() string {
+	if len(branchData) == 0 {
+		return ""
+	}
 	touchMap := binary.BigEndian.Uint16(branchData[0:])
 	afterMap := binary.BigEndian.Uint16(branchData[2:])
 	pos := 4
@@ -468,10 +472,10 @@ func NewHexBranchMerger(capacity uint64) *BranchMerger {
 
 // MergeHexBranches combines two branchData, number 2 coming after (and potentially shadowing) number 1
 func (m *BranchMerger) Merge(branch1 BranchData, branch2 BranchData) (BranchData, error) {
-	if branch2 == nil {
+	if len(branch2) == 0 {
 		return branch1, nil
 	}
-	if branch1 == nil {
+	if len(branch1) == 0 {
 		return branch2, nil
 	}