Use gohashtree for hashing (#95)

* Test

* Simplify doHash

* Add Hash elem

* Combine hash and gohashtree

* Remove comment

* Update Changelog

* Update README

CHANGELOG.md

@@ -1,5 +1,6 @@
 # 0.1.2 (Unreleased)
 
+- feat: Add `HashFn` abstraction and introduce `gohashtree` hashing [[GH-95](https://github.com/ferranbt/fastssz/issues/95)]
 - feat: `sszgen` for alias to byte array [[GH-55](https://github.com/ferranbt/fastssz/issues/55)]
 - feat: `sszgen` include version in generated header file [[GH-101](https://github.com/ferranbt/fastssz/issues/101)]
 - feat: support `time.Time` type as native object [[GH-100](https://github.com/ferranbt/fastssz/issues/100)]

README.md

@@ -62,19 +62,21 @@ goos: linux
 goarch: amd64
 pkg: github.com/ferranbt/fastssz/spectests
 cpu: AMD Ryzen 5 2400G with Radeon Vega Graphics
-BenchmarkMarshalFast
-BenchmarkMarshalFast-8        	  268454	      4166 ns/op	    8192 B/op	       1 allocs/op
-BenchmarkMarshalSuperFast
-BenchmarkMarshalSuperFast-8   	  883546	      1226 ns/op	       0 B/op	       0 allocs/op
-BenchmarkUnMarshalFast
-BenchmarkUnMarshalFast-8      	   67159	     17772 ns/op	   11900 B/op	     210 allocs/op
-BenchmarkHashTreeRootFast
-BenchmarkHashTreeRootFast-8   	   24508	     45571 ns/op	       0 B/op	       0 allocs/op
+BenchmarkMarshal_Fast
+BenchmarkMarshal_Fast-8             	  291054	      4088 ns/op	    8192 B/op	       1 allocs/op
+BenchmarkMarshal_SuperFast
+BenchmarkMarshal_SuperFast-8        	  798883	      1354 ns/op	       0 B/op	       0 allocs/op
+BenchmarkUnMarshal_Fast
+BenchmarkUnMarshal_Fast-8           	   64065	     17614 ns/op	   11900 B/op	     210 allocs/op
+BenchmarkHashTreeRoot_Fast
+BenchmarkHashTreeRoot_Fast-8        	   25863	     45932 ns/op	       0 B/op	       0 allocs/op
+BenchmarkHashTreeRoot_SuperFast
+BenchmarkHashTreeRoot_SuperFast-8   	   54078	     21999 ns/op	       0 B/op	       0 allocs/op
 PASS
 ok  	github.com/ferranbt/fastssz/spectests	5.501s
 ```
 
-# Package reference
+## Package reference
 
 To reference a struct from another package use the '--include' flag to point to that package.
 
@@ -91,3 +93,9 @@ There are some caveats required to use this functionality.
 
 - If multiple input paths import the same package, all of them need to import it with the same alias if any.
 - If the folder of the package is not the same as the name of the package, any input file that imports this package needs to do it with an alias.
+
+## Fast HashTreeRoot
+
+`Fastssz` integrates with Prysm [gohashtree](https://github.com/prysmaticlabs/gohashtree) library to do high performance and concurrent Sha256 hashing. It achieves a 2x performance improvement with respect to the normal sequential hashing. As of now, this feature is not yet enabled by default since it does not use the `gohashtree` main branch. You can track the updates on [this](https://github.com/prysmaticlabs/gohashtree/issues/4) issue.
+
+In order to use this feature, enable manually the hash function in the Hasher like in the benchmark example.

go.mod

@@ -4,7 +4,12 @@ go 1.18
 
 require (
 	github.com/golang/snappy v0.0.3
-	github.com/minio/sha256-simd v0.1.1
+	github.com/minio/sha256-simd v1.0.0
 	github.com/mitchellh/mapstructure v1.3.2
 	gopkg.in/yaml.v2 v2.3.0
 )
+
+require (
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 // indirect
+)

go.sum

@@ -1,9 +1,14 @@
 github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
-github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg=
 github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw=
+github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48/go.mod h1:4pWaT30XoEx1j8KNJf3TV+E3mQkaufn7mf+jRNb/Fuk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=

hash.go

@@ -0,0 +1,25 @@
+package ssz
+
+import "hash"
+
+type HashFn func(dst []byte, input []byte) error
+
+func NativeHashWrapper(hashFn hash.Hash) HashFn {
+	return func(dst []byte, input []byte) error {
+		hash := func(dst []byte, src []byte) {
+			hashFn.Write(src[:32])
+			hashFn.Write(src[32:64])
+			hashFn.Sum(dst)
+			hashFn.Reset()
+		}
+
+		layerLen := len(input) / 32
+		if layerLen%2 == 1 {
+			layerLen++
+		}
+		for i := 0; i < layerLen; i += 2 {
+			hash(input[(i/2)*32:][:0], input[i*32:])
+		}
+		return nil
+	}
+}

hasher.go

@@ -68,19 +68,21 @@ type Hasher struct {
 	tmp []byte
 
 	// sha256 hash function
-	hash hash.Hash
+	hash HashFn
 }
 
-// NewHasher creates a new Hasher object
+// NewHasher creates a new Hasher object with sha256 hash
 func NewHasher() *Hasher {
-	return &Hasher{
-		hash: sha256.New(),
-		tmp:  make([]byte, 32),
-	}
+	return NewHasherWithHash(sha256.New())
 }
 
-// NewHasher creates a new Hasher object with a custom hash function
+// NewHasherWithHash creates a new Hasher object with a custom hash.Hash function
 func NewHasherWithHash(hh hash.Hash) *Hasher {
+	return NewHasherWithHashFn(NativeHashWrapper(hh))
+}
+
+// NewHasherWithHashFn creates a new Hasher object with a custom HashFn function
+func NewHasherWithHashFn(hh HashFn) *Hasher {
 	return &Hasher{
 		hash: hh,
 		tmp:  make([]byte, 32),
@@ -90,7 +92,6 @@ func NewHasherWithHash(hh hash.Hash) *Hasher {
 // Reset resets the Hasher obj
 func (h *Hasher) Reset() {
 	h.buf = h.buf[:0]
-	h.hash.Reset()
 }
 
 func (h *Hasher) AppendBytes32(b []byte) {
@@ -278,9 +279,11 @@ func (h *Hasher) MerkleizeWithMixin(indx int, num, limit uint64) {
 		output[indx] = 0
 	}
 	MarshalUint64(output[:0], num)
+	input = append(input, output...)
 
-	input = h.doHash(input, input, output)
-	h.buf = append(h.buf[:indx], input...)
+	// input is of the form [<input><size>] of 64 bytes
+	h.hash(input, input)
+	h.buf = append(h.buf[:indx], input[:32]...)
 }
 
 func (h *Hasher) Hash() []byte {
@@ -336,14 +339,6 @@ func getDepth(d uint64) uint8 {
 	return 64 - uint8(bits.LeadingZeros(i)) - 1
 }
 
-func (h *Hasher) doHash(dst []byte, a []byte, b []byte) []byte {
-	h.hash.Write(a)
-	h.hash.Write(b)
-	h.hash.Sum(dst[:0])
-	h.hash.Reset()
-	return dst
-}
-
 func (h *Hasher) merkleizeImpl(dst []byte, input []byte, limit uint64) []byte {
 	count := uint64(len(input) / 32)
 	if limit == 0 {
@@ -367,10 +362,6 @@ func (h *Hasher) merkleizeImpl(dst []byte, input []byte, limit uint64) []byte {
 		return append(dst, zeroHashes[depth][:]...)
 	}
 
-	getPos := func(i int) []byte {
-		return input[i*32 : i*32+32]
-	}
-
 	for i := uint8(0); i < depth; i++ {
 		layerLen := len(input) / 32
 		oddNodeLength := layerLen%2 == 1
@@ -382,9 +373,8 @@ func (h *Hasher) merkleizeImpl(dst []byte, input []byte, limit uint64) []byte {
 		}
 
 		outputLen := (layerLen / 2) * 32
-		for i := 0; i < layerLen; i += 2 {
-			h.doHash(getPos(i/2), getPos(i), getPos(i+1))
-		}
+
+		h.hash(input, input)
 		input = input[:outputLen]
 	}
 

hasher_test.go

@@ -1,7 +1,10 @@
 package ssz
 
 import (
+	"fmt"
 	"testing"
+
+	"github.com/prysmaticlabs/gohashtree"
 )
 
 func TestDepth(t *testing.T) {
@@ -53,3 +56,19 @@ func TestNextPowerOfTwo(t *testing.T) {
 		}
 	}
 }
+
+func TestHashGoHashTree(t *testing.T) {
+
+	a := make([]byte, 32)
+	b := make([]byte, 32)
+	a[0] = 1
+	b[0] = 2
+
+	buf := []byte{}
+	buf = append(buf, a...)
+	buf = append(buf, b...)
+
+	gohashtree.Hash(buf, buf)
+
+	fmt.Println(buf)
+}

spectests/structs_test.go

@@ -11,6 +11,7 @@ import (
 
 	ssz "github.com/ferranbt/fastssz"
 	"github.com/golang/snappy"
+	"github.com/prysmaticlabs/gohashtree"
 
 	"gopkg.in/yaml.v2"
 )
@@ -165,6 +166,7 @@ func checkSSZEncoding(t *testing.T, fork fork, fileName, structName string, base
 		fatal("HashTreeRoot", err)
 	}
 	if !bytes.Equal(root[:], output.root) {
+		//fmt.Println("- bad root -")
 		fatal("HashTreeRoot_equal", fmt.Errorf("bad root"))
 	}
 
@@ -186,7 +188,7 @@ func checkSSZEncoding(t *testing.T, fork fork, fileName, structName string, base
 
 const benchmarkTestCase = "../eth2.0-spec-tests/tests/mainnet/phase0/ssz_static/BeaconBlock/ssz_random/case_4"
 
-func BenchmarkMarshalFast(b *testing.B) {
+func BenchmarkMarshal_Fast(b *testing.B) {
 	obj := new(BeaconBlock)
 	readValidGenericSSZ(nil, benchmarkTestCase, obj)
 
@@ -198,7 +200,7 @@ func BenchmarkMarshalFast(b *testing.B) {
 	}
 }
 
-func BenchmarkMarshalSuperFast(b *testing.B) {
+func BenchmarkMarshal_SuperFast(b *testing.B) {
 	obj := new(BeaconBlock)
 	readValidGenericSSZ(nil, benchmarkTestCase, obj)
 
@@ -212,7 +214,7 @@ func BenchmarkMarshalSuperFast(b *testing.B) {
 	}
 }
 
-func BenchmarkUnMarshalFast(b *testing.B) {
+func BenchmarkUnMarshal_Fast(b *testing.B) {
 	obj := new(BeaconBlock)
 	readValidGenericSSZ(nil, benchmarkTestCase, obj)
 
@@ -232,14 +234,28 @@ func BenchmarkUnMarshalFast(b *testing.B) {
 	}
 }
 
-func BenchmarkHashTreeRootFast(b *testing.B) {
+func BenchmarkHashTreeRoot_Fast(b *testing.B) {
 	obj := new(BeaconBlock)
 	readValidGenericSSZ(nil, benchmarkTestCase, obj)
 
 	b.ReportAllocs()
 	b.ResetTimer()
 
-	hh := ssz.DefaultHasherPool.Get()
+	hh := ssz.NewHasher()
+	for i := 0; i < b.N; i++ {
+		obj.HashTreeRootWith(hh)
+		hh.Reset()
+	}
+}
+
+func BenchmarkHashTreeRoot_SuperFast(b *testing.B) {
+	obj := new(BeaconBlock)
+	readValidGenericSSZ(nil, benchmarkTestCase, obj)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	hh := ssz.NewHasherWithHashFn(gohashtree.Hash)
 	for i := 0; i < b.N; i++ {
 		obj.HashTreeRootWith(hh)
 		hh.Reset()