Add blake2b hashing algorithm (#5926)

* Add 	golang.org/x/crypto/blake2b to vendor

* Add blake2b hashing algorithm

Allow BLAKE2b as a file hashing algorithm (https://blake2.net/). In my benchmarks
it is quite fast.

| Hash Algorithm                  | Time per 100 MiB File (ns) | MiB / sec   |
|---------------------------------|----------------------------|-------------|
| BenchmarkHashFile/blake2b_256-8 | 127116193                  | 786.6818353 |
| BenchmarkHashFile/blake2b_512-8 | 127239979                  | 785.9165082 |
| BenchmarkHashFile/blake2b_384-8 | 129671956                  | 771.1767686 |
| BenchmarkHashFile/sha1-8        | 131347484                  | 761.3392884 |
| BenchmarkHashFile/md5-8         | 170146968                  | 587.7271936 |
| BenchmarkHashFile/sha512-8      | 200803749                  | 497.9986703 |
| BenchmarkHashFile/sha384-8      | 201153073                  | 497.1338419 |
| BenchmarkHashFile/sha512_224-8  | 201987854                  | 495.0792734 |
| BenchmarkHashFile/sha512_256-8  | 202126889                  | 494.7387282 |
| BenchmarkHashFile/sha256-8      | 297884549                  | 335.7005267 |
| BenchmarkHashFile/sha224-8      | 299384125                  | 334.0190466 |
| BenchmarkHashFile/sha3_224-8    | 335496603                  | 298.0656111 |
| BenchmarkHashFile/sha3_256-8    | 352318502                  | 283.834086  |
| BenchmarkHashFile/sha3_384-8    | 461460154                  | 216.7034339 |
| BenchmarkHashFile/sha3_512-8    | 651817080                  | 153.4172747 |

CHANGELOG.asciidoc

@@ -118,6 +118,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 *Auditbeat*
 
 - Add support for SHA3 hash algorithms to the file integrity module. {issue}5345[5345]
+- Add support for BLAKE2b hash algorithms to the file integrity module. {pull}5926[5926]
 - Add dashboards for Linux audit framework events (overview, executions, sockets). {pull}5516[5516]
 - Add support for recursive file watches under macOS {pull}5575[5575] and Linux. {pull}5833[5833]
 

NOTICE.txt

@@ -3651,7 +3651,7 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/crypto
-Revision: e8f229864d71a49e5fdc4a9a134c5f85c4c33d64
+Revision: d585fd2cc9195196078f516b69daff6744ef5e84
 License type (autodetected): BSD-3-Clause
 ./vendor/golang.org/x/crypto/LICENSE:
 --------------------------------------------------------------------

auditbeat/auditbeat.reference.yml

@@ -94,9 +94,10 @@ auditbeat.modules:
   # Limit on the size of files that will be hashed. Default is "100 MiB".
   max_file_size: 100 MiB
 
-  # Hash types to compute when the file changes. Supported types are md5, sha1,
-  # sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, sha3_256,
-  # sha3_384 and sha3_512. Default is sha1.
+  # Hash types to compute when the file changes. Supported types are
+  # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384,
+  # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512.
+  # Default is sha1.
   hash_types: [sha1]
 
   # Detect changes to files included in subdirectories. Disabled by default.

auditbeat/docs/fields.asciidoc

@@ -2274,6 +2274,27 @@ type: boolean
 
 Boolean indicating if the event includes any file hashes.
 
+[float]
+=== `audit.file.blake2b_256`
+
+type: keyword
+
+BLAKE2b-256 hash of the file.
+
+[float]
+=== `audit.file.blake2b_384`
+
+type: keyword
+
+BLAKE2b-384 hash of the file.
+
+[float]
+=== `audit.file.blake2b_512`
+
+type: keyword
+
+BLAKE2b-512 hash of the file.
+
 [float]
 === `audit.file.md5`
 

auditbeat/docs/modules/file_integrity.asciidoc

@@ -99,9 +99,9 @@ a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`
 `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`.
 
 *`hash_types`*:: A list of hash types to compute when the file changes.
-The supported hash types are md5, sha1, sha224, sha256, sha384, sha512,
-sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512. The default
-value is sha1.
+The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`,
+`sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha512_224`, `sha512_256`,
+`sha3_224`, `sha3_256`, `sha3_384`, and `sha3_512`. The default value is `sha1`.
 
 *`recursive`*:: By default, the watches set to the paths specified in
 `paths` are not recursive. This means that only changes to the contents

auditbeat/module/file_integrity/_meta/config.yml.tpl

@@ -56,9 +56,10 @@
   # Limit on the size of files that will be hashed. Default is "100 MiB".
   max_file_size: 100 MiB
 
-  # Hash types to compute when the file changes. Supported types are md5, sha1,
-  # sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, sha3_256,
-  # sha3_384 and sha3_512. Default is sha1.
+  # Hash types to compute when the file changes. Supported types are
+  # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384,
+  # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512.
+  # Default is sha1.
   hash_types: [sha1]
 
   # Detect changes to files included in subdirectories. Disabled by default.

auditbeat/module/file_integrity/_meta/docs.asciidoc

@@ -94,9 +94,9 @@ a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`
 `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`.
 
 *`hash_types`*:: A list of hash types to compute when the file changes.
-The supported hash types are md5, sha1, sha224, sha256, sha384, sha512,
-sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512. The default
-value is sha1.
+The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`,
+`sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha512_224`, `sha512_256`,
+`sha3_224`, `sha3_256`, `sha3_384`, and `sha3_512`. The default value is `sha1`.
 
 *`recursive`*:: By default, the watches set to the paths specified in
 `paths` are not recursive. This means that only changes to the contents

auditbeat/module/file_integrity/_meta/fields.yml

@@ -83,6 +83,18 @@
         type: boolean
         description: Boolean indicating if the event includes any file hashes.
 
+      - name: blake2b_256
+        type: keyword
+        description: BLAKE2b-256 hash of the file.
+
+      - name: blake2b_384
+        type: keyword
+        description: BLAKE2b-384 hash of the file.
+
+      - name: blake2b_512
+        type: keyword
+        description: BLAKE2b-512 hash of the file.
+
       - name: md5
         type: keyword
         description: MD5 hash of the file.

auditbeat/module/file_integrity/config.go

@@ -21,22 +21,31 @@ func (t *HashType) Unpack(v string) error {
 	return nil
 }
 
-var validHashes = []HashType{MD5, SHA1, SHA224, SHA256, SHA384, SHA3_224, SHA3_256, SHA3_384, SHA3_512, SHA512, SHA512_224, SHA512_256}
+var validHashes = []HashType{
+	BLAKE2B_256, BLAKE2B_384, BLAKE2B_512,
+	MD5,
+	SHA1,
+	SHA224, SHA256, SHA384, SHA512, SHA512_224, SHA512_256,
+	SHA3_224, SHA3_256, SHA3_384, SHA3_512,
+}
 
 // Enum of hash types.
 const (
-	MD5        HashType = "md5"
-	SHA1       HashType = "sha1"
-	SHA224     HashType = "sha224"
-	SHA256     HashType = "sha256"
-	SHA384     HashType = "sha384"
-	SHA3_224   HashType = "sha3_224"
-	SHA3_256   HashType = "sha3_256"
-	SHA3_384   HashType = "sha3_384"
-	SHA3_512   HashType = "sha3_512"
-	SHA512     HashType = "sha512"
-	SHA512_224 HashType = "sha512_224"
-	SHA512_256 HashType = "sha512_256"
+	BLAKE2B_256 HashType = "blake2b_256"
+	BLAKE2B_384 HashType = "blake2b_384"
+	BLAKE2B_512 HashType = "blake2b_512"
+	MD5         HashType = "md5"
+	SHA1        HashType = "sha1"
+	SHA224      HashType = "sha224"
+	SHA256      HashType = "sha256"
+	SHA384      HashType = "sha384"
+	SHA3_224    HashType = "sha3_224"
+	SHA3_256    HashType = "sha3_256"
+	SHA3_384    HashType = "sha3_384"
+	SHA3_512    HashType = "sha3_512"
+	SHA512      HashType = "sha512"
+	SHA512_224  HashType = "sha512_224"
+	SHA512_256  HashType = "sha512_256"
 )
 
 // Config contains the configuration parameters for the file integrity

auditbeat/module/file_integrity/event.go

@@ -18,7 +18,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-
+	"golang.org/x/crypto/blake2b"
 	"golang.org/x/crypto/sha3"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -323,6 +323,15 @@ func hashFile(name string, hashType ...HashType) (map[HashType][]byte, error) {
 	var hashes []hash.Hash
 	for _, name := range hashType {
 		switch name {
+		case BLAKE2B_256:
+			h, _ := blake2b.New256(nil)
+			hashes = append(hashes, h)
+		case BLAKE2B_384:
+			h, _ := blake2b.New384(nil)
+			hashes = append(hashes, h)
+		case BLAKE2B_512:
+			h, _ := blake2b.New512(nil)
+			hashes = append(hashes, h)
 		case MD5:
 			hashes = append(hashes, md5.New())
 		case SHA1:

auditbeat/module/file_integrity/event_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"
 
+	"bytes"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -153,18 +155,21 @@ func TestHashFile(t *testing.T) {
 	t.Run("valid hashes", func(t *testing.T) {
 		// Computed externally.
 		expectedHashes := map[HashType][]byte{
-			MD5:        mustDecodeHex("c897d1410af8f2c74fba11b1db511e9e"),
-			SHA1:       mustDecodeHex("f951b101989b2c3b7471710b4e78fc4dbdfa0ca6"),
-			SHA224:     mustDecodeHex("d301812e62eec9b1e68c0b861e62f374e0d77e8365f5ddd6cccc8693"),
-			SHA256:     mustDecodeHex("ecf701f727d9e2d77c4aa49ac6fbbcc997278aca010bddeeb961c10cf54d435a"),
-			SHA384:     mustDecodeHex("ec8d147738b2e4bf6f5c5ac50a9a7593fb1ee2de01474d6f8a6c7fdb7ac945580772a5225a4c7251a7c0697acb7b8405"),
-			SHA512:     mustDecodeHex("f5408390735bf3ef0bb8aaf66eff4f8ca716093d2fec50996b479b3527e5112e3ea3b403e9e62c72155ac1e08a49b476f43ab621e1a5fc2bbb0559d8258a614d"),
-			SHA512_224: mustDecodeHex("fde054253f43a95559f1b6eeb8e2edba4124957b43b85d7fcb4d20d5"),
-			SHA512_256: mustDecodeHex("3380f6a625aac19cbdddc598ab07aea195bae000f8d4c8cd6bb8870ac25df15d"),
-			SHA3_224:   mustDecodeHex("62e3515dae95bbd0e105bee840b7dc3b47f6d6bc772c259dbc0da31a"),
-			SHA3_256:   mustDecodeHex("3cb5385a2987ca45888d7877fbcf92b4854f7155ae19c96cecc7ea1300c6f5a4"),
-			SHA3_384:   mustDecodeHex("f19539818b4f29fa0ee599db4113fd81b77cd1119682e6d799a052849d2e40ef0dad84bc947ba2dee742d9731f1b9e9b"),
-			SHA3_512:   mustDecodeHex("f0a2c0f9090c1fd6dedf211192e36a6668d2b3c7f57a35419acb1c4fc7dfffc267bbcd90f5f38676caddcab652f6aacd1ed4e0ad0a8e1e4b98f890b62b6c7c5c"),
+			BLAKE2B_256: mustDecodeHex("0f0cc1f0ea4ef962d6a150ae0b77bc320b57ed24e1609b933fa2274484f59145"),
+			BLAKE2B_384: mustDecodeHex("b819d90f648da6effff2393acb1884d2638642b3524c329832c073c9364149fcdedb522914ef9c2c92f007a42366139a"),
+			BLAKE2B_512: mustDecodeHex("fc13029e8a5ce67ad5a70f0cc659a4b30df9d791b125835e434606c6127ee37ebbc8b216389682ddfa84380789db09f2535d2a9837454414ea3ff00ec0801150"),
+			MD5:         mustDecodeHex("c897d1410af8f2c74fba11b1db511e9e"),
+			SHA1:        mustDecodeHex("f951b101989b2c3b7471710b4e78fc4dbdfa0ca6"),
+			SHA224:      mustDecodeHex("d301812e62eec9b1e68c0b861e62f374e0d77e8365f5ddd6cccc8693"),
+			SHA256:      mustDecodeHex("ecf701f727d9e2d77c4aa49ac6fbbcc997278aca010bddeeb961c10cf54d435a"),
+			SHA384:      mustDecodeHex("ec8d147738b2e4bf6f5c5ac50a9a7593fb1ee2de01474d6f8a6c7fdb7ac945580772a5225a4c7251a7c0697acb7b8405"),
+			SHA512:      mustDecodeHex("f5408390735bf3ef0bb8aaf66eff4f8ca716093d2fec50996b479b3527e5112e3ea3b403e9e62c72155ac1e08a49b476f43ab621e1a5fc2bbb0559d8258a614d"),
+			SHA512_224:  mustDecodeHex("fde054253f43a95559f1b6eeb8e2edba4124957b43b85d7fcb4d20d5"),
+			SHA512_256:  mustDecodeHex("3380f6a625aac19cbdddc598ab07aea195bae000f8d4c8cd6bb8870ac25df15d"),
+			SHA3_224:    mustDecodeHex("62e3515dae95bbd0e105bee840b7dc3b47f6d6bc772c259dbc0da31a"),
+			SHA3_256:    mustDecodeHex("3cb5385a2987ca45888d7877fbcf92b4854f7155ae19c96cecc7ea1300c6f5a4"),
+			SHA3_384:    mustDecodeHex("f19539818b4f29fa0ee599db4113fd81b77cd1119682e6d799a052849d2e40ef0dad84bc947ba2dee742d9731f1b9e9b"),
+			SHA3_512:    mustDecodeHex("f0a2c0f9090c1fd6dedf211192e36a6668d2b3c7f57a35419acb1c4fc7dfffc267bbcd90f5f38676caddcab652f6aacd1ed4e0ad0a8e1e4b98f890b62b6c7c5c"),
 		}
 
 		f, err := ioutil.TempFile("", "input.txt")
@@ -191,7 +196,10 @@ func TestHashFile(t *testing.T) {
 				if !ok {
 					t.Fatalf("hash type not found in expected hashes: %v", hashType)
 				}
-				assert.Equal(t, expected, hash, "%v hash incorrect", hashType)
+				if !bytes.Equal(expected, hash) {
+					t.Errorf("%v hash incorrect, got: %v, want: %v", hashType,
+						hex.EncodeToString(hash), hex.EncodeToString(expected))
+				}
 			}
 		}
 

auditbeat/module/file_integrity/flatbuffers.go

@@ -65,6 +65,12 @@ func fbWriteHash(b *flatbuffers.Builder, hashes map[HashType][]byte) flatbuffers
 	schema.HashStart(b)
 	for hashType, offset := range offsets {
 		switch hashType {
+		case BLAKE2B_256:
+			schema.HashAddBlake2b256(b, offset)
+		case BLAKE2B_384:
+			schema.HashAddBlake2b384(b, offset)
+		case BLAKE2B_512:
+			schema.HashAddBlake2b512(b, offset)
 		case MD5:
 			schema.HashAddMd5(b, offset)
 		case SHA1:
@@ -258,6 +264,15 @@ func fbDecodeHash(e *schema.Event) map[HashType][]byte {
 		var producer func(i int) int8
 
 		switch hashType {
+		case BLAKE2B_256:
+			length = hash.Blake2b256Length()
+			producer = hash.Blake2b256
+		case BLAKE2B_384:
+			length = hash.Blake2b384Length()
+			producer = hash.Blake2b384
+		case BLAKE2B_512:
+			length = hash.Blake2b512Length()
+			producer = hash.Blake2b512
 		case MD5:
 			length = hash.Md5Length()
 			producer = hash.Md5

auditbeat/module/file_integrity/schema.fbs

@@ -52,6 +52,11 @@ table Hash {
   sha3_256:[byte];
   sha3_384:[byte];
   sha3_512:[byte];
+
+  // Blake2b
+  blake2b_256: [byte];
+  blake2b_384: [byte];
+  blake2b_512: [byte];
 }
 
 table Event {

auditbeat/module/file_integrity/schema/Hash.go

@@ -230,8 +230,59 @@ func (rcv *Hash) Sha3512Length() int {
 	return 0
 }
 
+func (rcv *Hash) Blake2b256(j int) int8 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(28))
+	if o != 0 {
+		a := rcv._tab.Vector(o)
+		return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1))
+	}
+	return 0
+}
+
+func (rcv *Hash) Blake2b256Length() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(28))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
+func (rcv *Hash) Blake2b384(j int) int8 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
+	if o != 0 {
+		a := rcv._tab.Vector(o)
+		return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1))
+	}
+	return 0
+}
+
+func (rcv *Hash) Blake2b384Length() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
+func (rcv *Hash) Blake2b512(j int) int8 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
+	if o != 0 {
+		a := rcv._tab.Vector(o)
+		return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1))
+	}
+	return 0
+}
+
+func (rcv *Hash) Blake2b512Length() int {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
+	if o != 0 {
+		return rcv._tab.VectorLen(o)
+	}
+	return 0
+}
+
 func HashStart(builder *flatbuffers.Builder) {
-	builder.StartObject(12)
+	builder.StartObject(15)
 }
 func HashAddMd5(builder *flatbuffers.Builder, md5 flatbuffers.UOffsetT) {
 	builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(md5), 0)
@@ -305,6 +356,24 @@ func HashAddSha3512(builder *flatbuffers.Builder, sha3512 flatbuffers.UOffsetT)
 func HashStartSha3512Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
 	return builder.StartVector(1, numElems, 1)
 }
+func HashAddBlake2b256(builder *flatbuffers.Builder, blake2b256 flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(12, flatbuffers.UOffsetT(blake2b256), 0)
+}
+func HashStartBlake2b256Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(1, numElems, 1)
+}
+func HashAddBlake2b384(builder *flatbuffers.Builder, blake2b384 flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(blake2b384), 0)
+}
+func HashStartBlake2b384Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(1, numElems, 1)
+}
+func HashAddBlake2b512(builder *flatbuffers.Builder, blake2b512 flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(14, flatbuffers.UOffsetT(blake2b512), 0)
+}
+func HashStartBlake2b512Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+	return builder.StartVector(1, numElems, 1)
+}
 func HashEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
 	return builder.EndObject()
 }