Skip to content

Commit

Permalink
Add blake2b hashing algorithm
Browse files Browse the repository at this point in the history
Allow BLAKE2b as a file hashing algorithm (https://blake2.net/). In my benchmarks
it is quite fast.

| Hash Algorithm                  | Time per 100 MiB File (ns) | MiB / sec   |
|---------------------------------|----------------------------|-------------|
| BenchmarkHashFile/blake2b_256-8 | 127116193                  | 786.6818353 |
| BenchmarkHashFile/blake2b_512-8 | 127239979                  | 785.9165082 |
| BenchmarkHashFile/blake2b_384-8 | 129671956                  | 771.1767686 |
| BenchmarkHashFile/sha1-8        | 131347484                  | 761.3392884 |
| BenchmarkHashFile/md5-8         | 170146968                  | 587.7271936 |
| BenchmarkHashFile/sha512-8      | 200803749                  | 497.9986703 |
| BenchmarkHashFile/sha384-8      | 201153073                  | 497.1338419 |
| BenchmarkHashFile/sha512_224-8  | 201987854                  | 495.0792734 |
| BenchmarkHashFile/sha512_256-8  | 202126889                  | 494.7387282 |
| BenchmarkHashFile/sha256-8      | 297884549                  | 335.7005267 |
| BenchmarkHashFile/sha224-8      | 299384125                  | 334.0190466 |
| BenchmarkHashFile/sha3_224-8    | 335496603                  | 298.0656111 |
| BenchmarkHashFile/sha3_256-8    | 352318502                  | 283.834086  |
| BenchmarkHashFile/sha3_384-8    | 461460154                  | 216.7034339 |
| BenchmarkHashFile/sha3_512-8    | 651817080                  | 153.4172747 |
  • Loading branch information
andrewkroh committed Dec 19, 2017
1 parent 64f429b commit 7abe013
Show file tree
Hide file tree
Showing 14 changed files with 192 additions and 41 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
*Auditbeat*

- Add support for SHA3 hash algorithms to the file integrity module. {issue}5345[5345]
- Add support for BLAKE2b hash algorithms to the file integrity module. {pull}5926[5926]
- Add dashboards for Linux audit framework events (overview, executions, sockets). {pull}5516[5516]
- Add support for recursive file watches under macOS {pull}5575[5575] and Linux. {pull}5833[5833]

Expand Down
2 changes: 1 addition & 1 deletion NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3651,7 +3651,7 @@ THE SOFTWARE.

--------------------------------------------------------------------
Dependency: golang.org/x/crypto
Revision: e8f229864d71a49e5fdc4a9a134c5f85c4c33d64
Revision: d585fd2cc9195196078f516b69daff6744ef5e84
License type (autodetected): BSD-3-Clause
./vendor/golang.org/x/crypto/LICENSE:
--------------------------------------------------------------------
Expand Down
7 changes: 4 additions & 3 deletions auditbeat/auditbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,10 @@ auditbeat.modules:
# Limit on the size of files that will be hashed. Default is "100 MiB".
max_file_size: 100 MiB

# Hash types to compute when the file changes. Supported types are md5, sha1,
# sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, sha3_256,
# sha3_384 and sha3_512. Default is sha1.
# Hash types to compute when the file changes. Supported types are
# blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384,
# sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512.
# Default is sha1.
hash_types: [sha1]

# Detect changes to files included in subdirectories. Disabled by default.
Expand Down
21 changes: 21 additions & 0 deletions auditbeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -2256,6 +2256,27 @@ type: boolean
Boolean indicating if the event includes any file hashes.
[float]
=== `audit.file.blake2b_256`
type: keyword
BLAKE2b-256 hash of the file.
[float]
=== `audit.file.blake2b_384`
type: keyword
BLAKE2b-384 hash of the file.
[float]
=== `audit.file.blake2b_512`
type: keyword
BLAKE2b-512 hash of the file.
[float]
=== `audit.file.md5`
Expand Down
6 changes: 3 additions & 3 deletions auditbeat/docs/modules/file_integrity.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@ a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`
`mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`.

*`hash_types`*:: A list of hash types to compute when the file changes.
The supported hash types are md5, sha1, sha224, sha256, sha384, sha512,
sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512. The default
value is sha1.
The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`,
`sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha512_224`, `sha512_256`,
`sha3_224`, `sha3_256`, `sha3_384`, and `sha3_512`. The default value is `sha1`.

*`recursive`*:: By default, the watches set to the paths specified in
`paths` are not recursive. This means that only changes to the contents
Expand Down
7 changes: 4 additions & 3 deletions auditbeat/module/file_integrity/_meta/config.yml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,10 @@
# Limit on the size of files that will be hashed. Default is "100 MiB".
max_file_size: 100 MiB

# Hash types to compute when the file changes. Supported types are md5, sha1,
# sha224, sha256, sha384, sha512, sha512_224, sha512_256, sha3_224, sha3_256,
# sha3_384 and sha3_512. Default is sha1.
# Hash types to compute when the file changes. Supported types are
# blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384,
# sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512.
# Default is sha1.
hash_types: [sha1]

# Detect changes to files included in subdirectories. Disabled by default.
Expand Down
6 changes: 3 additions & 3 deletions auditbeat/module/file_integrity/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@ a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`
`mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`.

*`hash_types`*:: A list of hash types to compute when the file changes.
The supported hash types are md5, sha1, sha224, sha256, sha384, sha512,
sha512_224, sha512_256, sha3_224, sha3_256, sha3_384 and sha3_512. The default
value is sha1.
The supported hash types are `blake2b_256`, `blake2b_384`, `blake2b_512`, `md5`,
`sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha512_224`, `sha512_256`,
`sha3_224`, `sha3_256`, `sha3_384`, and `sha3_512`. The default value is `sha1`.

*`recursive`*:: By default, the watches set to the paths specified in
`paths` are not recursive. This means that only changes to the contents
Expand Down
12 changes: 12 additions & 0 deletions auditbeat/module/file_integrity/_meta/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,18 @@
type: boolean
description: Boolean indicating if the event includes any file hashes.

- name: blake2b_256
type: keyword
description: BLAKE2b-256 hash of the file.

- name: blake2b_384
type: keyword
description: BLAKE2b-384 hash of the file.

- name: blake2b_512
type: keyword
description: BLAKE2b-512 hash of the file.

- name: md5
type: keyword
description: MD5 hash of the file.
Expand Down
35 changes: 22 additions & 13 deletions auditbeat/module/file_integrity/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,31 @@ func (t *HashType) Unpack(v string) error {
return nil
}

var validHashes = []HashType{MD5, SHA1, SHA224, SHA256, SHA384, SHA3_224, SHA3_256, SHA3_384, SHA3_512, SHA512, SHA512_224, SHA512_256}
var validHashes = []HashType{
BLAKE2B_256, BLAKE2B_384, BLAKE2B_512,
MD5,
SHA1,
SHA224, SHA256, SHA384, SHA512, SHA512_224, SHA512_256,
SHA3_224, SHA3_256, SHA3_384, SHA3_512,
}

// Enum of hash types.
const (
MD5 HashType = "md5"
SHA1 HashType = "sha1"
SHA224 HashType = "sha224"
SHA256 HashType = "sha256"
SHA384 HashType = "sha384"
SHA3_224 HashType = "sha3_224"
SHA3_256 HashType = "sha3_256"
SHA3_384 HashType = "sha3_384"
SHA3_512 HashType = "sha3_512"
SHA512 HashType = "sha512"
SHA512_224 HashType = "sha512_224"
SHA512_256 HashType = "sha512_256"
BLAKE2B_256 HashType = "blake2b_256"
BLAKE2B_384 HashType = "blake2b_384"
BLAKE2B_512 HashType = "blake2b_512"
MD5 HashType = "md5"
SHA1 HashType = "sha1"
SHA224 HashType = "sha224"
SHA256 HashType = "sha256"
SHA384 HashType = "sha384"
SHA3_224 HashType = "sha3_224"
SHA3_256 HashType = "sha3_256"
SHA3_384 HashType = "sha3_384"
SHA3_512 HashType = "sha3_512"
SHA512 HashType = "sha512"
SHA512_224 HashType = "sha512_224"
SHA512_256 HashType = "sha512_256"
)

// Config contains the configuration parameters for the file integrity
Expand Down
11 changes: 10 additions & 1 deletion auditbeat/module/file_integrity/event.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"time"

"github.com/pkg/errors"

"golang.org/x/crypto/blake2b"
"golang.org/x/crypto/sha3"

"github.com/elastic/beats/libbeat/common"
Expand Down Expand Up @@ -315,6 +315,15 @@ func hashFile(name string, hashType ...HashType) (map[HashType][]byte, error) {
var hashes []hash.Hash
for _, name := range hashType {
switch name {
case BLAKE2B_256:
h, _ := blake2b.New256(nil)
hashes = append(hashes, h)
case BLAKE2B_384:
h, _ := blake2b.New384(nil)
hashes = append(hashes, h)
case BLAKE2B_512:
h, _ := blake2b.New512(nil)
hashes = append(hashes, h)
case MD5:
hashes = append(hashes, md5.New())
case SHA1:
Expand Down
34 changes: 21 additions & 13 deletions auditbeat/module/file_integrity/event_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"testing"
"time"

"bytes"

"github.com/stretchr/testify/assert"
)

Expand Down Expand Up @@ -134,18 +136,21 @@ func TestHashFile(t *testing.T) {
t.Run("valid hashes", func(t *testing.T) {
// Computed externally.
expectedHashes := map[HashType][]byte{
MD5: mustDecodeHex("c897d1410af8f2c74fba11b1db511e9e"),
SHA1: mustDecodeHex("f951b101989b2c3b7471710b4e78fc4dbdfa0ca6"),
SHA224: mustDecodeHex("d301812e62eec9b1e68c0b861e62f374e0d77e8365f5ddd6cccc8693"),
SHA256: mustDecodeHex("ecf701f727d9e2d77c4aa49ac6fbbcc997278aca010bddeeb961c10cf54d435a"),
SHA384: mustDecodeHex("ec8d147738b2e4bf6f5c5ac50a9a7593fb1ee2de01474d6f8a6c7fdb7ac945580772a5225a4c7251a7c0697acb7b8405"),
SHA512: mustDecodeHex("f5408390735bf3ef0bb8aaf66eff4f8ca716093d2fec50996b479b3527e5112e3ea3b403e9e62c72155ac1e08a49b476f43ab621e1a5fc2bbb0559d8258a614d"),
SHA512_224: mustDecodeHex("fde054253f43a95559f1b6eeb8e2edba4124957b43b85d7fcb4d20d5"),
SHA512_256: mustDecodeHex("3380f6a625aac19cbdddc598ab07aea195bae000f8d4c8cd6bb8870ac25df15d"),
SHA3_224: mustDecodeHex("62e3515dae95bbd0e105bee840b7dc3b47f6d6bc772c259dbc0da31a"),
SHA3_256: mustDecodeHex("3cb5385a2987ca45888d7877fbcf92b4854f7155ae19c96cecc7ea1300c6f5a4"),
SHA3_384: mustDecodeHex("f19539818b4f29fa0ee599db4113fd81b77cd1119682e6d799a052849d2e40ef0dad84bc947ba2dee742d9731f1b9e9b"),
SHA3_512: mustDecodeHex("f0a2c0f9090c1fd6dedf211192e36a6668d2b3c7f57a35419acb1c4fc7dfffc267bbcd90f5f38676caddcab652f6aacd1ed4e0ad0a8e1e4b98f890b62b6c7c5c"),
BLAKE2B_256: mustDecodeHex("0f0cc1f0ea4ef962d6a150ae0b77bc320b57ed24e1609b933fa2274484f59145"),
BLAKE2B_384: mustDecodeHex("b819d90f648da6effff2393acb1884d2638642b3524c329832c073c9364149fcdedb522914ef9c2c92f007a42366139a"),
BLAKE2B_512: mustDecodeHex("fc13029e8a5ce67ad5a70f0cc659a4b30df9d791b125835e434606c6127ee37ebbc8b216389682ddfa84380789db09f2535d2a9837454414ea3ff00ec0801150"),
MD5: mustDecodeHex("c897d1410af8f2c74fba11b1db511e9e"),
SHA1: mustDecodeHex("f951b101989b2c3b7471710b4e78fc4dbdfa0ca6"),
SHA224: mustDecodeHex("d301812e62eec9b1e68c0b861e62f374e0d77e8365f5ddd6cccc8693"),
SHA256: mustDecodeHex("ecf701f727d9e2d77c4aa49ac6fbbcc997278aca010bddeeb961c10cf54d435a"),
SHA384: mustDecodeHex("ec8d147738b2e4bf6f5c5ac50a9a7593fb1ee2de01474d6f8a6c7fdb7ac945580772a5225a4c7251a7c0697acb7b8405"),
SHA512: mustDecodeHex("f5408390735bf3ef0bb8aaf66eff4f8ca716093d2fec50996b479b3527e5112e3ea3b403e9e62c72155ac1e08a49b476f43ab621e1a5fc2bbb0559d8258a614d"),
SHA512_224: mustDecodeHex("fde054253f43a95559f1b6eeb8e2edba4124957b43b85d7fcb4d20d5"),
SHA512_256: mustDecodeHex("3380f6a625aac19cbdddc598ab07aea195bae000f8d4c8cd6bb8870ac25df15d"),
SHA3_224: mustDecodeHex("62e3515dae95bbd0e105bee840b7dc3b47f6d6bc772c259dbc0da31a"),
SHA3_256: mustDecodeHex("3cb5385a2987ca45888d7877fbcf92b4854f7155ae19c96cecc7ea1300c6f5a4"),
SHA3_384: mustDecodeHex("f19539818b4f29fa0ee599db4113fd81b77cd1119682e6d799a052849d2e40ef0dad84bc947ba2dee742d9731f1b9e9b"),
SHA3_512: mustDecodeHex("f0a2c0f9090c1fd6dedf211192e36a6668d2b3c7f57a35419acb1c4fc7dfffc267bbcd90f5f38676caddcab652f6aacd1ed4e0ad0a8e1e4b98f890b62b6c7c5c"),
}

f, err := ioutil.TempFile("", "input.txt")
Expand All @@ -172,7 +177,10 @@ func TestHashFile(t *testing.T) {
if !ok {
t.Fatalf("hash type not found in expected hashes: %v", hashType)
}
assert.Equal(t, expected, hash, "%v hash incorrect", hashType)
if !bytes.Equal(expected, hash) {
t.Errorf("%v hash incorrect, got: %v, want: %v", hashType,
hex.EncodeToString(hash), hex.EncodeToString(expected))
}
}
}

Expand Down
15 changes: 15 additions & 0 deletions auditbeat/module/file_integrity/flatbuffers.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ func fbWriteHash(b *flatbuffers.Builder, hashes map[HashType][]byte) flatbuffers
schema.HashStart(b)
for hashType, offset := range offsets {
switch hashType {
case BLAKE2B_256:
schema.HashAddBlake2b256(b, offset)
case BLAKE2B_384:
schema.HashAddBlake2b384(b, offset)
case BLAKE2B_512:
schema.HashAddBlake2b512(b, offset)
case MD5:
schema.HashAddMd5(b, offset)
case SHA1:
Expand Down Expand Up @@ -249,6 +255,15 @@ func fbDecodeHash(e *schema.Event) map[HashType][]byte {
var producer func(i int) int8

switch hashType {
case BLAKE2B_256:
length = hash.Blake2b256Length()
producer = hash.Blake2b256
case BLAKE2B_384:
length = hash.Blake2b384Length()
producer = hash.Blake2b384
case BLAKE2B_512:
length = hash.Blake2b512Length()
producer = hash.Blake2b512
case MD5:
length = hash.Md5Length()
producer = hash.Md5
Expand Down
5 changes: 5 additions & 0 deletions auditbeat/module/file_integrity/schema.fbs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ table Hash {
sha3_256:[byte];
sha3_384:[byte];
sha3_512:[byte];

// Blake2b
blake2b_256: [byte];
blake2b_384: [byte];
blake2b_512: [byte];
}

table Event {
Expand Down
71 changes: 70 additions & 1 deletion auditbeat/module/file_integrity/schema/Hash.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,8 +230,59 @@ func (rcv *Hash) Sha3512Length() int {
return 0
}

func (rcv *Hash) Blake2b256(j int) int8 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(28))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1))
}
return 0
}

func (rcv *Hash) Blake2b256Length() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(28))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}

func (rcv *Hash) Blake2b384(j int) int8 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1))
}
return 0
}

func (rcv *Hash) Blake2b384Length() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}

func (rcv *Hash) Blake2b512(j int) int8 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1))
}
return 0
}

func (rcv *Hash) Blake2b512Length() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}

func HashStart(builder *flatbuffers.Builder) {
builder.StartObject(12)
builder.StartObject(15)
}
func HashAddMd5(builder *flatbuffers.Builder, md5 flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(md5), 0)
Expand Down Expand Up @@ -305,6 +356,24 @@ func HashAddSha3512(builder *flatbuffers.Builder, sha3512 flatbuffers.UOffsetT)
func HashStartSha3512Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func HashAddBlake2b256(builder *flatbuffers.Builder, blake2b256 flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(12, flatbuffers.UOffsetT(blake2b256), 0)
}
func HashStartBlake2b256Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func HashAddBlake2b384(builder *flatbuffers.Builder, blake2b384 flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(blake2b384), 0)
}
func HashStartBlake2b384Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func HashAddBlake2b512(builder *flatbuffers.Builder, blake2b512 flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(14, flatbuffers.UOffsetT(blake2b512), 0)
}
func HashStartBlake2b512Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func HashEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

0 comments on commit 7abe013

Please sign in to comment.