Set source.bytes/packets for uni-directional netflow (elastic#14111)

This populates the `source.bytes` and `source.packets` fields for uni-directional netflow events. Previously only `network.bytes`/`network.packets` would be set. The input would already populate the source fields for bi-directional flows. This also fixes an issue where the totals in `network.bytes` and `network.packets` were incorrectly calculated for bi-directional flows. Closes elastic#11473 (cherry picked from commit cbc040a)
andrewkroh · Oct 21, 2019 · 3f2c53c · 3f2c53c
1 parent fced999
commit 3f2c53c
Show file tree

Hide file tree

Showing 46 changed files with 1,621 additions and 501 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Filebeat*
 
 - Fix azure fields names. {pull}14098[14098] {pull}14132[14132]
+- Fix calculation of `network.bytes` and `network.packets` for bi-directional netflow events. {pull}14111[14111]
 
 *Heartbeat*
 
@@ -80,6 +81,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358]
 - Add filebeat azure module with activitylogs, auditlogs, signinlogs filesets. {pull}13776[13776] {pull}14033[14033] {pull}14107[14107]
 - Add support for all the ObjectCreated events in S3 input. {pull}14077[14077]
+- Add `source.bytes` and `source.packets` for uni-directional netflow events. {pull}14111[14111]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/input/netflow/convert.go b/x-pack/filebeat/input/netflow/convert.go
@@ -262,26 +262,23 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
 		revPkts, hasRevPkts = getKeyUint64(flow.Fields, "reversePacketTotalCount")
 	}
 
-	if hasRevBytes || hasRevPkts {
-		if hasBytes {
-			ecsSource["bytes"] = countBytes
-			ecsDest["bytes"] = revBytes
-		}
-		if hasPkts {
-			ecsSource["packets"] = revBytes
-			ecsDest["packets"] = revPkts
-		}
-		countBytes += revBytes
-		countPkts += revPkts
+	if hasRevBytes {
+		ecsDest["bytes"] = revBytes
+	}
+
+	if hasRevPkts {
+		ecsDest["packets"] = revPkts
 	}
 
 	if hasBytes {
+		ecsSource["bytes"] = countBytes
 		if hasRevBytes {
 			countBytes += revBytes
 		}
 		ecsNetwork["bytes"] = countBytes
 	}
 	if hasPkts {
+		ecsSource["packets"] = countPkts
 		if hasRevPkts {
 			countPkts += revPkts
 		}

diff --git a/x-pack/filebeat/input/netflow/decoder/examples/examples b/x-pack/filebeat/input/netflow/decoder/examples/examples
diff --git a/...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json b/...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json
@@ -71,13 +71,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.236.5.4",
           "locality": "private",
           "mac": "00:50:56:b9:26:46",
+          "packets": 0,
           "port": 51917
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2018-04-18T08:16:47Z",
@@ -149,13 +152,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "64.235.151.76",
           "locality": "public",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 443
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     }
   ]
 }
diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json
@@ -59,13 +59,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.130.239",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 65105
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -125,13 +128,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 81,
           "ip": "10.99.252.50",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -191,13 +197,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.130.239",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 65105
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -257,13 +266,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 81,
           "ip": "10.98.243.20",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -323,13 +335,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.168.140",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 52344
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -389,13 +404,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 113,
           "ip": "10.98.243.20",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -455,13 +473,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.168.140",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 50294
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -521,13 +542,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 113,
           "ip": "10.98.243.20",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     }
   ]
 }