Set source.bytes/packets for uni-directional netflow

This populates the `source.bytes` and `source.packets` fields for uni-directional netflow events. Previously only `network.bytes`/`network.packets` would be set. The input would already populate the source fields for bi-directional flows. This also fixes an issue where the totals in `network.bytes` and `network.packets` were incorrectly calculated for bi-directional flows.
elastic · Oct 17, 2019 · dabd9f5 · dabd9f5
1 parent 00bd5b6
commit dabd9f5
Show file tree

Hide file tree

Showing 46 changed files with 1,621 additions and 501 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix cisco module's asa and ftd filesets parsing of domain names where an IP address is expected. {issue}14034[14034]
 - Fixed increased memory usage with large files when multiline pattern does not match. {issue}14068[14068]
 - panw module: Use geo.name instead of geo.country_iso_code for free-form location. {issue}13272[13272]
+- Fix calculation of `network.bytes` and `network.packets` for bi-directional netflow events. {pull}14111[14111]
 
 *Heartbeat*
 
@@ -354,6 +355,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add ExpandEventListFromField config option in the kafka input. {pull}13965[13965]
 - Add ELB fileset to AWS module. {pull}14020[14020]
 - Add module for MISP (Malware Information Sharing Platform). {pull}13805[13805]
+- Add `source.bytes` and `source.packets` for uni-directional netflow events. {pull}14111[14111]
 
 *Heartbeat*
 - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]

diff --git a/x-pack/filebeat/input/netflow/convert.go b/x-pack/filebeat/input/netflow/convert.go
@@ -262,26 +262,23 @@ func flowToBeatEvent(flow record.Record) (event beat.Event) {
 		revPkts, hasRevPkts = getKeyUint64(flow.Fields, "reversePacketTotalCount")
 	}
 
-	if hasRevBytes || hasRevPkts {
-		if hasBytes {
-			ecsSource["bytes"] = countBytes
-			ecsDest["bytes"] = revBytes
-		}
-		if hasPkts {
-			ecsSource["packets"] = revBytes
-			ecsDest["packets"] = revPkts
-		}
-		countBytes += revBytes
-		countPkts += revPkts
+	if hasRevBytes {
+		ecsDest["bytes"] = revBytes
+	}
+
+	if hasRevPkts {
+		ecsDest["packets"] = revPkts
 	}
 
 	if hasBytes {
+		ecsSource["bytes"] = countBytes
 		if hasRevBytes {
 			countBytes += revBytes
 		}
 		ecsNetwork["bytes"] = countBytes
 	}
 	if hasPkts {
+		ecsSource["packets"] = countPkts
 		if hasRevPkts {
 			countPkts += revPkts
 		}

diff --git a/x-pack/filebeat/input/netflow/decoder/examples/examples b/x-pack/filebeat/input/netflow/decoder/examples/examples
diff --git a/...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json b/...t/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json
@@ -71,13 +71,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.236.5.4",
           "locality": "private",
           "mac": "00:50:56:b9:26:46",
+          "packets": 0,
           "port": 51917
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2018-04-18T08:16:47Z",
@@ -149,13 +152,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "64.235.151.76",
           "locality": "public",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 443
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     }
   ]
 }
diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json
@@ -59,13 +59,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.130.239",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 65105
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -125,13 +128,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 81,
           "ip": "10.99.252.50",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -191,13 +197,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.130.239",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 65105
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -257,13 +266,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 81,
           "ip": "10.98.243.20",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -323,13 +335,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.168.140",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 52344
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -389,13 +404,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 113,
           "ip": "10.98.243.20",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -455,13 +473,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 0,
           "ip": "10.99.168.140",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 0,
           "port": 50294
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     },
     {
       "Timestamp": "2017-06-29T13:58:28Z",
@@ -521,13 +542,16 @@
           "ip": "192.0.2.1"
         },
         "source": {
+          "bytes": 113,
           "ip": "10.98.243.20",
           "locality": "private",
           "mac": "00:00:00:00:00:00",
+          "packets": 1,
           "port": 53
         }
       },
-      "Private": null
+      "Private": null,
+      "TimeSeries": false
     }
   ]
 }