Add Netflow dashboards based on Logstash Netflow

Convert Logstash dashboards to work on Filebeat netflow data. Changes - Change all UUIDs to not conflict with Logstash dashboards. - Rename fields used in dashboards. ``` // These fields don't exist in the same capacity in ECS so just destination. "geoip.autonomous_system": "destination.as.organization.name", "geoip.city_name": "destination.geo.city_name", "geoip.country_name": "destination.geo.country_name", "geoip.location": "destination.geo.location", "geoip_dst.autonomous_system": "destination.as.organization.name", "geoip_src.autonomous_system": "source.as.organization.name", "host": "agent.hostname", "netflow.bytes": "network.bytes", "netflow.direction": "network.direction", "netflow.dst_addr": "destination.ip", "netflow.dst_port_name": "destination.port", "netflow.flow_locality": "flow.locality", "netflow.input_snmp": "netflow.ingress_interface", "netflow.ip_version": "network.type", "netflow.last_switched": "event.end", "netflow.output_snmp": "netflow.egress_interface", "netflow.packets": "network.packets", "netflow.protocol_name": "network.transport", "netflow.src_addr": "source.ip", "netflow.src_port_name": "source.port", "netflow.tcp_flags_label": "netflow.tcp_control_bits", "netflow.tos": "netflow.ip_class_of_service", "netflow.version": "netflow.exporter.version", "netflow.vlan": "netflow.vlan_id", ``` - Change index pattern from netflow-* to filebeat-* - Add "input.type: netflow" filter. Uses can remove this to view other flow data in dashboards (this mostly works fine b/c of ECS). - Prepend [Filebeat] to dashboards. And append [Filebeat] to visualizations. - Update netflow pipeline to enrich flows with ASN info.
elastic · Jul 11, 2019 · bd4a644 · bd4a644
1 parent 13338e1
commit bd4a644
Show file tree

Hide file tree

Showing 19 changed files with 8,329 additions and 60 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -228,6 +228,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `google-pubsub` input type for consuming messages from a Google Cloud Pub/Sub topic subscription. {pull}12746[12746]
 - Add module for ingesting Cisco IOS logs over syslog. {pull}12748[12748]
 - Add module for ingesting Google Cloud VPC flow logs. {pull}12747[12747]
+- Add netflow dashboards based on Logstash netflow. {pull}12857[12857]
 
 *Heartbeat*
 

diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go
@@ -23,9 +23,12 @@ import (
 	"io/ioutil"
 	"log"
 	"net/url"
+	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/libbeat/dashboards"
 	"github.com/elastic/beats/libbeat/kibana"
 )
@@ -84,6 +87,7 @@ func main() {
 		if err != nil {
 			log.Fatalf("Failed to export dashboards from YML file: %v", err)
 		}
+		log.Println("Done exporting dashboards from", *ymlFile)
 		return
 	}
 
@@ -121,9 +125,14 @@ func exportSingleDashboard(client *kibana.Client, dashboard, output string) erro
 		return fmt.Errorf("failed to export the dashboard: %+v", err)
 	}
 	result = dashboards.DecodeExported(result)
+
+	if err = os.MkdirAll(filepath.Dir(output), 0755); err != nil {
+		return errors.Wrap(err, "failed to create directory for dashboard")
+	}
+
 	err = ioutil.WriteFile(output, []byte(result.StringToPrint()), dashboards.OutputPermission)
 	if err != nil {
-		return fmt.Errorf("failed to save the dashboards: %+v", err)
+		return fmt.Errorf("failed to save the dashboard: %+v", err)
 	}
 	return nil
 }
diff --git a/dev-tools/mage/dashboard.go b/dev-tools/mage/dashboard.go
@@ -44,15 +44,20 @@ func ExportDashboard() error {
 		return err
 	}
 
+	dashboardCmd := sh.RunCmd("go", "run", filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"))
+
 	// TODO: This is currently hardcoded for KB 7, we need to figure out what we do for KB 8 if applicable
 	file := CWD("module", module, "_meta/kibana/7/dashboard", id+".json")
 
-	dashboardCmd := sh.RunCmd("go", "run",
-		filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),
-		"-output", file, "-dashboard", id,
-	)
+	args := []string{
+		"-output", file,
+		"-dashboard", id,
+	}
+	if kibanaURL := EnvOr("KIBANA_URL", ""); kibanaURL != "" {
+		args = append(args, "-kibana", kibanaURL)
+	}
 
-	return dashboardCmd()
+	return dashboardCmd(args...)
 }
 
 // ImportDashboards imports dashboards to Kibana using the Beat setup command.

diff --git a/dev-tools/mage/target/dashboards/dashboards.go b/dev-tools/mage/target/dashboards/dashboards.go
@@ -57,8 +57,9 @@ func (Dashboards) Import() error {
 // directory.
 //
 // Required environment variables:
-// - MODULE: Name of the module
-// - ID:     Dashboard ID
+// - KIBANA_URL: URL of Kibana
+// - MODULE:     Name of the module
+// - ID:         Dashboard ID
 func (Dashboards) Export() error {
 	return devtools.ExportDashboard()
 }
diff --git a/libbeat/dashboards/export.go b/libbeat/dashboards/export.go
@@ -65,6 +65,9 @@ func ExportAllFromYml(client *kibana.Client, ymlPath string) ([]common.MapStr, L
 	if err != nil {
 		return nil, ListYML{}, errors.Wrap(err, "error reading the list of dashboards")
 	}
+	if len(list.Dashboards) == 0 {
+		return nil, ListYML{}, errors.Errorf("dashboards list is empty in file %v", ymlPath)
+	}
 
 	results, err := ExportAll(client, list)
 

diff --git a/x-pack/filebeat/input/netflow/_meta/kibana/7/dashboard/filebeat-network-flows-top-n.json b/x-pack/filebeat/input/netflow/_meta/kibana/7/dashboard/filebeat-network-flows-top-n.json
@@ -12,7 +12,7 @@
             }
           }
         },
-        "savedSearchId": "c1e2ccd0-1ae5-11e9-9eb0-d1ab52900288",
+        "savedSearchId": "2e9de603-5372-4a9c-a787-5c28f4cfcb7d",
         "title": "Source Port and Transport [Filebeat Flows]",
         "uiStateJSON": {},
         "version": 1,
@@ -77,7 +77,7 @@
           "type": "pie"
         }
       },
-      "id": "3bc31900-1ae7-11e9-9eb0-d1ab52900288",
+      "id": "3b0d8bf6-450e-4f5f-abf4-74b8204fa5e7",
       "type": "visualization",
       "updated_at": "2019-01-18T16:16:16.527Z",
       "version": 2
@@ -204,7 +204,7 @@
           "type": "input_control_vis"
         }
       },
-      "id": "b957b010-1ae7-11e9-9eb0-d1ab52900288",
+      "id": "e3ba8d3e-b63a-49b0-977a-167b0886b4f3",
       "type": "visualization",
       "updated_at": "2019-01-18T16:16:16.527Z",
       "version": 2
@@ -221,7 +221,7 @@
             }
           }
         },
-        "savedSearchId": "c1e2ccd0-1ae5-11e9-9eb0-d1ab52900288",
+        "savedSearchId": "2e9de603-5372-4a9c-a787-5c28f4cfcb7d",
         "title": "Destination Port and Transport [Filebeat Flows]",
         "uiStateJSON": {},
         "version": 1,
@@ -286,7 +286,7 @@
           "type": "pie"
         }
       },
-      "id": "44042280-1ae7-11e9-9eb0-d1ab52900288",
+      "id": "ca65f3b7-8a3d-45bb-9e44-8b6511665c84",
       "type": "visualization",
       "updated_at": "2019-01-18T16:16:16.527Z",
       "version": 2
@@ -303,7 +303,7 @@
             }
           }
         },
-        "savedSearchId": "c1e2ccd0-1ae5-11e9-9eb0-d1ab52900288",
+        "savedSearchId": "2e9de603-5372-4a9c-a787-5c28f4cfcb7d",
         "title": "Top Sources Table [Filebeat Flows]",
         "uiStateJSON": {
           "vis": {
@@ -398,7 +398,7 @@
           "type": "table"
         }
       },
-      "id": "846bac40-1ae6-11e9-9eb0-d1ab52900288",
+      "id": "1439876d-0ae4-439c-a65d-e92faa0715f9",
       "type": "visualization",
       "updated_at": "2019-01-18T16:39:24.499Z",
       "version": 3
@@ -415,7 +415,7 @@
             }
           }
         },
-        "savedSearchId": "c1e2ccd0-1ae5-11e9-9eb0-d1ab52900288",
+        "savedSearchId": "2e9de603-5372-4a9c-a787-5c28f4cfcb7d",
         "title": "Top Destinations Table [Filebeat Flows]",
         "uiStateJSON": {
           "vis": {
@@ -510,7 +510,7 @@
           "type": "table"
         }
       },
-      "id": "8d0c61f0-1ae6-11e9-9eb0-d1ab52900288",
+      "id": "db9408ac-e6f5-41da-b677-530c31cda2ec",
       "type": "visualization",
       "updated_at": "2019-01-18T16:39:44.417Z",
       "version": 3
@@ -527,7 +527,7 @@
             }
           }
         },
-        "savedSearchId": "c1e2ccd0-1ae5-11e9-9eb0-d1ab52900288",
+        "savedSearchId": "2e9de603-5372-4a9c-a787-5c28f4cfcb7d",
         "title": "Flows Over Time [Filebeat Flows]",
         "uiStateJSON": {
           "vis": {
@@ -684,7 +684,7 @@
           "type": "area"
         }
       },
-      "id": "e7c6efa0-1ae8-11e9-9eb0-d1ab52900288",
+      "id": "7b3caa8b-91d9-4a6a-b768-e4e3a8c36c3e",
       "type": "visualization",
       "updated_at": "2019-01-18T16:16:16.527Z",
       "version": 2
@@ -746,7 +746,7 @@
         "title": "Network Flow Search [Filebeat]",
         "version": 1
       },
-      "id": "c1e2ccd0-1ae5-11e9-9eb0-d1ab52900288",
+      "id": "2e9de603-5372-4a9c-a787-5c28f4cfcb7d",
       "type": "search",
       "updated_at": "2019-01-18T16:16:16.527Z",
       "version": 2
@@ -779,7 +779,7 @@
               "x": 0,
               "y": 8
             },
-            "id": "3bc31900-1ae7-11e9-9eb0-d1ab52900288",
+            "id": "3b0d8bf6-450e-4f5f-abf4-74b8204fa5e7",
             "panelIndex": "1",
             "type": "visualization",
             "version": "7.0.0"
@@ -793,7 +793,7 @@
               "x": 0,
               "y": 0
             },
-            "id": "b957b010-1ae7-11e9-9eb0-d1ab52900288",
+            "id": "e3ba8d3e-b63a-49b0-977a-167b0886b4f3",
             "panelIndex": "2",
             "type": "visualization",
             "version": "7.0.0"
@@ -807,7 +807,7 @@
               "x": 24,
               "y": 8
             },
-            "id": "44042280-1ae7-11e9-9eb0-d1ab52900288",
+            "id": "ca65f3b7-8a3d-45bb-9e44-8b6511665c84",
             "panelIndex": "3",
             "type": "visualization",
             "version": "7.0.0"
@@ -821,7 +821,7 @@
               "x": 0,
               "y": 23
             },
-            "id": "846bac40-1ae6-11e9-9eb0-d1ab52900288",
+            "id": "1439876d-0ae4-439c-a65d-e92faa0715f9",
             "panelIndex": "4",
             "type": "visualization",
             "version": "7.0.0"
@@ -835,7 +835,7 @@
               "x": 24,
               "y": 23
             },
-            "id": "8d0c61f0-1ae6-11e9-9eb0-d1ab52900288",
+            "id": "db9408ac-e6f5-41da-b677-530c31cda2ec",
             "panelIndex": "5",
             "type": "visualization",
             "version": "7.0.0"
@@ -849,7 +849,7 @@
               "x": 24,
               "y": 0
             },
-            "id": "e7c6efa0-1ae8-11e9-9eb0-d1ab52900288",
+            "id": "7b3caa8b-91d9-4a6a-b768-e4e3a8c36c3e",
             "panelIndex": "6",
             "type": "visualization",
             "version": "7.0.0"
@@ -859,7 +859,7 @@
         "title": "Top-N Flows [Filebeat Flows]",
         "version": 1
       },
-      "id": "1374fe40-1ae8-11e9-9eb0-d1ab52900288",
+      "id": "951e423f-80f4-4d01-aaae-141139f053ee",
       "type": "dashboard",
       "updated_at": "2019-01-18T16:40:54.334Z",
       "version": 4

diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go
@@ -13,13 +13,18 @@ import (
 
 	"github.com/magefile/mage/mg"
 
+	// mage:import
+	"github.com/elastic/beats/dev-tools/mage/target/dashboards"
+
 	devtools "github.com/elastic/beats/dev-tools/mage"
 	filebeat "github.com/elastic/beats/filebeat/scripts/mage"
 )
 
 func init() {
 	devtools.BeatDescription = "Filebeat sends log files to Logstash or directly to Elasticsearch."
 	devtools.BeatLicense = "Elastic License"
+
+	dashboards.RegisterImportDeps(Build, collectDashboards)
 }
 
 // Aliases provides compatibility with CI while we transition all Beats
@@ -100,8 +105,8 @@ func fieldsYML() error {
 	return devtools.GenerateFieldsYAML(devtools.OSSBeatDir("module"), "module", "input")
 }
 
-// Dashboards collects all the dashboards and generates index patterns.
-func Dashboards() error {
+// collectDashboards collects all the dashboards and generates index patterns.
+func collectDashboards() error {
 	return devtools.KibanaDashboards(devtools.OSSBeatDir("module"), "module", "input")
 }
 
@@ -125,7 +130,7 @@ func configYML() error {
 
 // Update is an alias for executing fields, dashboards, config.
 func Update() {
-	mg.SerialDeps(Fields, Dashboards, Config, includeList,
+	mg.SerialDeps(Fields, collectDashboards, Config, includeList,
 		filebeat.PrepareModulePackagingXPack)
 }