diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a0e555166cc..b63bff11ce4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -277,6 +277,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914] - Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916] - Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918] +- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738] *Heartbeat* diff --git a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.json b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.json deleted file mode 100644 index 7d662ab7da1..00000000000 --- a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek capture_loss.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.capture_loss.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.capture_loss.ts" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml new file mode 100644 index 00000000000..3c6171bc045 --- /dev/null +++ b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml @@ -0,0 +1,21 @@ +description: Pipeline for normalizing Zeek capture_loss.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.capture_loss.ts + formats: + - UNIX +- remove: + field: zeek.capture_loss.ts +- set: + field: event.kind + value: metric +- set: + field: event.type + value: info +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/capture_loss/manifest.yml b/x-pack/filebeat/module/zeek/capture_loss/manifest.yml index 97ae0f09d40..5349b0581c6 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/manifest.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/manifest.yml @@ -11,5 +11,5 @@ var: - name: tags default: [zeek.capture_loss] -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/capture_loss.yml diff --git a/x-pack/filebeat/module/zeek/capture_loss/test/capture_loss-json.log-expected.json b/x-pack/filebeat/module/zeek/capture_loss/test/capture_loss-json.log-expected.json index 0ae18ff9c37..14f20eb3189 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/test/capture_loss-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/capture_loss/test/capture_loss-json.log-expected.json @@ -2,7 +2,9 @@ { "@timestamp": "2019-09-10T16:19:28.465Z", "event.dataset": "zeek.capture_loss", + "event.kind": "metric", "event.module": "zeek", + "event.type": "info", "fileset.name": "capture_loss", "input.type": "log", "log.offset": 0, diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml index 14c5b529708..f91d24f8020 100644 --- a/x-pack/filebeat/module/zeek/connection/config/connection.yml +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -75,20 +75,27 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network {{ if .community_id }} - if: equals.network.transport: icmp then: community_id: fields: - source_ip: source.address - destination_ip: destination.address icmp_type: zeek.connection.icmp.type icmp_code: zeek.connection.icmp.code else: community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json deleted file mode 100644 index a930fd08ec9..00000000000 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek conn.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.connection.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.connection.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "script": { - "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", - "params": { - "scale": 1000000000 - }, - "if": "ctx.temp?.duration != null" - } - }, - { - "remove": { - "field": "temp.duration", - "ignore_missing": true - } - }, - { - "script": { - "source": "if (ctx.zeek.connection.local_orig) ctx.tags.add(\"local_orig\");", - "if": "ctx.zeek.connection.local_orig != null" - } - }, - { - "script": { - "source": "if (ctx.zeek.connection.local_resp) ctx.tags.add(\"local_resp\");", - "if": "ctx.zeek.connection.local_resp != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "script": { - "source": "ctx.network.packets = ctx.source.packets + ctx.destination.packets", - "ignore_failure": true - } - }, - { - "script": { - "source": "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes", - "ignore_failure": true - } - }, - { - "script": { - "source": "if (ctx.zeek.connection.local_orig == true && ctx.zeek.connection.local_resp == true) {ctx.network.direction = \"internal\"} else if (ctx.zeek.connection.local_orig == true && ctx.zeek.connection.local_resp == false) {ctx.network.direction = \"outbound\"} else if (ctx.zeek.connection.local_orig == false && ctx.zeek.connection.local_resp == true) {ctx.network.direction = \"inbound\"} else {ctx.network.direction = \"external\"}" - } - }, - { - "geoip": { - "field": "destination.ip", - "target_field": "destination.geo" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo" - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "destination.ip", - "target_field": "destination.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.asn", - "target_field": "destination.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.organization_name", - "target_field": "destination.as.organization.name", - "ignore_missing": true - } - }, - { - "script": { - "source": "if (ctx.zeek.connection.state == \"S0\") {ctx.zeek.connection.state_message = \"Connection attempt seen, no reply.\"} else if (ctx.zeek.connection.state == \"S1\") {ctx.zeek.connection.state_message = \"Connection established, not terminated.\"} else if (ctx.zeek.connection.state == \"SF\") {ctx.zeek.connection.state_message = \"Normal establishment and termination.\"} else if (ctx.zeek.connection.state == \"REJ\") {ctx.zeek.connection.state_message = \"Connection attempt rejected.\"} else if (ctx.zeek.connection.state == \"S2\") {ctx.zeek.connection.state_message = \" Connection established and close attempt by originator seen (but no reply from responder).\"} else if (ctx.zeek.connection.state == \"S3\") {ctx.zeek.connection.state_message = \"Connection established and close attempt by responder seen (but no reply from originator).\"} else if (ctx.zeek.connection.state == \"RSTO\") {ctx.zeek.connection.state_message = \"Connection established, originator aborted (sent a RST).\"} else if (ctx.zeek.connection.state == \"RSTR\") {ctx.zeek.connection.state_message = \"Responder sent a RST.\"} else if (ctx.zeek.connection.state == \"RSTOS0\") {ctx.zeek.connection.state_message = \"Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder.\"} else if (ctx.zeek.connection.state == \"RSTRH\") {ctx.zeek.connection.state_message = \"Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator.\"} else if (ctx.zeek.connection.state == \"SH\") {ctx.zeek.connection.state_message = \"Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open).\"} else if (ctx.zeek.connection.state == \"SHR\") {ctx.zeek.connection.state_message = \"Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.\"} else if (ctx.zeek.connection.state == \"OTH\") {ctx.zeek.connection.state_message = \"No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).\"}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml new file mode 100644 index 00000000000..b660079324a --- /dev/null +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml @@ -0,0 +1,187 @@ +description: Pipeline for normalizing Zeek conn.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.connection.ts + formats: + - UNIX +- remove: + field: zeek.connection.ts +- set: + field: event.id + value: '{{zeek.session_id}}' + if: ctx.zeek.session_id != null +- script: + source: ctx.event.duration = Math.round(ctx.temp.duration * params.scale) + params: + scale: 1000000000 + if: ctx.temp?.duration != null +- remove: + field: temp.duration + ignore_missing: true +- script: + source: if (ctx.zeek.connection.local_orig) ctx.tags.add("local_orig"); + if: ctx.zeek.connection.local_orig != null +- script: + source: if (ctx.zeek.connection.local_resp) ctx.tags.add("local_resp"); + if: ctx.zeek.connection.local_resp != null +- set: + field: source.ip + value: '{{source.address}}' +- append: + field: related.ip + value: '{{source.address}}' +- set: + field: destination.ip + value: '{{destination.address}}' +- append: + field: related.ip + value: '{{destination.address}}' +- script: + source: ctx.network.packets = ctx.source.packets + ctx.destination.packets + ignore_failure: true +- script: + source: ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes + ignore_failure: true +- script: + source: >- + if (ctx?.zeek?.connection?.local_orig == true) { + if (ctx?.zeek?.connection?.local_resp == true) { + ctx.network.direction = "internal"; + } else { + ctx.network.direction = "outbound"; + } + } else { + if (ctx?.zeek?.connection?.local_resp == true) { + ctx.network.direction = "inbound"; + } else { + ctx.network.direction = "external"; + } + } +- geoip: + field: destination.ip + target_field: destination.geo +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- set: + field: event.kind + value: event +- append: + field: event.category + value: network +- script: + params: + S0: + conn_str: "Connection attempt seen, no reply." + types: + - connection + - start + S1: + conn_str: "Connection established, not terminated." + types: + - connection + - start + SF: + conn_str: "Normal establishment and termination." + types: + - connection + - start + - end + REG: + conn_str: "Connection attempt rejected." + types: + - connection + - start + - denied + S2: + conn_str: "Connection established and close attempt by originator seen (but no reply from responder)." + types: + - connection + - info + S3: + conn_str: "Connection established and close attempt by responder seen (but no reply from originator)." + types: + - connection + - info + RSTO: + conn_str: "Connection established, originator aborted (sent a RST)." + types: + - connection + - info + RSTR: + conn_str: "Responder sent a RST." + types: + - connection + - info + RSTOS0: + conn_str: "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder." + types: + - connection + - info + RSTRH: + conn_str: "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator." + types: + - connection + - info + SH: + conn_str: "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)." + types: + - connection + - info + SHR: + conn_str: "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + types: + - connection + - info + OTH: + conn_str: "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + types: + - connection + - info + source: >- + if (ctx?.zeek?.connection?.state == null) { + return; + } + if (params.containsKey(ctx.zeek.connection.state)) { + ctx.zeek.connection.state_message = params[ctx.zeek.connection.state]["conn_str"]; + ctx.event.type = params[ctx.zeek.connection.state]["types"]; + } +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/connection/manifest.yml b/x-pack/filebeat/module/zeek/connection/manifest.yml index 0361f0c89fa..0acad34d69c 100644 --- a/x-pack/filebeat/module/zeek/connection/manifest.yml +++ b/x-pack/filebeat/module/zeek/connection/manifest.yml @@ -13,7 +13,7 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/connection.yml requires.processors: diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json index 4e5615a3a51..35a539b1493 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -6,10 +6,20 @@ "destination.ip": "192.168.86.1", "destination.packets": 1, "destination.port": 53, + "event.category": [ + "network", + "network" + ], "event.dataset": "zeek.connection", "event.duration": 76967000, "event.id": "CAcJw21BbVedgFnYH3", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "start", + "end" + ], "fileset.name": "connection", "input.type": "log", "log.offset": 0, @@ -19,6 +29,10 @@ "network.packets": 2, "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "192.168.86.167", + "192.168.86.1" + ], "service.type": "zeek", "source.address": "192.168.86.167", "source.bytes": 103, @@ -51,10 +65,20 @@ "destination.ip": "8.8.8.8", "destination.packets": 1, "destination.port": 53, + "event.category": [ + "network", + "network" + ], "event.dataset": "zeek.connection", "event.duration": 76967000, "event.id": "CAcJw21BbVedgFnYH4", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "start", + "end" + ], "fileset.name": "connection", "input.type": "log", "log.offset": 398, @@ -64,6 +88,10 @@ "network.packets": 2, "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "192.168.86.167", + "8.8.8.8" + ], "service.type": "zeek", "source.address": "192.168.86.167", "source.bytes": 103, @@ -95,10 +123,20 @@ "destination.ip": "8.8.8.8", "destination.packets": 1, "destination.port": 53, + "event.category": [ + "network", + "network" + ], "event.dataset": "zeek.connection", "event.duration": 76967000, "event.id": "CAcJw21BbVedgFnYH5", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "start", + "end" + ], "fileset.name": "connection", "input.type": "log", "log.offset": 792, @@ -108,6 +146,10 @@ "network.packets": 2, "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "4.4.2.2", + "8.8.8.8" + ], "service.type": "zeek", "source.address": "4.4.2.2", "source.as.number": 3356, @@ -137,9 +179,18 @@ "destination.bytes": 0, "destination.ip": "198.51.100.249", "destination.packets": 0, + "event.category": [ + "network", + "network" + ], "event.dataset": "zeek.connection", "event.id": "Cc6NJ3GRlfjE44I3h", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "info" + ], "fileset.name": "connection", "input.type": "log", "log.offset": 1181, @@ -148,6 +199,10 @@ "network.direction": "external", "network.packets": 1, "network.transport": "icmp", + "related.ip": [ + "192.0.2.205", + "198.51.100.249" + ], "service.type": "zeek", "source.address": "192.0.2.205", "source.bytes": 107, @@ -165,4 +220,4 @@ "zeek.connection.state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", "zeek.session_id": "Cc6NJ3GRlfjE44I3h" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml index e7875bca0df..0ba1b0fc673 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml @@ -36,7 +36,23 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info {{ if .community_id }} - community_id: {{ end }} diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.json deleted file mode 100644 index 0f274438186..00000000000 --- a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek dce_rpc.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.dce_rpc.ts", - "formats": ["UNIX"] - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "remove": { - "field": "zeek.dce_rpc.ts" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml new file mode 100644 index 00000000000..1ecda252cc8 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek dce_rpc.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.dce_rpc.ts + formats: + - UNIX +- remove: + field: zeek.dce_rpc.ts +- append: + field: related.ip + value: '{{source.ip}}' +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: '{{destination.ip}}' +- geoip: + field: destination.ip + target_field: destination.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- set: + field: event.action + value: '{{zeek.dce_rpc.operation}}' + if: "ctx?.zeek?.dce_rpc?.operation != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/dce_rpc/manifest.yml b/x-pack/filebeat/module/zeek/dce_rpc/manifest.yml index 853c7084f7e..21ba27eac96 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/manifest.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/dce_rpc.yml diff --git a/x-pack/filebeat/module/zeek/dce_rpc/test/dce_rpc-json.log-expected.json b/x-pack/filebeat/module/zeek/dce_rpc/test/dce_rpc-json.log-expected.json index 881f30d1b79..6128801caa7 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/test/dce_rpc-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dce_rpc/test/dce_rpc-json.log-expected.json @@ -4,14 +4,29 @@ "destination.address": "172.16.128.202", "destination.ip": "172.16.128.202", "destination.port": 445, + "event.action": "BrowserrQueryOtherDomains", + "event.category": [ + "network" + ], "event.dataset": "zeek.dce_rpc", "event.id": "CsNHVHa1lzFtvJzT8", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "dce_rpc", "input.type": "log", "log.offset": 0, + "network.community_id": "1:SJNAD5vtzZuhQjGtfaI8svTnyuw=", "network.protocol": "dce_rpc", "network.transport": "tcp", + "related.ip": [ + "172.16.133.6", + "172.16.128.202" + ], "service.type": "zeek", "source.address": "172.16.133.6", "source.ip": "172.16.133.6", diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml index 5878c8d7894..97c45a17920 100644 --- a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml +++ b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml @@ -94,9 +94,27 @@ processors: fields: port: 67 + - convert: + fields: + - {from: "zeek.dhcp.address.client", to: "source.address"} + - {from: "zeek.dhcp.address.client", to: "source.ip", type: "ip"} + - {from: "zeek.dhcp.address.client", to: "client.address"} + - {from: "zeek.dhcp.address.server", to: "destination.address"} + - {from: "zeek.dhcp.address.server", to: "destination.ip", type: "ip"} + - {from: "zeek.dhcp.address.server", to: "server.address"} + - {from: "zeek.dhcp.domain", to: "network.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info {{ if .community_id }} - community_id: - fields: - source.address: zeek.dhcp.address.client - destination.address: zeek.dhcp.address.server {{ end }} diff --git a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.json deleted file mode 100644 index 92c1a43dd4a..00000000000 --- a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek dhcp.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.dhcp.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.dhcp.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.address", - "value": "{{zeek.dhcp.address.client}}", - "if": "ctx.zeek.dhcp.address?.client != null" - } - }, - { - "set": { - "field": "client.address", - "value": "{{zeek.dhcp.address.client}}", - "if": "ctx.zeek.dhcp.address?.client != null" - } - }, - { - "set": { - "field": "destination.address", - "value": "{{zeek.dhcp.address.server}}", - "if": "ctx.zeek.dhcp.address?.server != null" - } - }, - { - "set": { - "field": "server.address", - "value": "{{zeek.dhcp.address.server}}", - "if": "ctx.zeek.dhcp.address?.server != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}", - "if": "ctx.source?.address != null" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}", - "if": "ctx.destination?.address != null" - } - }, - { - "set": { - "field": "network.name", - "value": "{{zeek.dhcp.domain}}", - "if": "ctx.zeek.dhcp.domain != null" - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml new file mode 100644 index 00000000000..49df687ecc3 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml @@ -0,0 +1,27 @@ +description: Pipeline for normalizing Zeek dhcp.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.dhcp.ts + formats: + - UNIX +- remove: + field: zeek.dhcp.ts +- set: + field: event.id + value: '{{zeek.session_id}}' + if: ctx.zeek.session_id != null +- append: + field: related.ip + value: '{{source.ip}}' + if: 'ctx?.source?.ip != null' +- append: + field: related.ip + value: '{{destination.ip}}' + if: 'ctx?.destination?.ip != null' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/dhcp/manifest.yml b/x-pack/filebeat/module/zeek/dhcp/manifest.yml index a09038725e3..7cb434b1955 100644 --- a/x-pack/filebeat/module/zeek/dhcp/manifest.yml +++ b/x-pack/filebeat/module/zeek/dhcp/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/dhcp.yml diff --git a/x-pack/filebeat/module/zeek/dhcp/test/dhcp-json.log-expected.json b/x-pack/filebeat/module/zeek/dhcp/test/dhcp-json.log-expected.json index 63fd7367dd8..ec36a36c503 100644 --- a/x-pack/filebeat/module/zeek/dhcp/test/dhcp-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dhcp/test/dhcp-json.log-expected.json @@ -5,15 +5,29 @@ "destination.address": "192.168.199.254", "destination.ip": "192.168.199.254", "destination.port": 67, + "event.category": [ + "network" + ], "event.dataset": "zeek.dhcp", "event.id": "{0=CmWOt6VWaNGqXYcH6, 1=CLObLo4YHn0u23Tp8a}", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "dhcp", "input.type": "log", "log.offset": 0, + "network.community_id": "1:HsGjbon+HsK9xnMq+1A32BR9C4Y=", "network.name": "localdomain", "network.protocol": "dhcp", "network.transport": "udp", + "related.ip": [ + "192.168.199.132", + "192.168.199.254" + ], "server.address": "192.168.199.254", "service.type": "zeek", "source.address": "192.168.199.132", diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml index 4dec34b4b59..d059b4c79f9 100644 --- a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml +++ b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml @@ -46,9 +46,23 @@ processors: ignore_missing: true fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.json deleted file mode 100644 index 3f7e3c4baee..00000000000 --- a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek dnp3.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.dnp3.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.dnp3.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml new file mode 100644 index 00000000000..ad4670dc350 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml @@ -0,0 +1,64 @@ +description: Pipeline for normalizing Zeek dnp3.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.dnp3.ts + formats: + - UNIX +- remove: + field: zeek.dnp3.ts +- set: + field: event.action + value: '{{zeek.dnp3.function.request}}' + if: "ctx?.zeek?.dnp3?.function?.request != null" +- set: + field: event.action + value: '{{zeek.dnp3.function.reply}}' + if: "ctx?.zeek?.dnp3?.function?.reply != null" +- lowercase: + field: event.action + ignore_missing: true +- geoip: + field: destination.ip + target_field: destination.geo +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/dnp3/manifest.yml b/x-pack/filebeat/module/zeek/dnp3/manifest.yml index 73488debb12..98de1c3af82 100644 --- a/x-pack/filebeat/module/zeek/dnp3/manifest.yml +++ b/x-pack/filebeat/module/zeek/dnp3/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/dnp3.yml diff --git a/x-pack/filebeat/module/zeek/dnp3/test/dnp3-json.log-expected.json b/x-pack/filebeat/module/zeek/dnp3/test/dnp3-json.log-expected.json index 040dabff377..fa386feb1ce 100644 --- a/x-pack/filebeat/module/zeek/dnp3/test/dnp3-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dnp3/test/dnp3-json.log-expected.json @@ -4,9 +4,19 @@ "destination.address": "127.0.0.1", "destination.ip": "127.0.0.1", "destination.port": 20000, + "event.action": "read", + "event.category": [ + "network" + ], "event.dataset": "zeek.dnp3", "event.id": "CQV6tj1w1t4WzQpHoe", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "dnp3", "input.type": "log", "log.offset": 0, diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml index 96e67d9f840..7b4c332f5df 100644 --- a/x-pack/filebeat/module/zeek/dns/config/dns.yml +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -13,6 +13,11 @@ processors: - decode_json_fields: fields: [event.original] target: zeek.dns + - registered_domain: + ignore_missing: true + ignore_failure: true + field: zeek.dns.query + target_field: dns.question.registered_domain - script: lang: javascript id: zeek_dns_flags @@ -105,12 +110,54 @@ processors: evt.Put("event.duration", rttSec * 1000000000); } + function addTopLevelDomain(evt) { + var rd = evt.Get("dns.question.registered_domain"); + if (!rd) { + return; + } + var firstPeriod = rd.indexOf("."); + if (firstPeriod == -1) { + return; + } + evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1)); + } + + function addEventOutcome(evt) { + var rcode = evt.Get("zeek.dns.rcode"); + if (rcode == null) { + return; + } + if (rcode == 0) { + evt.Put("event.outcome", "success"); + } else { + evt.Put("event.outcome", "failure"); + } + } + + function addRelatedIP(evt) { + var related = []; + var src = evt.Get("zeek.dns.id.orig_h"); + if (src != null) { + related.push(src); + } + var dst = evt.Get("zeek.dns.id.resp_h"); + if (dst != null) { + related.push(dst); + } + if (related.length > 0) { + evt.Put("related.ip", related); + } + } + function process(evt) { addDnsHeaderFlags(evt); addDnsQuestionClass(evt); addDnsAnswers(evt); setDnsType(evt); addEventDuration(evt); + addTopLevelDomain(evt); + addEventOutcome(evt); + addRelatedIP(evt); } - convert: ignore_missing: true @@ -136,13 +183,18 @@ processors: - {from: zeek.dns.query, to: dns.question.name} - {from: zeek.dns.qtype_name, to: dns.question.type} - {from: zeek.dns.rcode_name, to: dns.response_code} - - registered_domain: - ignore_missing: true - ignore_failure: true - field: dns.question.name - target_field: dns.question.registered_domain + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info + - protocol {{ if .community_id }} - - community_id: ~ + - community_id: {{ end }} - timestamp: ignore_missing: true diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json index a8e2cd94b3a..0c01c52e428 100644 --- a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -26,22 +26,37 @@ "dns.question.class": "IN", "dns.question.name": "dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io", "dns.question.registered_domain": "es.io", + "dns.question.top_level_domain": "io", "dns.question.type": "A", "dns.resolved_ip": [ "35.199.178.4" ], "dns.response_code": "NOERROR", "dns.type": "answer", + "event.category": [ + "network" + ], "event.dataset": "zeek.dns", "event.duration": 76967000, "event.id": "CAcJw21BbVedgFnYH3", + "event.kind": "event", "event.module": "zeek", "event.original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":15209,\"rtt\":0.076967,\"query\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":1,\"qtype_name\":\"A\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"proxy-production-us-west1.gcp.cloud.es.io\",\"proxy-production-us-west1-v1-009.gcp.cloud.es.io\",\"35.199.178.4\"],\"TTLs\":[119.0,119.0,59.0],\"rejected\":false}", + "event.outcome": "success", + "event.type": [ + "connection", + "info", + "protocol" + ], "fileset.name": "dns", "input.type": "log", "log.offset": 0, "network.community_id": "1:Z26DBGVYoBKQ1FT6qfPaAqBnJik=", "network.transport": "udp", + "related.ip": [ + "192.168.86.167", + "192.168.86.1" + ], "service.type": "zeek", "source.address": "192.168.86.167", "source.ip": "192.168.86.167", @@ -84,17 +99,31 @@ "dns.question.class": "IN", "dns.question.name": "_googlecast._tcp.local", "dns.question.registered_domain": "_tcp.local", + "dns.question.top_level_domain": "local", "dns.question.type": "PTR", "dns.type": "query", + "event.category": [ + "network" + ], "event.dataset": "zeek.dns", "event.id": "C19a1k4lTv46YMbeOk", + "event.kind": "event", "event.module": "zeek", "event.original": "{\"ts\":1567095830.680046,\"uid\":\"C19a1k4lTv46YMbeOk\",\"id.orig_h\":\"fe80::4ef:15cf:769f:ff21\",\"id.orig_p\":5353,\"id.resp_h\":\"ff02::fb\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":12,\"qtype_name\":\"PTR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false}", + "event.type": [ + "connection", + "info", + "protocol" + ], "fileset.name": "dns", "input.type": "log", "log.offset": 566, "network.community_id": "1:Jq0sRtlGSMjsvMBE1ZYybbR2tI0=", "network.transport": "udp", + "related.ip": [ + "fe80::4ef:15cf:769f:ff21", + "ff02::fb" + ], "service.type": "zeek", "source.address": "fe80::4ef:15cf:769f:ff21", "source.ip": "fe80::4ef:15cf:769f:ff21", @@ -130,17 +159,32 @@ "dns.id": 0, "dns.question.name": "_googlecast._tcp.local", "dns.question.registered_domain": "_tcp.local", + "dns.question.top_level_domain": "local", "dns.response_code": "NOERROR", "dns.type": "answer", + "event.category": [ + "network" + ], "event.dataset": "zeek.dns", "event.id": "CdiVAw7jJw6gsX5H", + "event.kind": "event", "event.module": "zeek", "event.original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", + "event.outcome": "success", + "event.type": [ + "connection", + "info", + "protocol" + ], "fileset.name": "dns", "input.type": "log", "log.offset": 909, "network.community_id": "1:QIR5YXlirWwWA18ZyY/RnvQoaic=", "network.transport": "udp", + "related.ip": [ + "192.168.86.237", + "224.0.0.251" + ], "service.type": "zeek", "source.address": "192.168.86.237", "source.ip": "192.168.86.237", diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml index 9e6a0138ef2..0a31b70f6bd 100644 --- a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml +++ b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml @@ -36,10 +36,22 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.session_id", to: "event.id"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.json deleted file mode 100644 index 7a8958013fc..00000000000 --- a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek dpd.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.dpd.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.dpd.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml new file mode 100644 index 00000000000..f30ff172fa8 --- /dev/null +++ b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek dpd.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.dpd.ts + formats: + - UNIX +- remove: + field: zeek.dpd.ts +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/dpd/manifest.yml b/x-pack/filebeat/module/zeek/dpd/manifest.yml index b331bca2921..aeba0ef31fc 100644 --- a/x-pack/filebeat/module/zeek/dpd/manifest.yml +++ b/x-pack/filebeat/module/zeek/dpd/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/dpd.yml diff --git a/x-pack/filebeat/module/zeek/dpd/test/dpd-json.log-expected.json b/x-pack/filebeat/module/zeek/dpd/test/dpd-json.log-expected.json index d3f58dbd4e0..0d6173e172e 100644 --- a/x-pack/filebeat/module/zeek/dpd/test/dpd-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dpd/test/dpd-json.log-expected.json @@ -4,14 +4,26 @@ "destination.address": "192.168.10.10", "destination.ip": "192.168.10.10", "destination.port": 445, + "event.category": [ + "network" + ], "event.dataset": "zeek.dpd", "event.id": "CRrT7S1ccw9H6hzCR", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "info" + ], "fileset.name": "dpd", "input.type": "log", "log.offset": 0, "network.community_id": "1:b+Szw+ia464igf5e+MwW1WUzw9Y=", "network.transport": "tcp", + "related.ip": [ + "192.168.10.31", + "192.168.10.10" + ], "service.type": "zeek", "source.address": "192.168.10.31", "source.ip": "192.168.10.31", diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml index 7148b82a481..74259307f41 100644 --- a/x-pack/filebeat/module/zeek/files/config/files.yml +++ b/x-pack/filebeat/module/zeek/files/config/files.yml @@ -15,9 +15,25 @@ processors: fields: - from: "json" to: "zeek.files" - - from: "zeek.files.conn_uids" to: "zeek.files.session_ids" - ignore_missing: true fail_on_error: false + - convert: + fields: + - {from: "zeek.files.mime_type", to: "file.mime_type"} + - {from: "zeek.files.filename", to: "file.name"} + - {from: "zeek.files.total_bytes", to: "file.size"} + - {from: "zeek.files.md5", to: "file.hash.md5"} + - {from: "zeek.files.sha1", to: "file.hash.sha1"} + - {from: "zeek.files.sha256", to: "file.hash.sha256"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - file + type: + - info diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json deleted file mode 100644 index 1c47b4d0b42..00000000000 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek files.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.files.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.files.ts" - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.zeek.session_id = ctx.zeek.files.session_ids[0];", - "if": "ctx.zeek.files.session_ids != null", - "ignore_failure": true - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.zeek.files.rx_host = ctx.zeek.files.rx_hosts[0]; ctx.zeek.files.remove('rx_hosts');", - "ignore_failure": true - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.zeek.files.tx_host = ctx.zeek.files.tx_hosts[0]; ctx.zeek.files.remove('tx_hosts');", - "ignore_failure": true - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml new file mode 100644 index 00000000000..0d5abf9bdda --- /dev/null +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml @@ -0,0 +1,66 @@ +description: Pipeline for normalizing Zeek files.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.files.ts + formats: + - UNIX +- remove: + field: zeek.files.ts +- script: + lang: painless + source: ctx.zeek.session_id = ctx.zeek.files.session_ids[0]; + if: ctx.zeek.files.session_ids != null + ignore_failure: true +- set: + field: event.id + value: '{{zeek.session_id}}' + if: ctx.zeek.session_id != null +- foreach: + field: zeek.files.tx_hosts + processor: + append: + field: related.ip + value: "{{_ingest._value}}" + ignore_missing: true +- script: + lang: painless + source: ctx.zeek.files.tx_host = ctx.zeek.files.tx_hosts[0]; ctx.zeek.files.remove('tx_hosts'); + ignore_failure: true +- set: + field: server.ip + value: "{{zeek.files.tx_host}}" + if: "ctx?.zeek?.files?.tx_host != null" +- foreach: + field: zeek.files.rx_hosts + processor: + append: + field: related.ip + value: "{{_ingest._value}}" + ignore_missing: true +- script: + lang: painless + source: ctx.zeek.files.rx_host = ctx.zeek.files.rx_hosts[0]; ctx.zeek.files.remove('rx_hosts'); + ignore_failure: true +- set: + field: client.ip + value: "{{zeek.files.rx_host}}" + if: "ctx?.zeek?.files?.rx_host != null" +- append: + field: related.hash + value: "{{file.hash.md5}}" + if: "ctx?.file?.hash?.md5 != null" +- append: + field: related.hash + value: "{{file.hash.sha1}}" + if: "ctx?.file?.hash?.sha1 != null" +- append: + field: related.hash + value: "{{file.hash.sha256}}" + if: "ctx?.file?.hash?.sha256 != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/files/manifest.yml b/x-pack/filebeat/module/zeek/files/manifest.yml index 68b53467346..bef3d7211b6 100644 --- a/x-pack/filebeat/module/zeek/files/manifest.yml +++ b/x-pack/filebeat/module/zeek/files/manifest.yml @@ -13,7 +13,7 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/files.yml requires.processors: diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json index 4cc0e2d38e0..6fc38a5d22a 100644 --- a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json @@ -1,12 +1,32 @@ [ { "@timestamp": "2019-01-17T01:33:16.636Z", + "client.ip": "10.178.98.102", + "event.category": [ + "file" + ], "event.dataset": "zeek.files", "event.id": "C8I0zn3r9EPbfLgta6", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "info" + ], + "file.hash.md5": "79e4a9840d7d3a96d7c04fe2434c892e", + "file.hash.sha1": "a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436", + "file.mime_type": "application/pkix-cert", "fileset.name": "files", "input.type": "log", "log.offset": 0, + "related.hash": [ + "79e4a9840d7d3a96d7c04fe2434c892e", + "a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436" + ], + "related.ip": [ + "35.199.178.4", + "10.178.98.102" + ], + "server.ip": "35.199.178.4", "service.type": "zeek", "tags": [ "zeek.files" @@ -38,12 +58,32 @@ }, { "@timestamp": "2019-01-17T01:33:21.566Z", + "client.ip": "10.178.98.102", + "event.category": [ + "file" + ], "event.dataset": "zeek.files", "event.id": "C6sjVo23iNApLnlAt6", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "info" + ], + "file.hash.md5": "b9742f12eb97eff531d94f7800c6706c", + "file.hash.sha1": "b88d13fe319d342e7a808ce3a0a1158111fc3c2a", + "file.mime_type": "application/pkix-cert", "fileset.name": "files", "input.type": "log", "log.offset": 452, + "related.hash": [ + "b9742f12eb97eff531d94f7800c6706c", + "b88d13fe319d342e7a808ce3a0a1158111fc3c2a" + ], + "related.ip": [ + "17.134.127.250", + "10.178.98.102" + ], + "server.ip": "17.134.127.250", "service.type": "zeek", "tags": [ "zeek.files" diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml index 7c9e90cb96a..3e91ace4831 100644 --- a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml +++ b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml @@ -60,10 +60,27 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.ftp.user", to: "user.name"} + - {from: "zeek.ftp.command", to: "event.action"} + - {from: "zeek.ftp.mime.type", to: "file.mime_type"} + - {from: "zeek.ftp.file.size", to: "file.size"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.json deleted file mode 100644 index 06b896b53d3..00000000000 --- a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek ftp.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.ftp.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.ftp.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "dot_expander": { - "field": "data_channel.passive", - "path": "zeek.ftp" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml new file mode 100644 index 00000000000..7c15dce3ac5 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml @@ -0,0 +1,68 @@ +description: Pipeline for normalizing Zeek ftp.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.ftp.ts + formats: + - UNIX +- remove: + field: zeek.ftp.ts +- dot_expander: + field: data_channel.passive + path: zeek.ftp +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- geoip: + field: destination.ip + target_field: destination.geo +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/ftp/manifest.yml b/x-pack/filebeat/module/zeek/ftp/manifest.yml index 3dd47573af9..cf51575cf84 100644 --- a/x-pack/filebeat/module/zeek/ftp/manifest.yml +++ b/x-pack/filebeat/module/zeek/ftp/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/ftp.yml diff --git a/x-pack/filebeat/module/zeek/ftp/test/ftp.log-expected.json b/x-pack/filebeat/module/zeek/ftp/test/ftp.log-expected.json index 7de6cc8897c..e6a47bd369e 100644 --- a/x-pack/filebeat/module/zeek/ftp/test/ftp.log-expected.json +++ b/x-pack/filebeat/module/zeek/ftp/test/ftp.log-expected.json @@ -4,15 +4,32 @@ "destination.address": "192.168.1.231", "destination.ip": "192.168.1.231", "destination.port": 21, + "event.action": "EPSV", + "event.category": [ + "network" + ], "event.dataset": "zeek.ftp", "event.id": "CpQoCn3o28tke89zv9", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "info", + "protocol" + ], "fileset.name": "ftp", "input.type": "log", "log.offset": 0, "network.community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", "network.protocol": "ftp", "network.transport": "tcp", + "related.ip": [ + "192.168.1.182", + "192.168.1.231" + ], + "related.user": [ + "ftp" + ], "service.type": "zeek", "source.address": "192.168.1.182", "source.ip": "192.168.1.182", @@ -20,6 +37,7 @@ "tags": [ "zeek.ftp" ], + "user.name": "ftp", "zeek.ftp.command": "EPSV", "zeek.ftp.data_channel.originating_host": "192.168.1.182", "zeek.ftp.data_channel.passive": true, @@ -36,15 +54,33 @@ "destination.address": "192.168.1.231", "destination.ip": "192.168.1.231", "destination.port": 21, + "event.action": "RETR", + "event.category": [ + "network" + ], "event.dataset": "zeek.ftp", "event.id": "CpQoCn3o28tke89zv9", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "info", + "protocol" + ], + "file.size": 39424, "fileset.name": "ftp", "input.type": "log", "log.offset": 394, "network.community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", "network.protocol": "ftp", "network.transport": "tcp", + "related.ip": [ + "192.168.1.182", + "192.168.1.231" + ], + "related.user": [ + "ftp" + ], "service.type": "zeek", "source.address": "192.168.1.182", "source.ip": "192.168.1.182", @@ -52,6 +88,7 @@ "tags": [ "zeek.ftp" ], + "user.name": "ftp", "zeek.ftp.arg": "ftp://192.168.1.231/resume.doc", "zeek.ftp.command": "RETR", "zeek.ftp.file.size": 39424, @@ -66,15 +103,32 @@ "destination.address": "192.168.1.231", "destination.ip": "192.168.1.231", "destination.port": 21, + "event.action": "STOR", + "event.category": [ + "network" + ], "event.dataset": "zeek.ftp", "event.id": "CpQoCn3o28tke89zv9", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "info", + "protocol" + ], "fileset.name": "ftp", "input.type": "log", "log.offset": 688, "network.community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", "network.protocol": "ftp", "network.transport": "tcp", + "related.ip": [ + "192.168.1.182", + "192.168.1.231" + ], + "related.user": [ + "ftp" + ], "service.type": "zeek", "source.address": "192.168.1.182", "source.ip": "192.168.1.182", @@ -82,6 +136,7 @@ "tags": [ "zeek.ftp" ], + "user.name": "ftp", "zeek.ftp.arg": "ftp://192.168.1.231/uploads/README", "zeek.ftp.command": "STOR", "zeek.ftp.password": "ftp", diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml index 2c024397018..584160639cb 100644 --- a/x-pack/filebeat/module/zeek/http/config/http.yml +++ b/x-pack/filebeat/module/zeek/http/config/http.yml @@ -68,9 +68,26 @@ processors: ignore_missing: true fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "destination.port", to: "url.port"} + - {from: "http.request.method", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + - web + type: + - connection + - info + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json deleted file mode 100644 index af771f8c745..00000000000 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek http.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.http.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.http.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "set": { - "field": "url.port", - "value": "{{destination.port}}" - } - }, - { - "geoip": { - "field": "destination.ip", - "target_field": "destination.geo" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo" - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "destination.ip", - "target_field": "destination.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.asn", - "target_field": "destination.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.organization_name", - "target_field": "destination.as.organization.name", - "ignore_missing": true - } - }, - { - "user_agent": { - "field": "user_agent.original", - "ignore_missing": true - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml new file mode 100644 index 00000000000..62ffef0db45 --- /dev/null +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml @@ -0,0 +1,82 @@ +description: Pipeline for normalizing Zeek http.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.http.ts + formats: + - UNIX +- remove: + field: zeek.http.ts +- geoip: + field: destination.ip + target_field: destination.geo +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- user_agent: + field: user_agent.original + ignore_missing: true +- lowercase: + field: "http.request.method" + ignore_missing: true +- lowercase: + field: "event.action" + ignore_missing: true +- set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" +- set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{url.username}}" + if: "ctx?.url?.username != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/http/manifest.yml b/x-pack/filebeat/module/zeek/http/manifest.yml index a9ceabbaaa1..ddd253bb218 100644 --- a/x-pack/filebeat/module/zeek/http/manifest.yml +++ b/x-pack/filebeat/module/zeek/http/manifest.yml @@ -13,7 +13,7 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/http.yml requires.processors: diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json index 20d3fedb1c7..ee72065d771 100644 --- a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json @@ -13,12 +13,24 @@ "destination.geo.region_name": "California", "destination.ip": "17.253.5.203", "destination.port": 80, + "event.action": "get", + "event.category": [ + "network", + "web" + ], "event.dataset": "zeek.http", "event.id": "CCNp8v1SNzY7v9d1Ih", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "info", + "protocol" + ], "fileset.name": "http", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 3735, "http.response.status_code": 200, "http.version": "1.1", @@ -26,6 +38,10 @@ "log.offset": 0, "network.community_id": "1:dtBPRfpKEZyg1iOHss95buwv+cw=", "network.transport": "tcp", + "related.ip": [ + "10.178.98.102", + "17.253.5.203" + ], "service.type": "zeek", "source.address": "10.178.98.102", "source.ip": "10.178.98.102", @@ -35,7 +51,7 @@ ], "url.domain": "ocsp.apple.com", "url.original": "/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=", - "url.port": "80", + "url.port": 80, "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "com.apple.trustd/2.0", diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml index 38fe388bec0..2896ed72db9 100644 --- a/x-pack/filebeat/module/zeek/intel/config/intel.yml +++ b/x-pack/filebeat/module/zeek/intel/config/intel.yml @@ -61,3 +61,12 @@ processors: - zeek.intel.id.orig_p - zeek.intel.id.resp_h - zeek.intel.id.resp_p + - add_fields: + target: event + fields: + kind: alert + type: + - info +{{ if .community_id }} + - community_id: +{{ end }} diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml index 512cf67ff93..6a2bd6382ad 100644 --- a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml @@ -66,6 +66,15 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true + - append: + field: "related.ip" + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" + - append: + field: "related.ip" + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" + on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json index 1b2ac5464bf..d9de4e04efd 100644 --- a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json @@ -12,11 +12,19 @@ "destination.ip": "198.41.0.4", "destination.port": 53, "event.dataset": "zeek.intel", + "event.kind": "alert", "event.module": "zeek", "event.original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", + "event.type": [ + "info" + ], "fileset.name": "intel", "input.type": "log", "log.offset": 0, + "related.ip": [ + "192.168.1.1", + "198.41.0.4" + ], "service.type": "zeek", "source.address": "192.168.1.1", "source.ip": "192.168.1.1", diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml index 1ee45c0dc57..4d5783b8087 100644 --- a/x-pack/filebeat/module/zeek/irc/config/irc.yml +++ b/x-pack/filebeat/module/zeek/irc/config/irc.yml @@ -45,10 +45,28 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.irc.user", to: "user.name"} + - {from: "zeek.irc.command", to: "event.action"} + - {from: "zeek.irc.dcc.file.name", to: "file.name"} + - {from: "zeek.irc.dcc.file.size", to: "file.size"} + - {from: "zeek.irc.dcc.mime_type", to: "file.mime_type"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - info {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.json b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.json deleted file mode 100644 index 40723512349..00000000000 --- a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek irc.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.irc.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.irc.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml new file mode 100644 index 00000000000..ec04f4e7c93 --- /dev/null +++ b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml @@ -0,0 +1,65 @@ +description: Pipeline for normalizing Zeek irc.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.irc.ts + formats: + - UNIX +- remove: + field: zeek.irc.ts +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- geoip: + field: destination.ip + target_field: destination.geo +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/irc/manifest.yml b/x-pack/filebeat/module/zeek/irc/manifest.yml index ce7cd7b714e..3bf899fd2c0 100644 --- a/x-pack/filebeat/module/zeek/irc/manifest.yml +++ b/x-pack/filebeat/module/zeek/irc/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/irc.yml diff --git a/x-pack/filebeat/module/zeek/irc/test/irc-json.log-expected.json b/x-pack/filebeat/module/zeek/irc/test/irc-json.log-expected.json index 2a12e671ea5..245d1154e86 100644 --- a/x-pack/filebeat/module/zeek/irc/test/irc-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/irc/test/irc-json.log-expected.json @@ -2,17 +2,37 @@ { "@timestamp": "2013-12-20T15:44:10.647Z", "destination.address": "38.229.70.20", + "destination.as.number": 23028, + "destination.as.organization.name": "Team Cymru Inc.", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, "destination.ip": "38.229.70.20", "destination.port": 8000, + "event.action": "USER", + "event.category": [ + "network" + ], "event.dataset": "zeek.irc", "event.id": "CNJBX5FQdL62VUUP1", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "irc", "input.type": "log", "log.offset": 0, "network.community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", "network.protocol": "irc", "network.transport": "tcp", + "related.ip": [ + "10.180.156.249", + "38.229.70.20" + ], "service.type": "zeek", "source.address": "10.180.156.249", "source.ip": "10.180.156.249", @@ -28,17 +48,40 @@ { "@timestamp": "2013-12-20T15:44:10.647Z", "destination.address": "38.229.70.20", + "destination.as.number": 23028, + "destination.as.organization.name": "Team Cymru Inc.", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, "destination.ip": "38.229.70.20", "destination.port": 8000, + "event.action": "NICK", + "event.category": [ + "network" + ], "event.dataset": "zeek.irc", "event.id": "CNJBX5FQdL62VUUP1", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "irc", "input.type": "log", "log.offset": 206, "network.community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", "network.protocol": "irc", "network.transport": "tcp", + "related.ip": [ + "10.180.156.249", + "38.229.70.20" + ], + "related.user": [ + "xxxxx" + ], "service.type": "zeek", "source.address": "10.180.156.249", "source.ip": "10.180.156.249", @@ -46,6 +89,7 @@ "tags": [ "zeek.irc" ], + "user.name": "xxxxx", "zeek.irc.addl": "+iw xxxxx XxxxxxXxxx ", "zeek.irc.command": "NICK", "zeek.irc.user": "xxxxx", @@ -55,17 +99,40 @@ { "@timestamp": "2013-12-20T15:44:10.706Z", "destination.address": "38.229.70.20", + "destination.as.number": 23028, + "destination.as.organization.name": "Team Cymru Inc.", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, "destination.ip": "38.229.70.20", "destination.port": 8000, + "event.action": "JOIN", + "event.category": [ + "network" + ], "event.dataset": "zeek.irc", "event.id": "CNJBX5FQdL62VUUP1", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "irc", "input.type": "log", "log.offset": 432, "network.community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", "network.protocol": "irc", "network.transport": "tcp", + "related.ip": [ + "10.180.156.249", + "38.229.70.20" + ], + "related.user": [ + "xxxxx" + ], "service.type": "zeek", "source.address": "10.180.156.249", "source.ip": "10.180.156.249", @@ -73,6 +140,7 @@ "tags": [ "zeek.irc" ], + "user.name": "xxxxx", "zeek.irc.addl": " with channel key: '-'", "zeek.irc.command": "JOIN", "zeek.irc.nick": "molochtest", diff --git a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml index 4bbcf677b70..28c49507406 100644 --- a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml +++ b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml @@ -72,10 +72,33 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "source.address", to: "client.address"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "destination.address", to: "server.address"} + - {from: "zeek.kerberos.request_type", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - authentication + - dissect: + when: + contains: + zeek.kerberos.client: "/" + tokenizer: "%{user.name}/%{user.domain}" + field: zeek.kerberos.client + target_prefix: "" {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.json b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.json deleted file mode 100644 index 988e9b7f2b1..00000000000 --- a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek kerberos.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.kerberos.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.kerberos.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "client.address", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "server.address", - "value": "{{destination.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "script": { - "source": "ctx.zeek.kerberos.valid.days = Math.round( (ctx.zeek.kerberos.valid.until - ctx.zeek.kerberos.valid.from) / 86400 )", - "if": "ctx.zeek.kerberos.valid?.from != null && ctx.zeek.kerberos.valid?.until != null" - } - }, - { - "date": { - "field": "zeek.kerberos.valid.until", - "target_field": "zeek.kerberos.valid.until", - "formats": ["UNIX"], - "if": "ctx.zeek.kerberos.valid?.until != null" - } - }, - { - "date": { - "field": "zeek.kerberos.valid.from", - "target_field": "zeek.kerberos.valid.from", - "formats": ["UNIX"], - "if": "ctx.zeek.kerberos.valid?.from != null" - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml new file mode 100644 index 00000000000..05005491115 --- /dev/null +++ b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml @@ -0,0 +1,90 @@ +description: Pipeline for normalizing Zeek kerberos.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.kerberos.ts + formats: + - UNIX +- remove: + field: zeek.kerberos.ts +- script: + source: "ctx.zeek.kerberos.valid.days = Math.round( (ctx.zeek.kerberos.valid.until - ctx.zeek.kerberos.valid.from) / 86400 )" + if: "ctx.zeek.kerberos.valid?.from != null && ctx.zeek.kerberos.valid?.until != null" +- date: + field: zeek.kerberos.valid.until + target_field: zeek.kerberos.valid.until + formats: + - UNIX + if: ctx.zeek.kerberos.valid?.until != null +- date: + field: zeek.kerberos.valid.from + target_field: zeek.kerberos.valid.from + formats: + - UNIX + if: ctx.zeek.kerberos.valid?.from != null +- set: + field: event.outcome + value: success + if: "ctx?.zeek?.kerberos?.success == true" +- set: + field: event.outcome + value: failure + if: "ctx?.zeek?.kerberos?.success == false" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/kerberos/manifest.yml b/x-pack/filebeat/module/zeek/kerberos/manifest.yml index a2e040be371..4a94434f1d4 100644 --- a/x-pack/filebeat/module/zeek/kerberos/manifest.yml +++ b/x-pack/filebeat/module/zeek/kerberos/manifest.yml @@ -13,7 +13,7 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/kerberos.yml requires.processors: diff --git a/x-pack/filebeat/module/zeek/kerberos/test/kerberos-json.log-expected.json b/x-pack/filebeat/module/zeek/kerberos/test/kerberos-json.log-expected.json index a09e3ac8a4f..e01e42a4036 100644 --- a/x-pack/filebeat/module/zeek/kerberos/test/kerberos-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/kerberos/test/kerberos-json.log-expected.json @@ -5,15 +5,33 @@ "destination.address": "192.168.10.10", "destination.ip": "192.168.10.10", "destination.port": 88, + "event.action": "TGS", + "event.category": [ + "network" + ], "event.dataset": "zeek.kerberos", "event.id": "C56Flhb4WQBNkfMOl", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "protocol", + "authentication" + ], "fileset.name": "kerberos", "input.type": "log", "log.offset": 0, "network.community_id": "1:DW/lSsosl8gZ8pqO9kKMm7cZheQ=", "network.protocol": "kerberos", "network.transport": "tcp", + "related.ip": [ + "192.168.10.31", + "192.168.10.10" + ], + "related.user": [ + "RonHD" + ], "server.address": "192.168.10.10", "service.type": "zeek", "source.address": "192.168.10.31", @@ -22,6 +40,8 @@ "tags": [ "zeek.kerberos" ], + "user.domain": "CONTOSO.LOCAL", + "user.name": "RonHD", "zeek.kerberos.cipher": "aes256-cts-hmac-sha1-96", "zeek.kerberos.client": "RonHD/CONTOSO.LOCAL", "zeek.kerberos.forwardable": true, diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml index fec2b954224..6dc8c3004d4 100644 --- a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml +++ b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml @@ -39,10 +39,35 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.modbus.function", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol + - if: + has_fields: ['zeek.modbus.exception'] + then: + - add_fields: + target: event + fields: + outcome: failure + else: + - add_fields: + target: event + fields: + outcome: success {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.json b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.json deleted file mode 100644 index 78026f2dc87..00000000000 --- a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek modbus.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.modbus.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.modbus.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml new file mode 100644 index 00000000000..d053a541ef5 --- /dev/null +++ b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek modbus.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.modbus.ts + formats: + - UNIX +- remove: + field: zeek.modbus.ts +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/modbus/manifest.yml b/x-pack/filebeat/module/zeek/modbus/manifest.yml index 98e51ae2bec..e20412fadc6 100644 --- a/x-pack/filebeat/module/zeek/modbus/manifest.yml +++ b/x-pack/filebeat/module/zeek/modbus/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/modbus.yml diff --git a/x-pack/filebeat/module/zeek/modbus/test/modbus-json.log-expected.json b/x-pack/filebeat/module/zeek/modbus/test/modbus-json.log-expected.json index 9817176e098..ba9034a3621 100644 --- a/x-pack/filebeat/module/zeek/modbus/test/modbus-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/modbus/test/modbus-json.log-expected.json @@ -4,15 +4,29 @@ "destination.address": "192.168.1.164", "destination.ip": "192.168.1.164", "destination.port": 502, + "event.action": "READ_COILS", + "event.category": [ + "network" + ], "event.dataset": "zeek.modbus", "event.id": "CpIIXl4DFGswmjH2bl", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "modbus", "input.type": "log", "log.offset": 0, "network.community_id": "1:jEXbR2FqHyMgLJgyYyFQN3yxbpc=", "network.protocol": "modbus", "network.transport": "tcp", + "related.ip": [ + "192.168.1.10", + "192.168.1.164" + ], "service.type": "zeek", "source.address": "192.168.1.10", "source.ip": "192.168.1.10", diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml index fcd226131bc..b28262b5bd5 100644 --- a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml +++ b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml @@ -36,10 +36,37 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.mysql.cmd", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - database + - network + type: + - connection + - protocol + - if: + equals: + zeek.mysql.success: true + then: + - add_fields: + target: event + fields: + outcome: success + else: + - add_fields: + target: event + fields: + outcome: failure {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.json b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.json deleted file mode 100644 index ec55df982d7..00000000000 --- a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek mysql.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.mysql.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.mysql.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml new file mode 100644 index 00000000000..ca2c6c57172 --- /dev/null +++ b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml @@ -0,0 +1,83 @@ +description: Pipeline for normalizing Zeek mysql.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.mysql.ts + formats: + - UNIX +- remove: + field: zeek.mysql.ts +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: event.type + value: access + if: "ctx?.zeek?.mysql?.cmd != null && (ctx.zeek.mysql.cmd == 'connect' || ctx.zeek.mysql.cmd == 'connect_out')" +- append: + field: event.type + value: change + if: "ctx?.zeek?.mysql?.cmd != null && (ctx.zeek.mysql.cmd == 'init_db' || ctx.zeek.mysql.cmd == 'change_user' || ctx.zeek.mysql.cmd == 'set_option' || ctx.zeek.mysql.cmd == 'drop_db' || ctx.zeek.mysql.cmd == 'create_db' || ctx.zeek.mysql.cmd == 'process_kill' || ctx.zeek.mysql.cmd == 'delayed_insert')" +- append: + field: event.type + value: info + if: "ctx?.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd != 'init_db' && ctx.zeek.mysql.cmd != 'change_user' && ctx.zeek.mysql.cmd != 'set_option' && ctx.zeek.mysql.cmd != 'drop_db' && ctx.zeek.mysql.cmd != 'create_db' && ctx.zeek.mysql.cmd != 'process_kill' && ctx.zeek.mysql.cmd != 'delayed_insert' && ctx.zeek.mysql.cmd != 'connect' && ctx.zeek.mysql.cmd != 'connect_out'" +- append: + field: event.type + value: start + if: "ctx?.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd == 'connect'" +- append: + field: event.type + value: end + if: "ctx?.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd == 'connect_out'" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/mysql/manifest.yml b/x-pack/filebeat/module/zeek/mysql/manifest.yml index a16c6092cc7..1b7ec4edb19 100644 --- a/x-pack/filebeat/module/zeek/mysql/manifest.yml +++ b/x-pack/filebeat/module/zeek/mysql/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/mysql.yml diff --git a/x-pack/filebeat/module/zeek/mysql/test/mysql-json.log-expected.json b/x-pack/filebeat/module/zeek/mysql/test/mysql-json.log-expected.json index 279b1019404..bf68cae48fe 100644 --- a/x-pack/filebeat/module/zeek/mysql/test/mysql-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/mysql/test/mysql-json.log-expected.json @@ -4,15 +4,31 @@ "destination.address": "192.168.0.254", "destination.ip": "192.168.0.254", "destination.port": 3306, + "event.action": "query", + "event.category": [ + "database", + "network" + ], "event.dataset": "zeek.mysql", "event.id": "C5Hol527kLMUw36hj3", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "protocol", + "info" + ], "fileset.name": "mysql", "input.type": "log", "log.offset": 0, "network.community_id": "1:0HUQbshhYbATQXDHv/ysOs0DlZA=", "network.protocol": "mysql", "network.transport": "tcp", + "related.ip": [ + "192.168.0.254", + "192.168.0.254" + ], "service.type": "zeek", "source.address": "192.168.0.254", "source.ip": "192.168.0.254", diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml index 7f5c9c0869c..32ab849b6b5 100644 --- a/x-pack/filebeat/module/zeek/notice/config/notice.yml +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -78,9 +78,25 @@ processors: - drop_fields: fields: ["zeek.notice.remote_location", "zeek.notice.f"] + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.notice.file.total_bytes", to: "file.size"} + - {from: "zeek.notice.file.mime_type", to: "file.mime_type"} + - {from: "zeek.notice.note", to: "rule.name"} + - {from: "zeek.notice.msg", to: "rule.description"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: alert + category: + - intrusion_detection + type: + - info {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json deleted file mode 100644 index b343068d6c6..00000000000 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek notice.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.notice.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.notice.ts" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}", - "if": "ctx.destination?.address != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}", - "if": "ctx.source?.address != null" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "geoip": { - "field": "destination.ip", - "target_field": "destination.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "destination.ip", - "target_field": "destination.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.asn", - "target_field": "destination.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.organization_name", - "target_field": "destination.as.organization.name", - "ignore_missing": true - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml new file mode 100644 index 00000000000..c4dee6b78f2 --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml @@ -0,0 +1,71 @@ +description: Pipeline for normalizing Zeek notice.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.notice.ts + formats: + - UNIX +- remove: + field: zeek.notice.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: event.type + value: allowed + if: "ctx?.zeek?.notice?.dropped == false" +- append: + field: event.type + value: denied + if: "ctx?.zeek?.notice?.dropped == true" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/notice/manifest.yml b/x-pack/filebeat/module/zeek/notice/manifest.yml index 7b98a8efefc..e2bdf695027 100644 --- a/x-pack/filebeat/module/zeek/notice/manifest.yml +++ b/x-pack/filebeat/module/zeek/notice/manifest.yml @@ -13,7 +13,7 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/notice.yml requires.processors: diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json index 58a59ab4d7b..a5838e9f3f1 100644 --- a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json @@ -1,11 +1,24 @@ [ { "@timestamp": "2011-11-04T19:44:35.879Z", + "event.category": [ + "intrusion_detection" + ], "event.dataset": "zeek.notice", + "event.kind": "alert", "event.module": "zeek", + "event.type": [ + "info", + "allowed" + ], "fileset.name": "notice", "input.type": "log", "log.offset": 0, + "related.ip": [ + "172.16.238.1" + ], + "rule.description": "172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).", + "rule.name": "SSH::Password_Guessing", "service.type": "zeek", "source.address": "172.16.238.1", "source.ip": "172.16.238.1", @@ -32,11 +45,25 @@ "destination.geo.region_iso_code": "DE-HE", "destination.geo.region_name": "Hesse", "destination.ip": "207.154.238.205", + "event.category": [ + "intrusion_detection" + ], "event.dataset": "zeek.notice", + "event.kind": "alert", "event.module": "zeek", + "event.type": [ + "info", + "allowed" + ], "fileset.name": "notice", "input.type": "log", "log.offset": 357, + "related.ip": [ + "8.42.77.171", + "207.154.238.205" + ], + "rule.description": "8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s", + "rule.name": "Scan::Port_Scan", "service.type": "zeek", "source.address": "8.42.77.171", "source.as.number": 393552, diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml index 76cfecaaf54..55a6795b6fa 100644 --- a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml +++ b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml @@ -48,10 +48,39 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.ntlm.username", to: "user.name"} + - {from: "zeek.ntlm.domain", to: "user.domain"} + - add_fields: + target: event + fields: + kind: event + category: + - authentication + - network + type: + - info + - connection + - if: + equals: + zeek.ntlm.success: true + then: + - add_fields: + target: event + fields: + outcome: success + - if: + equals: + zeek.ntlm.success: false + then: + - add_fields: + target: event + fields: + outcome: failure {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.json deleted file mode 100644 index 680ea8815e0..00000000000 --- a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek ntlm.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.ntlm.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.ntlm.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml new file mode 100644 index 00000000000..9f76d461392 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml @@ -0,0 +1,67 @@ +description: Pipeline for normalizing Zeek ntlm.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.ntlm.ts + formats: + - UNIX +- remove: + field: zeek.ntlm.ts +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/ntlm/manifest.yml b/x-pack/filebeat/module/zeek/ntlm/manifest.yml index 0248af27d3b..545bef85aaa 100644 --- a/x-pack/filebeat/module/zeek/ntlm/manifest.yml +++ b/x-pack/filebeat/module/zeek/ntlm/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/ntlm.yml diff --git a/x-pack/filebeat/module/zeek/ntlm/test/ntlm-json.log-expected.json b/x-pack/filebeat/module/zeek/ntlm/test/ntlm-json.log-expected.json index 90aebbec10b..c85d3127476 100644 --- a/x-pack/filebeat/module/zeek/ntlm/test/ntlm-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ntlm/test/ntlm-json.log-expected.json @@ -4,15 +4,31 @@ "destination.address": "192.168.10.31", "destination.ip": "192.168.10.31", "destination.port": 445, + "event.category": [ + "authentication", + "network" + ], "event.dataset": "zeek.ntlm", "event.id": "CHphiNUKDC20fsy09", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "info", + "connection" + ], "fileset.name": "ntlm", "input.type": "log", "log.offset": 0, "network.community_id": "1:zxnXAE/Cme5fQhh6sJLs7GItc08=", "network.protocol": "ntlm", "network.transport": "tcp", + "related.ip": [ + "192.168.10.50", + "192.168.10.31" + ], + "related.user": [ + "JeffV" + ], "service.type": "zeek", "source.address": "192.168.10.50", "source.ip": "192.168.10.50", @@ -20,6 +36,8 @@ "tags": [ "zeek.ntlm" ], + "user.domain": "contoso.local", + "user.name": "JeffV", "zeek.ntlm.domain": "contoso.local", "zeek.ntlm.hostname": "ybaARon55QykXrgu", "zeek.ntlm.server.name.dns": "Victim-PC.contoso.local", diff --git a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml index a6a74d6d05e..f6298a36d1e 100644 --- a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml +++ b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml @@ -56,3 +56,7 @@ processors: ignore_missing: true fail_on_error: false + - add_fields: + target: event + fields: + kind: event diff --git a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.json deleted file mode 100644 index e56642bd4a8..00000000000 --- a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek ocsp.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.ocsp.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.ocsp.ts" - } - }, - { - "date": { - "field": "zeek.ocsp.revoke.date", - "target_field": "zeek.ocsp.revoke.date", - "formats": ["UNIX"], - "if": "ctx.zeek.ocsp.revoke?.date != null" - } - }, - { - "date": { - "field": "zeek.ocsp.update.this", - "target_field": "zeek.ocsp.update.this", - "formats": ["UNIX"], - "if": "ctx.zeek.ocsp.update?.this != null" - } - }, - { - "date": { - "field": "zeek.ocsp.update.next", - "target_field": "zeek.ocsp.update.next", - "formats": ["UNIX"], - "if": "ctx.zeek.ocsp.update?.next != null" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml new file mode 100644 index 00000000000..63a878825d7 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml @@ -0,0 +1,41 @@ +description: Pipeline for normalizing Zeek ocsp.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.ocsp.ts + formats: + - UNIX +- remove: + field: zeek.ocsp.ts +- date: + field: zeek.ocsp.revoke.date + target_field: zeek.ocsp.revoke.date + formats: + - UNIX + if: ctx.zeek.ocsp.revoke?.date != null +- date: + field: zeek.ocsp.update.this + target_field: zeek.ocsp.update.this + formats: + - UNIX + if: ctx.zeek.ocsp.update?.this != null +- date: + field: zeek.ocsp.update.next + target_field: zeek.ocsp.update.next + formats: + - UNIX + if: ctx.zeek.ocsp.update?.next != null +- append: + field: related.hash + value: "{{zeek.ocsp.issuerNameHash}}" + if: "ctx?.zeek?.ocsp?.issuerNameHash != null" +- append: + field: related.hash + value: "{{zeek.ocsp.issuerKeyHash}}" + if: "ctx?.zeek?.ocsp?.issuerKeyHash != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/ocsp/manifest.yml b/x-pack/filebeat/module/zeek/ocsp/manifest.yml index 739873d645f..35bcfccdcb6 100644 --- a/x-pack/filebeat/module/zeek/ocsp/manifest.yml +++ b/x-pack/filebeat/module/zeek/ocsp/manifest.yml @@ -11,5 +11,5 @@ var: - name: tags default: [zeek.ocsp] -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/ocsp.yml diff --git a/x-pack/filebeat/module/zeek/pe/config/pe.yml b/x-pack/filebeat/module/zeek/pe/config/pe.yml index ee4c78bb8cc..cf5f54396ad 100644 --- a/x-pack/filebeat/module/zeek/pe/config/pe.yml +++ b/x-pack/filebeat/module/zeek/pe/config/pe.yml @@ -21,3 +21,11 @@ processors: ignore_missing: true fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - file + type: + - info diff --git a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.json b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.json deleted file mode 100644 index f950772464c..00000000000 --- a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek pe.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.pe.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.pe.ts" - } - }, - { - "date": { - "field": "zeek.pe.compile_time", - "target_field": "zeek.pe.compile_time", - "formats": ["UNIX"], - "if": "ctx.zeek.pe.compile_time != null" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml new file mode 100644 index 00000000000..6a7fa7dca87 --- /dev/null +++ b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml @@ -0,0 +1,21 @@ +description: Pipeline for normalizing Zeek pe.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.pe.ts + formats: + - UNIX +- remove: + field: zeek.pe.ts +- date: + field: zeek.pe.compile_time + target_field: zeek.pe.compile_time + formats: + - UNIX + if: ctx.zeek.pe.compile_time != null +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/pe/manifest.yml b/x-pack/filebeat/module/zeek/pe/manifest.yml index 02a352c5dfd..16dfe2e4634 100644 --- a/x-pack/filebeat/module/zeek/pe/manifest.yml +++ b/x-pack/filebeat/module/zeek/pe/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/pe.yml diff --git a/x-pack/filebeat/module/zeek/pe/test/pe-json.log-expected.json b/x-pack/filebeat/module/zeek/pe/test/pe-json.log-expected.json index ccad0e8e2fc..3356f0ef793 100644 --- a/x-pack/filebeat/module/zeek/pe/test/pe-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/pe/test/pe-json.log-expected.json @@ -1,8 +1,15 @@ [ { "@timestamp": "2017-10-09T16:13:19.578Z", + "event.category": [ + "file" + ], "event.dataset": "zeek.pe", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "info" + ], "fileset.name": "pe", "input.type": "log", "log.offset": 0, diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml index fdbb468450c..38338b1c84f 100644 --- a/x-pack/filebeat/module/zeek/radius/config/radius.yml +++ b/x-pack/filebeat/module/zeek/radius/config/radius.yml @@ -36,10 +36,23 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.radius.username", to: "user.name"} + - {from: "zeek.radius.result", to: "event.outcome"} + - add_fields: + target: event + fields: + kind: event + category: + - authentication + - network + type: + - info + - connection {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.json b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.json deleted file mode 100644 index 72f645dd651..00000000000 --- a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek radius.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.radius.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.radius.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml new file mode 100644 index 00000000000..c69dfaefbb4 --- /dev/null +++ b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml @@ -0,0 +1,67 @@ +description: Pipeline for normalizing Zeek radius.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.radius.ts + formats: + - UNIX +- remove: + field: zeek.radius.ts +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/radius/manifest.yml b/x-pack/filebeat/module/zeek/radius/manifest.yml index 505abcbbbd6..f881f404d7a 100644 --- a/x-pack/filebeat/module/zeek/radius/manifest.yml +++ b/x-pack/filebeat/module/zeek/radius/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/radius.yml diff --git a/x-pack/filebeat/module/zeek/radius/test/radius-json.log-expected.json b/x-pack/filebeat/module/zeek/radius/test/radius-json.log-expected.json index 9b4ddfa91f2..894b85f435f 100644 --- a/x-pack/filebeat/module/zeek/radius/test/radius-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/radius/test/radius-json.log-expected.json @@ -4,15 +4,32 @@ "destination.address": "10.0.0.100", "destination.ip": "10.0.0.100", "destination.port": 1812, + "event.category": [ + "authentication", + "network" + ], "event.dataset": "zeek.radius", "event.id": "CRe9VD3flCDWbPmpIh", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "info", + "connection" + ], "fileset.name": "radius", "input.type": "log", "log.offset": 0, "network.community_id": "1:3SdDgWXPnheV2oGfVmxQjfwtr8E=", "network.protocol": "radius", "network.transport": "udp", + "related.ip": [ + "10.0.0.1", + "10.0.0.100" + ], + "related.user": [ + "John.McGuirk" + ], "service.type": "zeek", "source.address": "10.0.0.1", "source.ip": "10.0.0.1", @@ -20,6 +37,7 @@ "tags": [ "zeek.radius" ], + "user.name": "John.McGuirk", "zeek.radius.mac": "00:14:22:e9:54:5e", "zeek.radius.result": "success", "zeek.radius.username": "John.McGuirk", diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml index d9dac8f2e9b..b9b19e79dd7 100644 --- a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml +++ b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml @@ -69,10 +69,20 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - protocol + - info {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.json deleted file mode 100644 index ae56b98801f..00000000000 --- a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek rdp.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.rdp.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.rdp.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "convert": { - "field": "zeek.rdp.ssl", - "target_field": "tls.established", - "type": "boolean", - "ignore_missing": true - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml new file mode 100644 index 00000000000..d6b70dd92e6 --- /dev/null +++ b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml @@ -0,0 +1,68 @@ +description: Pipeline for normalizing Zeek rdp.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.rdp.ts + formats: + - UNIX +- remove: + field: zeek.rdp.ts +- convert: + field: zeek.rdp.ssl + target_field: tls.established + type: boolean + ignore_missing: true +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/rdp/manifest.yml b/x-pack/filebeat/module/zeek/rdp/manifest.yml index 044352bb2fd..b0c76c9f3a3 100644 --- a/x-pack/filebeat/module/zeek/rdp/manifest.yml +++ b/x-pack/filebeat/module/zeek/rdp/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/rdp.yml diff --git a/x-pack/filebeat/module/zeek/rdp/test/rdp-json.log-expected.json b/x-pack/filebeat/module/zeek/rdp/test/rdp-json.log-expected.json index 6d39caef60b..878eb3e2050 100644 --- a/x-pack/filebeat/module/zeek/rdp/test/rdp-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/rdp/test/rdp-json.log-expected.json @@ -4,15 +4,27 @@ "destination.address": "192.168.131.131", "destination.ip": "192.168.131.131", "destination.port": 3389, + "event.category": [ + "network" + ], "event.dataset": "zeek.rdp", "event.id": "C2PcYV7D3ntaHm056", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "protocol", + "info" + ], "fileset.name": "rdp", "input.type": "log", "log.offset": 0, "network.community_id": "1:PsQu6lSZioPVi0A5K7UaeGsVqS0=", "network.protocol": "rdp", "network.transport": "tcp", + "related.ip": [ + "192.168.131.1", + "192.168.131.131" + ], "service.type": "zeek", "source.address": "192.168.131.1", "source.ip": "192.168.131.1", diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml index 61e984131cd..f9a2618b02b 100644 --- a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml +++ b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml @@ -54,10 +54,20 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - info {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.json b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.json deleted file mode 100644 index 14ae112ffea..00000000000 --- a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek rfb.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.rfb.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.rfb.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml new file mode 100644 index 00000000000..8cf2cebdf4d --- /dev/null +++ b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek rfb.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.rfb.ts + formats: + - UNIX +- remove: + field: zeek.rfb.ts +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/rfb/manifest.yml b/x-pack/filebeat/module/zeek/rfb/manifest.yml index 2f96e4f618e..2b9daaab107 100644 --- a/x-pack/filebeat/module/zeek/rfb/manifest.yml +++ b/x-pack/filebeat/module/zeek/rfb/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/rfb.yml diff --git a/x-pack/filebeat/module/zeek/rfb/test/rfb-json.log-expected.json b/x-pack/filebeat/module/zeek/rfb/test/rfb-json.log-expected.json index c860f5377e3..83b5544b655 100644 --- a/x-pack/filebeat/module/zeek/rfb/test/rfb-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/rfb/test/rfb-json.log-expected.json @@ -4,15 +4,27 @@ "destination.address": "192.168.1.10", "destination.ip": "192.168.1.10", "destination.port": 5900, + "event.category": [ + "network" + ], "event.dataset": "zeek.rfb", "event.id": "CXoIzM3wH3fUwXtKN1", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "info" + ], "fileset.name": "rfb", "input.type": "log", "log.offset": 0, "network.community_id": "1:AtPVA5phuztnwqMfO/2142WXVdY=", "network.protocol": "rfb", "network.transport": "tcp", + "related.ip": [ + "192.168.1.123", + "192.168.1.10" + ], "service.type": "zeek", "source.address": "192.168.1.123", "source.ip": "192.168.1.123", diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml index bd22de69672..c94dbe5e40e 100644 --- a/x-pack/filebeat/module/zeek/sip/config/sip.yml +++ b/x-pack/filebeat/module/zeek/sip/config/sip.yml @@ -72,10 +72,24 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.sip.sequence.method", to: "event.action"} + - {from: "zeek.sip.uri", to: "url.full"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.json b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.json deleted file mode 100644 index c3b7eab58fb..00000000000 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek sip.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.sip.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.sip.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "grok": { - "field": "zeek.sip.seq", - "patterns": ["%{NUMBER:zeek.sip.sequence.number}"], - "ignore_missing": true - } - }, - { - "remove": { - "field": "zeek.sip.seq", - "ignore_missing": true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml new file mode 100644 index 00000000000..9982cb82d87 --- /dev/null +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -0,0 +1,83 @@ +description: Pipeline for normalizing Zeek sip.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.sip.ts + formats: + - UNIX +- remove: + field: zeek.sip.ts +- grok: + field: zeek.sip.seq + patterns: + - '%{NUMBER:zeek.sip.sequence.number}' + ignore_missing: true +- remove: + field: zeek.sip.seq + ignore_missing: true +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: event.type + value: error + if: "ctx?.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code >= 400" +- set: + field: event.outcome + value: failure + if: "ctx?.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code >= 400" +- set: + field: event.outcome + value: success + if: "ctx?.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code < 400" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/sip/manifest.yml b/x-pack/filebeat/module/zeek/sip/manifest.yml index 8b022a943af..8da0cc443dd 100644 --- a/x-pack/filebeat/module/zeek/sip/manifest.yml +++ b/x-pack/filebeat/module/zeek/sip/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/sip.yml diff --git a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json index c24f5405435..79b38a0717d 100644 --- a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json @@ -2,17 +2,38 @@ { "@timestamp": "2013-02-26T22:02:39.055Z", "destination.address": "74.63.41.218", + "destination.as.number": 29791, + "destination.as.organization.name": "Internap Corporation", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, "destination.ip": "74.63.41.218", "destination.port": 5060, + "event.action": "REGISTER", + "event.category": [ + "network" + ], "event.dataset": "zeek.sip", "event.id": "CPRLCB4eWHdjP852Bk", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "failure", + "event.type": [ + "connection", + "protocol", + "error" + ], "fileset.name": "sip", "input.type": "log", "log.offset": 0, "network.community_id": "1:t8Jl0amIXPHemzxKgsLjtkB+ewo=", "network.protocol": "sip", "network.transport": "udp", + "related.ip": [ + "172.16.133.19", + "74.63.41.218" + ], "service.type": "zeek", "source.address": "172.16.133.19", "source.ip": "172.16.133.19", @@ -20,6 +41,7 @@ "tags": [ "zeek.sip" ], + "url.full": "sip:newyork.voip.ms:5060", "zeek.session_id": "CPRLCB4eWHdjP852Bk", "zeek.sip.call_id": "8694cd7e-976e4fc3-d76f6e38@172.16.133.19", "zeek.sip.request.body_length": 0, @@ -45,24 +67,57 @@ { "@timestamp": "2005-01-14T17:58:02.965Z", "destination.address": "200.57.7.195", + "destination.as.number": 18734, + "destination.as.organization.name": "Operbes, S.A. de C.V.", + "destination.geo.city_name": "Mexico City", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "MX", + "destination.geo.location.lat": 19.4357, + "destination.geo.location.lon": -99.1438, + "destination.geo.region_iso_code": "MX-CMX", + "destination.geo.region_name": "Mexico City", "destination.ip": "200.57.7.195", "destination.port": 5060, + "event.action": "INVITE", + "event.category": [ + "network" + ], "event.dataset": "zeek.sip", "event.id": "ComJz236lSOcuOmix3", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "sip", "input.type": "log", "log.offset": 805, "network.community_id": "1:U/Makwsc8lm6pVKLfRMzoNTI++0=", "network.protocol": "sip", "network.transport": "udp", + "related.ip": [ + "200.57.7.204", + "200.57.7.195" + ], "service.type": "zeek", "source.address": "200.57.7.204", + "source.as.number": 18734, + "source.as.organization.name": "Operbes, S.A. de C.V.", + "source.geo.city_name": "Mexico City", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "MX", + "source.geo.location.lat": 19.4357, + "source.geo.location.lon": -99.1438, + "source.geo.region_iso_code": "MX-CMX", + "source.geo.region_name": "Mexico City", "source.ip": "200.57.7.204", "source.port": 5061, "tags": [ "zeek.sip" ], + "url.full": "sip:francisco@bestel.com:55060", "zeek.session_id": "ComJz236lSOcuOmix3", "zeek.sip.call_id": "12013223@200.57.7.195", "zeek.sip.request.body_length": 229, @@ -91,24 +146,57 @@ { "@timestamp": "2005-01-14T17:58:07.022Z", "destination.address": "200.57.7.195", + "destination.as.number": 18734, + "destination.as.organization.name": "Operbes, S.A. de C.V.", + "destination.geo.city_name": "Mexico City", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "MX", + "destination.geo.location.lat": 19.4357, + "destination.geo.location.lon": -99.1438, + "destination.geo.region_iso_code": "MX-CMX", + "destination.geo.region_name": "Mexico City", "destination.ip": "200.57.7.195", "destination.port": 5060, + "event.action": "REGISTER", + "event.category": [ + "network" + ], "event.dataset": "zeek.sip", "event.id": "CJZDWgixtwqXctWEg", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "sip", "input.type": "log", "log.offset": 1654, "network.community_id": "1:0hvHF/bh5wFKg7nfRXxsno4F198=", "network.protocol": "sip", "network.transport": "udp", + "related.ip": [ + "200.57.7.205", + "200.57.7.195" + ], "service.type": "zeek", "source.address": "200.57.7.205", + "source.as.number": 18734, + "source.as.organization.name": "Operbes, S.A. de C.V.", + "source.geo.city_name": "Mexico City", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "MX", + "source.geo.location.lat": 19.4357, + "source.geo.location.lon": -99.1438, + "source.geo.region_iso_code": "MX-CMX", + "source.geo.region_name": "Mexico City", "source.ip": "200.57.7.205", "source.port": 5061, "tags": [ "zeek.sip" ], + "url.full": "sip:Verso.com", "zeek.session_id": "CJZDWgixtwqXctWEg", "zeek.sip.call_id": "46E1C3CB36304F84A020CF6DD3F96461@Verso.com", "zeek.sip.request.body_length": 0, diff --git a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml index d9839c7dc16..ada63493d6f 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml @@ -78,10 +78,24 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.smb_cmd.command", to: "event.action"} + - {from: "zeek.smb_cmd.username", to: "user.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.json b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.json deleted file mode 100644 index 6b1f7f1b2af..00000000000 --- a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek smb_cmd.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.smb_cmd.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.smb_cmd.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "remove": { - "field": "zeek.smb_cmd.referenced_file", - "ignore_missing": true - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml new file mode 100644 index 00000000000..838e9f2e8bc --- /dev/null +++ b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml @@ -0,0 +1,82 @@ +description: Pipeline for normalizing Zeek smb_cmd.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.smb_cmd.ts + formats: + - UNIX +- remove: + field: zeek.smb_cmd.ts +- remove: + field: zeek.smb_cmd.referenced_file + ignore_missing: true +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- append: + field: event.type + value: error + if: "ctx?.zeek?.smb_cmd?.status != null && ctx.zeek.smb_cmd.status.toLowerCase() != 'success'" +- set: + field: event.outcome + value: success + if: "ctx?.zeek?.smb_cmd?.status != null && ctx.zeek.smb_cmd.status.toLowerCase() == 'success'" +- set: + field: event.outcome + value: failure + if: "ctx?.zeek?.smb_cmd?.status != null && ctx.zeek.smb_cmd.status.toLowerCase() != 'success'" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/smb_cmd/manifest.yml b/x-pack/filebeat/module/zeek/smb_cmd/manifest.yml index 089269869e8..a4ad3a78ce1 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/manifest.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/smb_cmd.yml diff --git a/x-pack/filebeat/module/zeek/smb_cmd/test/smb_cmd-json.log-expected.json b/x-pack/filebeat/module/zeek/smb_cmd/test/smb_cmd-json.log-expected.json index 872ce4a8238..e18caef3fd2 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/test/smb_cmd-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/smb_cmd/test/smb_cmd-json.log-expected.json @@ -4,15 +4,29 @@ "destination.address": "172.16.128.202", "destination.ip": "172.16.128.202", "destination.port": 445, + "event.action": "NT_CREATE_ANDX", + "event.category": [ + "network" + ], "event.dataset": "zeek.smb_cmd", "event.id": "CbT8mpAXseu6Pt4R7", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "success", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "smb_cmd", "input.type": "log", "log.offset": 0, "network.community_id": "1:SJNAD5vtzZuhQjGtfaI8svTnyuw=", "network.protocol": "smb", "network.transport": "tcp", + "related.ip": [ + "172.16.133.6", + "172.16.128.202" + ], "service.type": "zeek", "source.address": "172.16.133.6", "source.ip": "172.16.133.6", diff --git a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml index ed5d4cdecbb..8ab5ee36395 100644 --- a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml +++ b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml @@ -36,10 +36,26 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.smb_files.action", to: "event.action"} + - {from: "zeek.smb_files.name", to: "file.name"} + - {from: "zeek.smb_files.size", to: "file.size"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + - file + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.json deleted file mode 100644 index b4cfcfaa5b1..00000000000 --- a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek smb_files.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.smb_files.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.smb_files.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "dot_expander": { - "field": "times.accessed", - "path": "zeek.smb_files" - } - }, - { - "dot_expander": { - "field": "times.changed", - "path": "zeek.smb_files" - } - }, - { - "dot_expander": { - "field": "times.created", - "path": "zeek.smb_files" - } - }, - { - "dot_expander": { - "field": "times.modified", - "path": "zeek.smb_files" - } - }, - { - "date": { - "field": "zeek.smb_files.times.accessed", - "target_field": "zeek.smb_files.times.accessed", - "formats": ["UNIX"], - "if": "ctx.zeek.smb_files.times?.accessed != null" - } - }, - { - "date": { - "field": "zeek.smb_files.times.changed", - "target_field": "zeek.smb_files.times.changed", - "formats": ["UNIX"], - "if": "ctx.zeek.smb_files.times?.accessed != null" - } - }, - { - "date": { - "field": "zeek.smb_files.times.created", - "target_field": "zeek.smb_files.times.created", - "formats": ["UNIX"], - "if": "ctx.zeek.smb_files.times?.accessed != null" - } - }, - { - "date": { - "field": "zeek.smb_files.times.modified", - "target_field": "zeek.smb_files.times.modified", - "formats": ["UNIX"], - "if": "ctx.zeek.smb_files.times?.accessed != null" - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml new file mode 100644 index 00000000000..b2c7f52a29b --- /dev/null +++ b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml @@ -0,0 +1,135 @@ +description: Pipeline for normalizing Zeek smb_files.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.smb_files.ts + formats: + - UNIX +- remove: + field: zeek.smb_files.ts +- dot_expander: + field: times.accessed + path: zeek.smb_files +- dot_expander: + field: times.changed + path: zeek.smb_files +- dot_expander: + field: times.created + path: zeek.smb_files +- dot_expander: + field: times.modified + path: zeek.smb_files +- date: + field: zeek.smb_files.times.accessed + target_field: zeek.smb_files.times.accessed + formats: + - UNIX + if: ctx.zeek.smb_files.times?.accessed != null +- set: + field: file.accessed + value: "{{zeek.smb_files.times.accessed}}" + if: "ctx?.zeek?.smb_files?.times?.accessed != null" +- date: + field: zeek.smb_files.times.changed + target_field: zeek.smb_files.times.changed + formats: + - UNIX + if: ctx.zeek.smb_files.times?.accessed != null +- set: + field: file.ctime + value: "{{zeek.smb_files.times.changed}}" + if: "ctx?.zeek?.smb_files?.times?.changed != null" +- date: + field: zeek.smb_files.times.created + target_field: zeek.smb_files.times.created + formats: + - UNIX + if: ctx.zeek.smb_files.times?.accessed != null +- set: + field: file.created + value: "{{zeek.smb_files.times.created}}" + if: "ctx?.zeek?.smb_files?.times?.created != null" +- date: + field: zeek.smb_files.times.modified + target_field: zeek.smb_files.times.modified + formats: + - UNIX + if: ctx.zeek.smb_files.times?.accessed != null +- set: + field: file.mtime + value: "{{zeek.smb_files.times.modified}}" + if: "ctx?.zeek?.smb_files?.times?.modified != null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- set: + field: file.path + value: "{{zeek.smb_files.path}}\\{{zeek.smb_files.name}}" + if: "ctx?.zeek?.smb_files?.path != null && ctx?.zeek?.smb_files?.name != null" +- append: + field: event.type + value: deletion + if: "ctx?.zeek?.smb_files?.action == 'SMB::FILE_DELETE'" +- append: + field: event.type + value: change + if: "ctx?.zeek?.smb_files?.action == 'SMB::FILE_RENAME' || ctx?.zeek?.smb_files?.action == 'SMB::FILE_SET_ATTRIBUTE'" +- append: + field: event.type + value: info + if: "ctx?.zeek?.smb_files?.action != null && ctx.zeek.smb_files != 'SMB::FILE_DELETE' && ctx.zeek.smb_files != 'SMB::FILE_RENAME' && ctx.zeek.smb_files != 'SMB::FILE_SET_ATTRIBUTE'" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/smb_files/manifest.yml b/x-pack/filebeat/module/zeek/smb_files/manifest.yml index 154b445e765..f59a04153a5 100644 --- a/x-pack/filebeat/module/zeek/smb_files/manifest.yml +++ b/x-pack/filebeat/module/zeek/smb_files/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/smb_files.yml diff --git a/x-pack/filebeat/module/zeek/smb_files/test/smb_files-json.log-expected.json b/x-pack/filebeat/module/zeek/smb_files/test/smb_files-json.log-expected.json index fc7b8496d08..c7d5ab98b78 100644 --- a/x-pack/filebeat/module/zeek/smb_files/test/smb_files-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/smb_files/test/smb_files-json.log-expected.json @@ -4,15 +4,37 @@ "destination.address": "192.168.10.30", "destination.ip": "192.168.10.30", "destination.port": 445, + "event.action": "SMB::FILE_OPEN", + "event.category": [ + "network", + "file" + ], "event.dataset": "zeek.smb_files", "event.id": "C9YAaEzWLL62yWMn5", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol", + "info" + ], + "file.accessed": "2017-10-09T16:13:19.607Z", + "file.created": "2017-10-09T16:13:19.607Z", + "file.ctime": "2017-10-09T16:13:19.607Z", + "file.mtime": "2017-10-09T16:13:19.607Z", + "file.name": "PSEXESVC.exe", + "file.path": "\\\\\\\\admin-pc\\\\ADMIN$\\PSEXESVC.exe", + "file.size": 0, "fileset.name": "smb_files", "input.type": "log", "log.offset": 0, "network.community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", "network.protocol": "smb", "network.transport": "tcp", + "related.ip": [ + "192.168.10.31", + "192.168.10.30" + ], "service.type": "zeek", "source.address": "192.168.10.31", "source.ip": "192.168.10.31", diff --git a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml index 72ea3647344..0d0934c62c8 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml @@ -36,10 +36,22 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.json b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.json deleted file mode 100644 index c15ad371ed3..00000000000 --- a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek smb_mapping.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.smb_mapping.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.smb_mapping.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure": [{ - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml new file mode 100644 index 00000000000..b5752120267 --- /dev/null +++ b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek smb_mapping.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.smb_mapping.ts + formats: + - UNIX +- remove: + field: zeek.smb_mapping.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/smb_mapping/manifest.yml b/x-pack/filebeat/module/zeek/smb_mapping/manifest.yml index 403d2951c0c..7382e529b27 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/manifest.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/smb_mapping.yml diff --git a/x-pack/filebeat/module/zeek/smb_mapping/test/smb_mapping-json.log-expected.json b/x-pack/filebeat/module/zeek/smb_mapping/test/smb_mapping-json.log-expected.json index fbd3dd29693..71efd1e51ac 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/test/smb_mapping-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/smb_mapping/test/smb_mapping-json.log-expected.json @@ -4,15 +4,27 @@ "destination.address": "192.168.10.30", "destination.ip": "192.168.10.30", "destination.port": 445, + "event.category": [ + "network" + ], "event.dataset": "zeek.smb_mapping", "event.id": "C9YAaEzWLL62yWMn5", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "smb_mapping", "input.type": "log", "log.offset": 0, "network.community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", "network.protocol": "smb", "network.transport": "tcp", + "related.ip": [ + "192.168.10.31", + "192.168.10.30" + ], "service.type": "zeek", "source.address": "192.168.10.31", "source.ip": "192.168.10.31", diff --git a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml index af4855948ea..fc8c3b0074f 100644 --- a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml +++ b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml @@ -45,10 +45,23 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.smtp.tls", to: "tls.established", type: boolean} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.json deleted file mode 100644 index 44bc0b189aa..00000000000 --- a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek smtp.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.smtp.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.smtp.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "convert": { - "field": "zeek.smtp.tls", - "target_field": "tls.established", - "type": "boolean", - "ignore_missing": true - } - }, - { - "date": { - "field": "zeek.smtp.date", - "target_field": "zeek.smtp.date", - "formats": ["EEE, d MMM yyyy HH:mm:ss Z"], - "if": "ctx.zeek.smtp.date != null" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml new file mode 100644 index 00000000000..4424d3674ff --- /dev/null +++ b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml @@ -0,0 +1,69 @@ +description: Pipeline for normalizing Zeek smtp.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.smtp.ts + formats: + - UNIX +- remove: + field: zeek.smtp.ts +- date: + field: zeek.smtp.date + target_field: zeek.smtp.date + formats: + - EEE, d MMM yyyy HH:mm:ss Z + if: ctx.zeek.smtp.date != null +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/smtp/manifest.yml b/x-pack/filebeat/module/zeek/smtp/manifest.yml index 489c984b1c4..6d69b3b5e3e 100644 --- a/x-pack/filebeat/module/zeek/smtp/manifest.yml +++ b/x-pack/filebeat/module/zeek/smtp/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/smtp.yml diff --git a/x-pack/filebeat/module/zeek/smtp/test/smtp-json.log-expected.json b/x-pack/filebeat/module/zeek/smtp/test/smtp-json.log-expected.json index 3d4bd56ac4a..61e1be27bf6 100644 --- a/x-pack/filebeat/module/zeek/smtp/test/smtp-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/smtp/test/smtp-json.log-expected.json @@ -4,15 +4,27 @@ "destination.address": "192.168.1.9", "destination.ip": "192.168.1.9", "destination.port": 25, + "event.category": [ + "network" + ], "event.dataset": "zeek.smtp", "event.id": "CWWzPB3RjqhFf528c", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "smtp", "input.type": "log", "log.offset": 0, "network.community_id": "1:38H0puTqOoHT/5r2bKFUVSXifQw=", "network.protocol": "smtp", "network.transport": "tcp", + "related.ip": [ + "192.168.1.10", + "192.168.1.9" + ], "service.type": "zeek", "source.address": "192.168.1.10", "source.ip": "192.168.1.10", diff --git a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml index 76ff0c05f93..3431a990e0f 100644 --- a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml +++ b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml @@ -48,10 +48,22 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.json b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.json deleted file mode 100644 index 646b7edf845..00000000000 --- a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek snmp.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.snmp.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.snmp.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "date": { - "field": "zeek.snmp.up_since", - "target_field": "zeek.snmp.up_since", - "formats": ["UNIX"], - "if": "ctx.zeek.snmp.up_since != null" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml new file mode 100644 index 00000000000..f0070ef790d --- /dev/null +++ b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml @@ -0,0 +1,69 @@ +description: Pipeline for normalizing Zeek snmp.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.snmp.ts + formats: + - UNIX +- remove: + field: zeek.snmp.ts +- date: + field: zeek.snmp.up_since + target_field: zeek.snmp.up_since + formats: + - UNIX + if: ctx.zeek.snmp.up_since != null +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/snmp/manifest.yml b/x-pack/filebeat/module/zeek/snmp/manifest.yml index c11cd0b3491..b980b6fb82e 100644 --- a/x-pack/filebeat/module/zeek/snmp/manifest.yml +++ b/x-pack/filebeat/module/zeek/snmp/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/snmp.yml diff --git a/x-pack/filebeat/module/zeek/snmp/test/snmp-json.log-expected.json b/x-pack/filebeat/module/zeek/snmp/test/snmp-json.log-expected.json index 44cd6c16319..65345db7957 100644 --- a/x-pack/filebeat/module/zeek/snmp/test/snmp-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/snmp/test/snmp-json.log-expected.json @@ -4,15 +4,27 @@ "destination.address": "192.168.1.1", "destination.ip": "192.168.1.1", "destination.port": 161, + "event.category": [ + "network" + ], "event.dataset": "zeek.snmp", "event.id": "CnKW1B4w9fpRa6Nkf2", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "snmp", "input.type": "log", "log.offset": 0, "network.community_id": "1:X15ey/8/tEH+tlelK6P+GfgwBPc=", "network.protocol": "snmp", "network.transport": "udp", + "related.ip": [ + "192.168.1.2", + "192.168.1.1" + ], "service.type": "zeek", "source.address": "192.168.1.2", "source.ip": "192.168.1.2", diff --git a/x-pack/filebeat/module/zeek/socks/config/socks.yml b/x-pack/filebeat/module/zeek/socks/config/socks.yml index 5bf93e22f91..ddbcd51d0b0 100644 --- a/x-pack/filebeat/module/zeek/socks/config/socks.yml +++ b/x-pack/filebeat/module/zeek/socks/config/socks.yml @@ -45,10 +45,23 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.socks.user", to: "user.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.json b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.json deleted file mode 100644 index eabb2837d82..00000000000 --- a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek socks.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.socks.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.socks.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "dot_expander": { - "field": "bound.host", - "path": "zeek.socks" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml new file mode 100644 index 00000000000..04a84b13177 --- /dev/null +++ b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml @@ -0,0 +1,82 @@ +description: Pipeline for normalizing Zeek socks.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.socks.ts + formats: + - UNIX +- remove: + field: zeek.socks.ts +- dot_expander: + field: bound.host + path: zeek.socks +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- append: + field: event.type + value: error + if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'" +- append: + field: event.outcome + value: success + if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status == 'succeeded'" +- append: + field: event.outcome + value: failure + if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/socks/manifest.yml b/x-pack/filebeat/module/zeek/socks/manifest.yml index c24b9aae6db..68fea837fde 100644 --- a/x-pack/filebeat/module/zeek/socks/manifest.yml +++ b/x-pack/filebeat/module/zeek/socks/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/socks.yml diff --git a/x-pack/filebeat/module/zeek/socks/test/socks-json.log-expected.json b/x-pack/filebeat/module/zeek/socks/test/socks-json.log-expected.json index cf2a629e475..c8172d23d1a 100644 --- a/x-pack/filebeat/module/zeek/socks/test/socks-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/socks/test/socks-json.log-expected.json @@ -4,15 +4,30 @@ "destination.address": "127.0.0.1", "destination.ip": "127.0.0.1", "destination.port": 8080, + "event.category": [ + "network" + ], "event.dataset": "zeek.socks", "event.id": "Cmz4Cb4qCw1hGqYw1c", + "event.kind": "event", "event.module": "zeek", + "event.outcome": [ + "success" + ], + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "socks", "input.type": "log", "log.offset": 0, "network.community_id": "1:1Hp/o0hOC62lAwrV+a0ZKDE3rrs=", "network.protocol": "socks", "network.transport": "tcp", + "related.ip": [ + "127.0.0.1", + "127.0.0.1" + ], "service.type": "zeek", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", diff --git a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml index f463b62e895..e33f4e0e29e 100644 --- a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml +++ b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml @@ -57,10 +57,20 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.json deleted file mode 100644 index 2eefd208860..00000000000 --- a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek ssh.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.ssh.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.ssh.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml new file mode 100644 index 00000000000..019a44b89e0 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml @@ -0,0 +1,71 @@ +description: Pipeline for normalizing Zeek ssh.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.ssh.ts + formats: + - UNIX +- remove: + field: zeek.ssh.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- set: + field: event.outcome + value: failure + if: "ctx?.zeek?.ssh?.auth?.success != null && ctx.zeek.ssh.auth.success == false" +- set: + field: event.outcome + value: success + if: "ctx?.zeek?.ssh?.auth?.success != null && ctx.zeek.ssh.auth.success == true" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/ssh/manifest.yml b/x-pack/filebeat/module/zeek/ssh/manifest.yml index da635a43771..60249e25c21 100644 --- a/x-pack/filebeat/module/zeek/ssh/manifest.yml +++ b/x-pack/filebeat/module/zeek/ssh/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/ssh.yml diff --git a/x-pack/filebeat/module/zeek/ssh/test/ssh-json.log-expected.json b/x-pack/filebeat/module/zeek/ssh/test/ssh-json.log-expected.json index 8ab4788abc7..343aa7392e5 100644 --- a/x-pack/filebeat/module/zeek/ssh/test/ssh-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ssh/test/ssh-json.log-expected.json @@ -4,15 +4,28 @@ "destination.address": "192.168.1.1", "destination.ip": "192.168.1.1", "destination.port": 22, + "event.category": [ + "network" + ], "event.dataset": "zeek.ssh", "event.id": "CajWfz1b3qnnWT0BU9", + "event.kind": "event", "event.module": "zeek", + "event.outcome": "failure", + "event.type": [ + "connection", + "protocol" + ], "fileset.name": "ssh", "input.type": "log", "log.offset": 0, "network.community_id": "1:42tg9bemt74qgrdvJOy2n5Veg4A=", "network.protocol": "ssh", "network.transport": "tcp", + "related.ip": [ + "192.168.1.2", + "192.168.1.1" + ], "service.type": "zeek", "source.address": "192.168.1.2", "source.ip": "192.168.1.2", diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml index 878267f549a..88bfcc4b53e 100644 --- a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml +++ b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml @@ -56,10 +56,24 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "source.address", to: "client.address"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "destination.address", to: "server.address"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + kind: + - connection + - protocol {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.address {{ end }} diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml index 2a5ebf4ce7a..bbeaa24d1bd 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml @@ -10,22 +10,14 @@ processors: - UNIX - remove: field: zeek.ssl.ts -- set: - field: event.id - value: '{{zeek.session_id}}' - if: ctx.zeek.session_id != null -- set: - field: source.ip - value: '{{source.address}}' -- set: - field: destination.ip - value: '{{destination.address}}' - geoip: field: destination.ip target_field: destination.geo + ignore_missing: true - geoip: field: source.ip target_field: source.geo + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb field: source.ip @@ -248,7 +240,14 @@ processors: ctx.tls.version = parts[1].substring(0,1) + "." + parts[1].substring(1); } ctx.tls.version_protocol = parts[0].toLowerCase(); - +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json index d7d7ac33ff9..526a43a350b 100644 --- a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json @@ -1,6 +1,7 @@ [ { "@timestamp": "2019-01-17T01:32:16.805Z", + "client.address": "10.178.98.102", "destination.address": "35.199.178.4", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", @@ -13,14 +14,26 @@ "destination.geo.region_name": "California", "destination.ip": "35.199.178.4", "destination.port": 9243, + "event.category": [ + "network" + ], "event.dataset": "zeek.ssl", "event.id": "CAOvs1BMFCX2Eh0Y3", + "event.kind": [ + "connection", + "protocol" + ], "event.module": "zeek", "fileset.name": "ssl", "input.type": "log", "log.offset": 0, "network.community_id": "1:1PMhYqOKBIyRAQeMbg/pWiJ198g=", "network.transport": "tcp", + "related.ip": [ + "10.178.98.102", + "35.199.178.4" + ], + "server.address": "35.199.178.4", "service.type": "zeek", "source.address": "10.178.98.102", "source.ip": "10.178.98.102", @@ -59,6 +72,7 @@ }, { "@timestamp": "2019-01-17T01:32:16.805Z", + "client.address": "10.178.98.102", "destination.address": "35.199.178.4", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", @@ -71,14 +85,26 @@ "destination.geo.region_name": "California", "destination.ip": "35.199.178.4", "destination.port": 9243, + "event.category": [ + "network" + ], "event.dataset": "zeek.ssl", "event.id": "C3mki91FnnNtm0u1ok", + "event.kind": [ + "connection", + "protocol" + ], "event.module": "zeek", "fileset.name": "ssl", "input.type": "log", "log.offset": 635, "network.community_id": "1:zYbLmqRN6PLPB067HNAiAQISqvI=", "network.transport": "tcp", + "related.ip": [ + "10.178.98.102", + "35.199.178.4" + ], + "server.address": "35.199.178.4", "service.type": "zeek", "source.address": "10.178.98.102", "source.ip": "10.178.98.102", diff --git a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.json b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.json deleted file mode 100644 index 6115bc6c1d2..00000000000 --- a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek stats.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.stats.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.stats.ts" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml new file mode 100644 index 00000000000..c0347161190 --- /dev/null +++ b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml @@ -0,0 +1,18 @@ +description: Pipeline for normalizing Zeek stats.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.stats.ts + formats: + - UNIX +- remove: + field: zeek.stats.ts +- set: + field: event.kind + value: metric +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/stats/manifest.yml b/x-pack/filebeat/module/zeek/stats/manifest.yml index c4b122a19bf..f63ad40bf33 100644 --- a/x-pack/filebeat/module/zeek/stats/manifest.yml +++ b/x-pack/filebeat/module/zeek/stats/manifest.yml @@ -11,5 +11,5 @@ var: - name: tags default: [zeek.stats] -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/stats.yml diff --git a/x-pack/filebeat/module/zeek/stats/test/stats-json.log-expected.json b/x-pack/filebeat/module/zeek/stats/test/stats-json.log-expected.json index a2d8e3ab311..bcb5f24f2a2 100644 --- a/x-pack/filebeat/module/zeek/stats/test/stats-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/stats/test/stats-json.log-expected.json @@ -2,6 +2,7 @@ { "@timestamp": "2016-10-16T08:17:58.714Z", "event.dataset": "zeek.stats", + "event.kind": "metric", "event.module": "zeek", "fileset.name": "stats", "input.type": "log", diff --git a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml index b7accce096d..a8420237af0 100644 --- a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml +++ b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml @@ -41,10 +41,17 @@ processors: ignore_missing: true fail_on_error: false - + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.syslog.facility", to: "log.syslog.facility.name"} + - {from: "zeek.syslog.severity", to: "log.syslog.severity.name"} + - add_fields: + target: event + fields: + kind: event {{ if .community_id }} - community_id: - fields: - source_ip: source.address - destination_ip: destination.addresss {{ end }} diff --git a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.json b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.json deleted file mode 100644 index fcb98b1b91d..00000000000 --- a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek syslog.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.syslog.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.syslog.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml new file mode 100644 index 00000000000..7fd848682b1 --- /dev/null +++ b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek syslog.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.syslog.ts + formats: + - UNIX +- remove: + field: zeek.syslog.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/syslog/manifest.yml b/x-pack/filebeat/module/zeek/syslog/manifest.yml index 2d75d440d2f..8db76ab5b36 100644 --- a/x-pack/filebeat/module/zeek/syslog/manifest.yml +++ b/x-pack/filebeat/module/zeek/syslog/manifest.yml @@ -13,5 +13,5 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/syslog.yml diff --git a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml index 1cd1a7031fd..8b4b40e0234 100644 --- a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml +++ b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml @@ -27,3 +27,17 @@ processors: ignore_missing: true fail_on_error: false + - convert: + fields: + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - info diff --git a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.json b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.json deleted file mode 100644 index 9a755fa3913..00000000000 --- a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek traceroute.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.traceroute.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.traceroute.ts" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml new file mode 100644 index 00000000000..6fa5a0bc993 --- /dev/null +++ b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek traceroute.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.traceroute.ts + formats: + - UNIX +- remove: + field: zeek.traceroute.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/traceroute/manifest.yml b/x-pack/filebeat/module/zeek/traceroute/manifest.yml index c0dd44654df..0761e9b3bf4 100644 --- a/x-pack/filebeat/module/zeek/traceroute/manifest.yml +++ b/x-pack/filebeat/module/zeek/traceroute/manifest.yml @@ -11,5 +11,5 @@ var: - name: tags default: [zeek.traceroute] -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/traceroute.yml diff --git a/x-pack/filebeat/module/zeek/traceroute/test/traceroute-json.log-expected.json b/x-pack/filebeat/module/zeek/traceroute/test/traceroute-json.log-expected.json index 90bd0dd4eec..8fdfd983c94 100644 --- a/x-pack/filebeat/module/zeek/traceroute/test/traceroute-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/traceroute/test/traceroute-json.log-expected.json @@ -2,13 +2,30 @@ { "@timestamp": "2013-02-26T22:02:38.650Z", "destination.address": "8.8.8.8", + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", + "event.category": [ + "network" + ], "event.dataset": "zeek.traceroute", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "info" + ], "fileset.name": "traceroute", "input.type": "log", "log.offset": 0, "network.transport": "udp", + "related.ip": [ + "192.168.1.1", + "8.8.8.8" + ], "service.type": "zeek", "source.address": "192.168.1.1", "source.ip": "192.168.1.1", diff --git a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml index 3fdd2c1faaa..ed9af2117ad 100644 --- a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml +++ b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml @@ -36,3 +36,19 @@ processors: ignore_missing: true fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.tunnel.action", to: "event.action"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: event + category: + - network + type: + - connection diff --git a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.json b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.json deleted file mode 100644 index bc9eacce8b0..00000000000 --- a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek tunnel.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.tunnel.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.tunnel.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml new file mode 100644 index 00000000000..402bce5fa5d --- /dev/null +++ b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek tunnel.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.tunnel.ts + formats: + - UNIX +- remove: + field: zeek.tunnel.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/tunnel/manifest.yml b/x-pack/filebeat/module/zeek/tunnel/manifest.yml index ad3d712c33c..a0618a12b7e 100644 --- a/x-pack/filebeat/module/zeek/tunnel/manifest.yml +++ b/x-pack/filebeat/module/zeek/tunnel/manifest.yml @@ -11,5 +11,5 @@ var: - name: tags default: [zeek.tunnel] -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/tunnel.yml diff --git a/x-pack/filebeat/module/zeek/tunnel/test/tunnel-json.log-expected.json b/x-pack/filebeat/module/zeek/tunnel/test/tunnel-json.log-expected.json index 9504931de51..1e00e616e36 100644 --- a/x-pack/filebeat/module/zeek/tunnel/test/tunnel-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/tunnel/test/tunnel-json.log-expected.json @@ -2,15 +2,39 @@ { "@timestamp": "2018-12-10T01:34:26.743Z", "destination.address": "132.16.110.133", + "destination.as.number": 427, + "destination.as.organization.name": "Air Force Systems Networking", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, "destination.ip": "132.16.110.133", "destination.port": 8080, + "event.action": "Tunnel::DISCOVER", + "event.category": [ + "network" + ], "event.dataset": "zeek.tunnel", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "connection" + ], "fileset.name": "tunnel", "input.type": "log", "log.offset": 0, + "related.ip": [ + "132.16.146.79", + "132.16.110.133" + ], "service.type": "zeek", "source.address": "132.16.146.79", + "source.as.number": 427, + "source.as.organization.name": "Air Force Systems Networking", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, "source.ip": "132.16.146.79", "source.port": 0, "tags": [ diff --git a/x-pack/filebeat/module/zeek/weird/config/weird.yml b/x-pack/filebeat/module/zeek/weird/config/weird.yml index 6f67c90ae4f..1256f96902b 100644 --- a/x-pack/filebeat/module/zeek/weird/config/weird.yml +++ b/x-pack/filebeat/module/zeek/weird/config/weird.yml @@ -36,3 +36,19 @@ processors: ignore_missing: true fail_on_error: false + - convert: + fields: + - {from: "zeek.session_id", to: "event.id"} + - {from: "source.address", to: "source.ip", type: "ip"} + - {from: "destination.address", to: "destination.ip", type: "ip"} + - {from: "zeek.weird.name", to: "rule.name"} + ignore_missing: true + fail_on_error: false + - add_fields: + target: event + fields: + kind: alert + category: + - network + type: + - info diff --git a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.json b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.json deleted file mode 100644 index a97cdeb22bb..00000000000 --- a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek weird.log", - "processors": [ - { - "set": { - "field": "event.created", - "value": "{{_ingest.timestamp}}" - } - }, - { - "date": { - "field": "zeek.weird.ts", - "formats": ["UNIX"] - } - }, - { - "remove": { - "field": "zeek.weird.ts" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}", - "if": "ctx?.source?.address != null" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}", - "if": "ctx?.destination?.address != null" - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml new file mode 100644 index 00000000000..e0325d9a1c5 --- /dev/null +++ b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml @@ -0,0 +1,63 @@ +description: Pipeline for normalizing Zeek weird.log +processors: +- set: + field: event.created + value: '{{_ingest.timestamp}}' +- date: + field: zeek.weird.ts + formats: + - UNIX +- remove: + field: zeek.weird.ts +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zeek/weird/manifest.yml b/x-pack/filebeat/module/zeek/weird/manifest.yml index 63d48d32ee3..3e91c91c64a 100644 --- a/x-pack/filebeat/module/zeek/weird/manifest.yml +++ b/x-pack/filebeat/module/zeek/weird/manifest.yml @@ -11,5 +11,5 @@ var: - name: tags default: [zeek.weird] -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/weird.yml diff --git a/x-pack/filebeat/module/zeek/weird/test/weird-json.log-expected.json b/x-pack/filebeat/module/zeek/weird/test/weird-json.log-expected.json index f1fdb20678f..cc9f7f49508 100644 --- a/x-pack/filebeat/module/zeek/weird/test/weird-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/weird/test/weird-json.log-expected.json @@ -4,12 +4,24 @@ "destination.address": "192.168.1.2", "destination.ip": "192.168.1.2", "destination.port": 53, + "event.category": [ + "network" + ], "event.dataset": "zeek.weird", "event.id": "C1ralPp062bkwWt4e", + "event.kind": "alert", "event.module": "zeek", + "event.type": [ + "info" + ], "fileset.name": "weird", "input.type": "log", "log.offset": 0, + "related.ip": [ + "192.168.1.1", + "192.168.1.2" + ], + "rule.name": "dns_unmatched_reply", "service.type": "zeek", "source.address": "192.168.1.1", "source.ip": "192.168.1.1", @@ -24,11 +36,19 @@ }, { "@timestamp": "2020-01-28T16:00:59.342Z", + "event.category": [ + "network" + ], "event.dataset": "zeek.weird", + "event.kind": "alert", "event.module": "zeek", + "event.type": [ + "info" + ], "fileset.name": "weird", "input.type": "log", "log.offset": 197, + "rule.name": "non_ip_packet_in_ethernet", "service.type": "zeek", "tags": [ "zeek.weird" diff --git a/x-pack/filebeat/module/zeek/x509/config/x509.yml b/x-pack/filebeat/module/zeek/x509/config/x509.yml index 3bebeab5697..49a670e46e5 100644 --- a/x-pack/filebeat/module/zeek/x509/config/x509.yml +++ b/x-pack/filebeat/module/zeek/x509/config/x509.yml @@ -57,3 +57,9 @@ processors: ignore_missing: true fail_on_error: false + - add_fields: + target: event + fields: + kind: event + type: + - info diff --git a/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json b/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json index 1cff57241ba..fff83c5969e 100644 --- a/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json @@ -3,7 +3,11 @@ "@timestamp": "2018-12-03T20:00:00.143Z", "event.dataset": "zeek.x509", "event.id": "FxZ6gZ3YR6vFlIocq3", + "event.kind": "event", "event.module": "zeek", + "event.type": [ + "info" + ], "fileset.name": "x509", "input.type": "log", "log.offset": 0,