From 062e16c4a80ff603fd0f1f059c121f3b833fef50 Mon Sep 17 00:00:00 2001 From: alakahakai <lei.qiu@elastic.co> Date: Wed, 27 Feb 2019 19:28:02 -0800 Subject: [PATCH] Backport PR #10916 to 7.0 (#10972) * Update Zeek dashboard and README.md. Add support for notice.log. Update field descriptions --- CHANGELOG.next.asciidoc | 2 + filebeat/docs/fields.asciidoc | 564 ++++++++++++++- x-pack/filebeat/filebeat.reference.yml | 2 + .../filebeat/module/zeek/README-developer.md | 6 +- x-pack/filebeat/module/zeek/README.md | 6 +- x-pack/filebeat/module/zeek/_meta/config.yml | 2 + x-pack/filebeat/module/zeek/_meta/fields.yml | 354 +++++++++- .../7/dashboard/Filebeat-Zeek-Overview.json | 650 +++++++++++------- .../zeek/connection/config/connection.yml | 2 +- .../zeek/connection/ingest/pipeline.json | 35 +- .../module/zeek/connection/manifest.yml | 2 +- .../zeek/connection/test/connection-json.log | 2 + .../test/connection-json.log-expected.json | 87 ++- .../module/zeek/dns/ingest/pipeline.json | 13 +- x-pack/filebeat/module/zeek/dns/manifest.yml | 2 +- .../zeek/dns/test/dns-json.log-expected.json | 4 +- x-pack/filebeat/module/zeek/fields.go | 2 +- .../module/zeek/files/ingest/pipeline.json | 14 +- .../filebeat/module/zeek/files/manifest.yml | 2 +- .../files/test/files-json.log-expected.json | 8 +- .../module/zeek/http/ingest/pipeline.json | 13 +- x-pack/filebeat/module/zeek/http/manifest.yml | 2 +- .../http/test/http-json.log-expected.json | 4 +- x-pack/filebeat/module/zeek/module.yml | 2 +- .../module/zeek/notice/config/notice.yml | 80 +++ .../module/zeek/notice/ingest/pipeline.json | 46 ++ .../filebeat/module/zeek/notice/manifest.yml | 19 + .../module/zeek/notice/test/notice-json.log | 1 + .../notice/test/notice-json.log-expected.json | 23 + .../module/zeek/ssl/ingest/pipeline.json | 13 +- x-pack/filebeat/module/zeek/ssl/manifest.yml | 2 +- .../zeek/ssl/test/ssl-json.log-expected.json | 8 +- x-pack/filebeat/modules.d/zeek.yml.disabled | 2 + 33 files changed, 1637 insertions(+), 337 deletions(-) create mode 100644 x-pack/filebeat/module/zeek/notice/config/notice.yml create mode 100644 x-pack/filebeat/module/zeek/notice/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/zeek/notice/manifest.yml create mode 100644 x-pack/filebeat/module/zeek/notice/test/notice-json.log create mode 100644 x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c2f2e4400fa..e3728df4697 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -53,6 +53,8 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di *Filebeat* +- Fix errors in filebeat Zeek dashboard and README files. Add notice.log support. {pull}10916[10916] + *Heartbeat* *Journalbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 61f290e26f6..21eb12efe9f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -13447,6 +13447,9 @@ Fields from Zeek/Bro logs after normalization -- type: keyword +A unique identifier of the session + + -- *`zeek.connection.local_orig`*:: @@ -13454,6 +13457,9 @@ type: keyword -- type: boolean +Indicates whether the session is originated locally + + -- *`zeek.connection.local_resp`*:: @@ -13461,6 +13467,9 @@ type: boolean -- type: boolean +Indicates whether the session is responded locally + + -- *`zeek.connection.missed_bytes`*:: @@ -13468,6 +13477,9 @@ type: boolean -- type: long +Missed bytes for the session + + -- *`zeek.connection.state`*:: @@ -13475,6 +13487,9 @@ type: long -- type: keyword +Flags indicating the state of the session + + -- *`zeek.connection.history`*:: @@ -13482,6 +13497,9 @@ type: keyword -- type: keyword +Flags indicating the history of the session + + -- *`zeek.connection.orig_l2_addr`*:: @@ -13489,33 +13507,48 @@ type: keyword -- type: keyword +Link-layer address of the originator, if available + + -- -*`zeek.resp_l2_addr`*:: +*`zeek.connection.resp_l2_addr`*:: + -- type: keyword +Link-layer address of the responder, if available + + -- -*`zeek.vlan`*:: +*`zeek.connection.vlan`*:: + -- -type: keyword +type: integer + +VLAN identifier + -- -*`zeek.inner_vlan`*:: +*`zeek.connection.inner_vlan`*:: + -- -type: keyword +type: integer + +VLAN identifier + -- *`zeek.dns.trans_id`*:: + -- -type: integer +type: keyword + +DNS transaction identifier + -- @@ -13524,6 +13557,9 @@ type: integer -- type: double +Round trip time for the query and response + + -- *`zeek.dns.query`*:: @@ -13531,6 +13567,9 @@ type: double -- type: keyword +The domain name that is the subject of the DNS query + + -- *`zeek.dns.qclass`*:: @@ -13538,6 +13577,9 @@ type: keyword -- type: long +The QCLASS value specifying the class of the query + + -- *`zeek.dns.qclass_name`*:: @@ -13545,6 +13587,9 @@ type: long -- type: keyword +A descriptive name for the class of the query + + -- *`zeek.dns.qtype`*:: @@ -13552,6 +13597,9 @@ type: keyword -- type: long +A QTYPE value specifying the type of the query + + -- *`zeek.dns.qtype_name`*:: @@ -13559,6 +13607,9 @@ type: long -- type: keyword +A descriptive name for the type of the query + + -- *`zeek.dns.rcode`*:: @@ -13566,6 +13617,9 @@ type: keyword -- type: long +The response code value in DNS response messages + + -- *`zeek.dns.rcode_name`*:: @@ -13573,6 +13627,9 @@ type: long -- type: keyword +A descriptive name for the response code value + + -- *`zeek.dns.AA`*:: @@ -13580,6 +13637,10 @@ type: keyword -- type: boolean +The Authoritative Answer bit for response messages specifies that the responding +name server is an authority for the domain name in the question section + + -- *`zeek.dns.TC`*:: @@ -13587,6 +13648,9 @@ type: boolean -- type: boolean +The Truncation bit specifies that the message was truncated + + -- *`zeek.dns.RD`*:: @@ -13594,6 +13658,10 @@ type: boolean -- type: boolean +The Recursion Desired bit in a request message indicates that the client +wants recursive service for this query + + -- *`zeek.dns.RA`*:: @@ -13601,6 +13669,10 @@ type: boolean -- type: boolean +The Recursion Available bit in a response message indicates that the name +server supports recursive queries. + + -- *`zeek.dns.answers`*:: @@ -13608,6 +13680,9 @@ type: boolean -- type: keyword +The set of resource descriptions in the query answer + + -- *`zeek.dns.TTLs`*:: @@ -13615,6 +13690,9 @@ type: keyword -- type: double +The caching intervals of the associated RRs described by the answers field + + -- *`zeek.dns.rejected`*:: @@ -13622,6 +13700,9 @@ type: double -- type: boolean +Indicates whether the DNS query was rejected by the server + + -- *`zeek.dns.total_answers`*:: @@ -13629,6 +13710,9 @@ type: boolean -- type: integer +The total number of resource records in the reply + + -- *`zeek.dns.total_replies`*:: @@ -13636,6 +13720,9 @@ type: integer -- type: integer +The total number of resource records in the reply message + + -- *`zeek.dns.saw_query`*:: @@ -13643,6 +13730,9 @@ type: integer -- type: boolean +Whether the full DNS query has been seen + + -- *`zeek.dns.saw_reply`*:: @@ -13650,6 +13740,9 @@ type: boolean -- type: boolean +Whether the full DNS reply has been seen + + -- *`zeek.http.trans_depth`*:: @@ -13657,6 +13750,9 @@ type: boolean -- type: integer +Represents the pipelined depth into the connection of this request/response transaction + + -- *`zeek.http.status_msg`*:: @@ -13664,6 +13760,9 @@ type: integer -- type: keyword +Status message returned by the server + + -- *`zeek.http.info_code`*:: @@ -13671,6 +13770,9 @@ type: keyword -- type: integer +Last seen 1xx informational reply code returned by the server. + + -- *`zeek.http.info_msg`*:: @@ -13678,20 +13780,30 @@ type: integer -- type: keyword +Last seen 1xx informational reply message returned by the server. + + -- -*`zeek.http.filename`*:: +*`zeek.http.tags`*:: + -- type: keyword +A set of indicators of various attributes discovered and related to a particular +request/response pair. + + -- -*`zeek.http.tags`*:: +*`zeek.http.password`*:: + -- type: keyword +Password if basic-auth is performed for the request + + -- *`zeek.http.captured_password`*:: @@ -13699,6 +13811,9 @@ type: keyword -- type: boolean +Determines if the password will be captured for this request + + -- *`zeek.http.proxied`*:: @@ -13706,6 +13821,9 @@ type: boolean -- type: keyword +All of the headers that may indicate if the HTTP request was proxied + + -- *`zeek.http.range_request`*:: @@ -13713,6 +13831,9 @@ type: keyword -- type: boolean +Indicates if this request can assume 206 partial content in response + + -- *`zeek.http.client_header_names`*:: @@ -13720,6 +13841,10 @@ type: boolean -- type: keyword +The vector of HTTP header names sent by the client. No header values +are included here, just the header names. + + -- *`zeek.http.server_header_names`*:: @@ -13727,6 +13852,10 @@ type: keyword -- type: keyword +The vector of HTTP header names sent by the server. No header values +are included here, just the header names + + -- *`zeek.http.orig_fuids`*:: @@ -13734,6 +13863,9 @@ type: keyword -- type: keyword +An ordered vector of file unique IDs from the originator + + -- *`zeek.http.orig_mime_types`*:: @@ -13741,6 +13873,9 @@ type: keyword -- type: keyword +An ordered vector of mime types from the originator + + -- *`zeek.http.orig_filenames`*:: @@ -13748,6 +13883,9 @@ type: keyword -- type: keyword +An ordered vector of filenames from the originator + + -- *`zeek.http.resp_fuids`*:: @@ -13755,6 +13893,9 @@ type: keyword -- type: keyword +An ordered vector of file unique IDs from the responder + + -- *`zeek.http.resp_mime_types`*:: @@ -13762,6 +13903,9 @@ type: keyword -- type: keyword +An ordered vector of mime types from the responder + + -- *`zeek.http.resp_filenames`*:: @@ -13769,6 +13913,9 @@ type: keyword -- type: keyword +An ordered vector of filenames from the responder + + -- *`zeek.http.orig_mime_depth`*:: @@ -13776,6 +13923,9 @@ type: keyword -- type: integer +Current number of MIME entities in the HTTP request message body + + -- *`zeek.http.resp_mime_depth`*:: @@ -13783,6 +13933,9 @@ type: integer -- type: integer +Current number of MIME entities in the HTTP response message body + + -- *`zeek.files.fuid`*:: @@ -13790,6 +13943,9 @@ type: integer -- type: keyword +A file unique identifier + + -- *`zeek.files.tx_host`*:: @@ -13797,6 +13953,9 @@ type: keyword -- type: ip +The host that transferred the file + + -- *`zeek.files.rx_host`*:: @@ -13804,6 +13963,9 @@ type: ip -- type: ip +The host that received the file + + -- *`zeek.files.session_ids`*:: @@ -13811,6 +13973,9 @@ type: ip -- type: keyword +The sessions that have this file + + -- *`zeek.files.source`*:: @@ -13818,6 +13983,11 @@ type: keyword -- type: keyword +An identification of the source of the file data. E.g. it may be a network protocol +over which it was transferred, or a local file path which was read, or some other +input source + + -- *`zeek.files.depth`*:: @@ -13825,6 +13995,11 @@ type: keyword -- type: long +A value to represent the depth of this file in relation to its source. In SMTP, it +is the depth of the MIME attachment on the message. In HTTP, it is the depth of the +request within the TCP connection + + -- *`zeek.files.analyzers`*:: @@ -13832,6 +14007,9 @@ type: long -- type: keyword +A set of analysis types done during the file analysis + + -- *`zeek.files.mime_type`*:: @@ -13839,6 +14017,9 @@ type: keyword -- type: keyword +Mime type of the file + + -- *`zeek.files.filename`*:: @@ -13846,6 +14027,9 @@ type: keyword -- type: keyword +Name of the file if available + + -- *`zeek.files.local_orig`*:: @@ -13853,6 +14037,10 @@ type: keyword -- type: boolean +If the source of this file is a network connection, this field indicates if the data +originated from the local network or not + + -- *`zeek.files.is_orig`*:: @@ -13860,6 +14048,10 @@ type: boolean -- type: boolean +If the source of this file is a network connection, this field indicates if the file is +being sent by the originator of the connection or the responder + + -- *`zeek.files.duration`*:: @@ -13867,6 +14059,9 @@ type: boolean -- type: double +The duration the file was analyzed for. Not the duration of the session. + + -- *`zeek.files.seen_bytes`*:: @@ -13874,6 +14069,9 @@ type: double -- type: long +Number of bytes provided to the file analysis engine for the file + + -- *`zeek.files.total_bytes`*:: @@ -13881,6 +14079,9 @@ type: long -- type: long +Total number of bytes that are supposed to comprise the full file + + -- *`zeek.files.missing_bytes`*:: @@ -13888,6 +14089,10 @@ type: long -- type: long +The number of bytes in the file stream that were completely missed during the process +of analysis + + -- *`zeek.files.overflow_bytes`*:: @@ -13895,6 +14100,10 @@ type: long -- type: long +The number of bytes in the file stream that were not delivered to stream file analyzers. +This could be overlapping bytes or bytes that couldn't be reassembled + + -- *`zeek.files.timedout`*:: @@ -13902,6 +14111,9 @@ type: long -- type: boolean +Whether the file analysis timed out at least once for the file + + -- *`zeek.files.parent_fuid`*:: @@ -13909,6 +14121,10 @@ type: boolean -- type: keyword +Identifier associated with a container file from which this one was extracted as part of +the file analysis + + -- *`zeek.files.md5`*:: @@ -13916,6 +14132,9 @@ type: keyword -- type: keyword +An MD5 digest of the file contents + + -- *`zeek.files.sha1`*:: @@ -13923,6 +14142,9 @@ type: keyword -- type: keyword +A SHA1 digest of the file contents + + -- *`zeek.files.sha256`*:: @@ -13930,6 +14152,9 @@ type: keyword -- type: keyword +A SHA256 digest of the file contents. + + -- *`zeek.files.extracted`*:: @@ -13937,6 +14162,9 @@ type: keyword -- type: keyword +Local filename of extracted file + + -- *`zeek.files.extracted_cutoff`*:: @@ -13944,6 +14172,9 @@ type: keyword -- type: boolean +Indicate whether the file being extracted was cut off hence not extracted completely + + -- *`zeek.files.extracted_size`*:: @@ -13951,6 +14182,9 @@ type: boolean -- type: long +The number of bytes extracted to disk + + -- *`zeek.files.entropy`*:: @@ -13958,6 +14192,9 @@ type: long -- type: double +The information density of the contents of the file + + -- *`zeek.ssl.version`*:: @@ -13965,6 +14202,9 @@ type: double -- type: keyword +SSL/TLS version that was logged + + -- *`zeek.ssl.cipher`*:: @@ -13972,6 +14212,9 @@ type: keyword -- type: keyword +SSL/TLS cipher suite that was logged + + -- *`zeek.ssl.curve`*:: @@ -13979,6 +14222,9 @@ type: keyword -- type: keyword +Elliptic curve that was logged when using ECDH/ECDHE + + -- *`zeek.ssl.server_name`*:: @@ -13986,6 +14232,10 @@ type: keyword -- type: keyword +Value of the Server Name Indicator SSL/TLS extension. It indicates the server name +that the client was requesting + + -- *`zeek.ssl.resumed`*:: @@ -13993,6 +14243,10 @@ type: keyword -- type: boolean +Flag to indicate if the session was resumed reusing the key material exchanged in an +earlier connection + + -- *`zeek.ssl.next_protocol`*:: @@ -14000,6 +14254,9 @@ type: boolean -- type: keyword +Next protocol the server chose using the application layer next protocol extension + + -- *`zeek.ssl.established`*:: @@ -14007,6 +14264,9 @@ type: keyword -- type: boolean +Flag to indicate if this ssl session has been established successfully + + -- *`zeek.ssl.cert_chain`*:: @@ -14014,6 +14274,9 @@ type: boolean -- type: keyword +Chain of certificates offered by the server to validate its complete signing chain + + -- *`zeek.ssl.cert_chain_fuids`*:: @@ -14021,6 +14284,9 @@ type: keyword -- type: keyword +An ordered vector of certificate file identifiers for the certificates offered by the server + + -- *`zeek.ssl.client_cert_chain`*:: @@ -14028,6 +14294,9 @@ type: keyword -- type: keyword +Chain of certificates offered by the client to validate its complete signing chain + + -- *`zeek.ssl.client_cert_chain_fuids`*:: @@ -14035,6 +14304,9 @@ type: keyword -- type: keyword +An ordered vector of certificate file identifiers for the certificates offered by the client + + -- *`zeek.ssl.issuer`*:: @@ -14042,6 +14314,9 @@ type: keyword -- type: keyword +Subject of the signer of the X.509 certificate offered by the server + + -- *`zeek.ssl.client_issuer`*:: @@ -14049,6 +14324,9 @@ type: keyword -- type: keyword +Subject of the X.509 certificate offered by the client + + -- *`zeek.ssl.validation_status`*:: @@ -14056,6 +14334,19 @@ type: keyword -- type: keyword +Result of certificate validation for this connection + + +-- + +*`zeek.ssl.validation_code`*:: ++ +-- +type: keyword + +Result of certificate validation for this connection, given as OpenSSL validation code + + -- *`zeek.ssl.subject`*:: @@ -14063,6 +14354,9 @@ type: keyword -- type: keyword +Subject of the X.509 certificate offered by the server + + -- *`zeek.ssl.client_subject`*:: @@ -14070,6 +14364,9 @@ type: keyword -- type: keyword +Subject of the X.509 certificate offered by the client + + -- *`zeek.ssl.last_alert`*:: @@ -14077,5 +14374,256 @@ type: keyword -- type: keyword +Last alert that was seen during the connection + + +-- + +*`zeek.notice.connection_id`*:: ++ +-- +type: keyword + +Identifier of the related connection session + + +-- + +*`zeek.notice.icmp_id`*:: ++ +-- +type: keyword + +Identifier of the related ICMP session + + +-- + +*`zeek.notice.file.id`*:: ++ +-- +type: keyword + +An identifier associated with a single file that is related to this notice + + +-- + +*`zeek.notice.file.parent_id`*:: ++ +-- +type: keyword + +Identifier associated with a container file from which this one was extracted + + +-- + +*`zeek.notice.file.source`*:: ++ +-- +type: keyword + +An identification of the source of the file data. E.g. it may be a network protocol +over which it was transferred, or a local file path which was read, or some other +input source + + +-- + +*`zeek.notice.file.mime_type`*:: ++ +-- +type: keyword + +A mime type if the notice is related to a file + + +-- + +*`zeek.notice.file.is_orig`*:: ++ +-- +type: boolean + +If the source of this file is a network connection, this field indicates if the file is +being sent by the originator of the connection or the responder + + +-- + +*`zeek.notice.file.seen_bytes`*:: ++ +-- +type: long + +Number of bytes provided to the file analysis engine for the file + + +-- + +*`zeek.fnotice.file.total_bytes`*:: ++ +-- +type: long + +Total number of bytes that are supposed to comprise the full file + + +-- + +*`zeek.notice.file.missing_bytes`*:: ++ +-- +type: long + +The number of bytes in the file stream that were completely missed during the process +of analysis + + +-- + +*`zeek.notice.file.overflow_bytes`*:: ++ +-- +type: long + +The number of bytes in the file stream that were not delivered to stream file analyzers. +This could be overlapping bytes or bytes that couldn't be reassembled + + +-- + +*`zeek.notice.fuid`*:: ++ +-- +type: keyword + +A file unique ID if this notice is related to a file + + +-- + +*`zeek.notice.note`*:: ++ +-- +type: keyword + +The type of the notice + + +-- + +*`zeek.notice.msg`*:: ++ +-- +type: keyword + +The human readable message for the notice. + + +-- + +*`zeek.notice.sub`*:: ++ +-- +type: keyword + +The human readable sub-message + + +-- + +*`zeek.notice.n`*:: ++ +-- +type: long + +Associated count, or a status code + + +-- + +*`zeek.notice.peer_name`*:: ++ +-- +type: keyword + +Name of remote peer that raised this notice + + +-- + +*`zeek.notice.peer_descr`*:: ++ +-- +type: text + +Textual description for the peer that raised this notice + + +-- + +*`zeek.notice.actions`*:: ++ +-- +type: keyword + +The actions which have been applied to this notice + + +-- + +*`zeek.notice.email_body_sections`*:: ++ +-- +type: text + +By adding chunks of text into this element, other scripts can expand on notices +that are being emailed + + +-- + +*`zeek.notice.email_delay_tokens`*:: ++ +-- +type: keyword + +Adding a string token to this set will cause the built-in emailing functionality +to delay sending the email either the token has been removed or the email +has been delayed for the specified time duration + + +-- + +*`zeek.notice.identifier`*:: ++ +-- +type: keyword + +This field is provided when a notice is generated for the purpose of deduplicating notices + + +-- + +*`zeek.notice.suppress_for`*:: ++ +-- +type: double + +This field indicates the length of time that this unique notice should be suppressed + + +-- + +*`zeek.notice.dropped`*:: ++ +-- +type: boolean + +Indicate if the source IP address was dropped and denied network access + + -- diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index b9b7b0d4588..07604623621 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -427,6 +427,8 @@ filebeat.modules: enabled: true ssl: enabled: true + notice: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/module/zeek/README-developer.md b/x-pack/filebeat/module/zeek/README-developer.md index a1b431b64a6..20410f14c1b 100644 --- a/x-pack/filebeat/module/zeek/README-developer.md +++ b/x-pack/filebeat/module/zeek/README-developer.md @@ -14,7 +14,7 @@ brew install bro * Configure it to process network traffic and generate logs. * Edit `/usr/local/etc/node.cfg` to use the proper network interfaces. -* Edit `/usr/local/etc/network.cfg` to specify local networks accordingly. +* Edit `/usr/local/etc/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/local/share/bro/site/local.bro` to use JSON output. ### Install Zeek/Bro (for Ubuntu Linux) @@ -26,7 +26,7 @@ apt install broctl * Configure it to process network traffic and generate logs. * Edit `/etc/bro/node.cfg` to use the proper network interfaces. -* Edit `/etc/bro/network.cfg` to specify local networks accordingly. +* Edit `/etc/bro/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/share/bro/site/local.bro` to use JSON output. ## Start Zeek/Bro @@ -52,7 +52,7 @@ mage build Update filebeat.yml to point to Elasticsearch and Kibana. Setup Filebeat. ``` -./filebeat setup --modules zeek -e -E setup.dashboards.directory=build/kibana +./filebeat setup --modules zeek -e -E 'setup.dashboards.directory=build/kibana' ``` Enable the Filebeat zeek module diff --git a/x-pack/filebeat/module/zeek/README.md b/x-pack/filebeat/module/zeek/README.md index 44a51dbf456..740fff62641 100644 --- a/x-pack/filebeat/module/zeek/README.md +++ b/x-pack/filebeat/module/zeek/README.md @@ -14,7 +14,7 @@ brew install bro * Configure it to process network traffic and generate logs. * Edit `/usr/local/etc/node.cfg` to use the proper network interfaces. -* Edit `/usr/local/etc/network.cfg` to specify local networks accordingly. +* Edit `/usr/local/etc/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/local/share/bro/site/local.bro` to use JSON output. ### Install Zeek/Bro (for Ubuntu Linux) @@ -26,7 +26,7 @@ apt install broctl * Configure it to process network traffic and generate logs. * Edit `/etc/bro/node.cfg` to use the proper network interfaces. -* Edit `/etc/bro/network.cfg` to specify local networks accordingly. +* Edit `/etc/bro/networks.cfg` to specify local networks accordingly. * Set `redef LogAscii::use_json=T;` in `/usr/share/bro/site/local.bro` to use JSON output. ## Start Zeek/Bro @@ -44,7 +44,7 @@ Grab the filebeat binary from elastic.co, and install it by following the instru Update filebeat.yml to point to Elasticsearch and Kibana. Setup Filebeat. ``` -./filebeat setup --modules zeek -e -E setup.dashboards.directory=build/kibana +./filebeat setup --modules zeek -e -E 'setup.dashboards.enabled=true' ``` Enable the Filebeat zeek module diff --git a/x-pack/filebeat/module/zeek/_meta/config.yml b/x-pack/filebeat/module/zeek/_meta/config.yml index a79fc0456c2..22bf8b09f27 100644 --- a/x-pack/filebeat/module/zeek/_meta/config.yml +++ b/x-pack/filebeat/module/zeek/_meta/config.yml @@ -10,6 +10,8 @@ enabled: true ssl: enabled: true + notice: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/module/zeek/_meta/fields.yml b/x-pack/filebeat/module/zeek/_meta/fields.yml index 60c59f4e75d..cba71f9f4e7 100644 --- a/x-pack/filebeat/module/zeek/_meta/fields.yml +++ b/x-pack/filebeat/module/zeek/_meta/fields.yml @@ -10,278 +10,610 @@ fields: - name: session_id type: keyword + description: > + A unique identifier of the session - name: connection.local_orig type: boolean + description: > + Indicates whether the session is originated locally - name: connection.local_resp type: boolean + description: > + Indicates whether the session is responded locally - name: connection.missed_bytes type: long + description: > + Missed bytes for the session - name: connection.state type: keyword + description: > + Flags indicating the state of the session - name: connection.history type: keyword + description: > + Flags indicating the history of the session - name: connection.orig_l2_addr type: keyword + description: > + Link-layer address of the originator, if available - - name: resp_l2_addr + - name: connection.resp_l2_addr type: keyword + description: > + Link-layer address of the responder, if available - - name: vlan - type: keyword + - name: connection.vlan + type: integer + description: > + VLAN identifier - - name: inner_vlan - type: keyword + - name: connection.inner_vlan + type: integer + description: > + VLAN identifier - name: dns.trans_id - type: integer + type: keyword + description: > + DNS transaction identifier - name: dns.rtt type: double + description: > + Round trip time for the query and response - name: dns.query type: keyword + description: > + The domain name that is the subject of the DNS query - name: dns.qclass type: long + description: > + The QCLASS value specifying the class of the query - name: dns.qclass_name type: keyword + description: > + A descriptive name for the class of the query - name: dns.qtype type: long + description: > + A QTYPE value specifying the type of the query - name: dns.qtype_name type: keyword + description: > + A descriptive name for the type of the query - name: dns.rcode type: long + description: > + The response code value in DNS response messages - name: dns.rcode_name type: keyword + description: > + A descriptive name for the response code value - name: dns.AA type: boolean + description: | + The Authoritative Answer bit for response messages specifies that the responding + name server is an authority for the domain name in the question section - name: dns.TC type: boolean + description: > + The Truncation bit specifies that the message was truncated - name: dns.RD type: boolean + description: | + The Recursion Desired bit in a request message indicates that the client + wants recursive service for this query - name: dns.RA type: boolean + description: | + The Recursion Available bit in a response message indicates that the name + server supports recursive queries. - name: dns.answers type: keyword + description: > + The set of resource descriptions in the query answer - name: dns.TTLs type: double + description: > + The caching intervals of the associated RRs described by the answers field - name: dns.rejected type: boolean + description: > + Indicates whether the DNS query was rejected by the server - name: dns.total_answers type: integer + description: > + The total number of resource records in the reply - name: dns.total_replies type: integer + description: > + The total number of resource records in the reply message - name: dns.saw_query type: boolean + description: > + Whether the full DNS query has been seen - name: dns.saw_reply type: boolean + description: > + Whether the full DNS reply has been seen - name: http.trans_depth type: integer + description: > + Represents the pipelined depth into the connection of this request/response transaction - name: http.status_msg type: keyword + description: > + Status message returned by the server - name: http.info_code type: integer + description: > + Last seen 1xx informational reply code returned by the server. - name: http.info_msg type: keyword + description: > + Last seen 1xx informational reply message returned by the server. - - name: http.filename + - name: http.tags type: keyword + description: | + A set of indicators of various attributes discovered and related to a particular + request/response pair. - - name: http.tags + + - name: http.password type: keyword + description: > + Password if basic-auth is performed for the request - name: http.captured_password type: boolean + description: > + Determines if the password will be captured for this request - name: http.proxied type: keyword + description: > + All of the headers that may indicate if the HTTP request was proxied - name: http.range_request type: boolean + description: > + Indicates if this request can assume 206 partial content in response - name: http.client_header_names type: keyword + description: | + The vector of HTTP header names sent by the client. No header values + are included here, just the header names. - name: http.server_header_names type: keyword + description: | + The vector of HTTP header names sent by the server. No header values + are included here, just the header names - name: http.orig_fuids type: keyword + description: > + An ordered vector of file unique IDs from the originator - name: http.orig_mime_types type: keyword + description: > + An ordered vector of mime types from the originator - name: http.orig_filenames type: keyword + description: > + An ordered vector of filenames from the originator - name: http.resp_fuids type: keyword + description: > + An ordered vector of file unique IDs from the responder - name: http.resp_mime_types type: keyword + description: > + An ordered vector of mime types from the responder - name: http.resp_filenames type: keyword + description: > + An ordered vector of filenames from the responder - name: http.orig_mime_depth type: integer + description: > + Current number of MIME entities in the HTTP request message body - name: http.resp_mime_depth type: integer + description: > + Current number of MIME entities in the HTTP response message body - name: files.fuid type: keyword + description: > + A file unique identifier - name: files.tx_host type: ip + description: > + The host that transferred the file - name: files.rx_host type: ip + description: > + The host that received the file - name: files.session_ids type: keyword + description: > + The sessions that have this file - name: files.source type: keyword + description: | + An identification of the source of the file data. E.g. it may be a network protocol + over which it was transferred, or a local file path which was read, or some other + input source - name: files.depth type: long - - - names: files.direction - type: keyword + description: | + A value to represent the depth of this file in relation to its source. In SMTP, it + is the depth of the MIME attachment on the message. In HTTP, it is the depth of the + request within the TCP connection - name: files.analyzers type: keyword + description: > + A set of analysis types done during the file analysis - name: files.mime_type type: keyword + description: > + Mime type of the file - name: files.filename type: keyword + description: > + Name of the file if available - name: files.local_orig type: boolean + description: | + If the source of this file is a network connection, this field indicates if the data + originated from the local network or not - name: files.is_orig type: boolean + description: | + If the source of this file is a network connection, this field indicates if the file is + being sent by the originator of the connection or the responder - name: files.duration type: double + description: > + The duration the file was analyzed for. Not the duration of the session. - name: files.seen_bytes type: long + description: > + Number of bytes provided to the file analysis engine for the file - name: files.total_bytes type: long + description: > + Total number of bytes that are supposed to comprise the full file - name: files.missing_bytes type: long + description: | + The number of bytes in the file stream that were completely missed during the process + of analysis - name: files.overflow_bytes type: long + description: | + The number of bytes in the file stream that were not delivered to stream file analyzers. + This could be overlapping bytes or bytes that couldn't be reassembled - name: files.timedout type: boolean + description: > + Whether the file analysis timed out at least once for the file - name: files.parent_fuid type: keyword + description: | + Identifier associated with a container file from which this one was extracted as part of + the file analysis - name: files.md5 type: keyword + description: > + An MD5 digest of the file contents - name: files.sha1 type: keyword + description: > + A SHA1 digest of the file contents - name: files.sha256 type: keyword + description: > + A SHA256 digest of the file contents. - name: files.extracted type: keyword + description: > + Local filename of extracted file - name: files.extracted_cutoff type: boolean + description: > + Indicate whether the file being extracted was cut off hence not extracted completely - name: files.extracted_size type: long + description: > + The number of bytes extracted to disk - name: files.entropy type: double + description: > + The information density of the contents of the file - name: ssl.version type: keyword + description: > + SSL/TLS version that was logged - name: ssl.cipher type: keyword + description: > + SSL/TLS cipher suite that was logged - name: ssl.curve type: keyword + description: > + Elliptic curve that was logged when using ECDH/ECDHE - name: ssl.server_name type: keyword + description: | + Value of the Server Name Indicator SSL/TLS extension. It indicates the server name + that the client was requesting - name: ssl.resumed type: boolean + description: | + Flag to indicate if the session was resumed reusing the key material exchanged in an + earlier connection - name: ssl.next_protocol type: keyword + description: > + Next protocol the server chose using the application layer next protocol extension - name: ssl.established type: boolean + description: > + Flag to indicate if this ssl session has been established successfully - name: ssl.cert_chain type: keyword + description: > + Chain of certificates offered by the server to validate its complete signing chain - name: ssl.cert_chain_fuids type: keyword + description: > + An ordered vector of certificate file identifiers for the certificates offered by the server - name: ssl.client_cert_chain type: keyword + description: > + Chain of certificates offered by the client to validate its complete signing chain - name: ssl.client_cert_chain_fuids type: keyword + description: > + An ordered vector of certificate file identifiers for the certificates offered by the client - name: ssl.issuer type: keyword + description: > + Subject of the signer of the X.509 certificate offered by the server - name: ssl.client_issuer type: keyword + description: > + Subject of the X.509 certificate offered by the client - name: ssl.validation_status type: keyword + description: > + Result of certificate validation for this connection + + - name: ssl.validation_code + type: keyword + description: > + Result of certificate validation for this connection, given as OpenSSL validation code - name: ssl.subject type: keyword + description: > + Subject of the X.509 certificate offered by the server - name: ssl.client_subject type: keyword + description: > + Subject of the X.509 certificate offered by the client - name: ssl.last_alert type: keyword + description: > + Last alert that was seen during the connection + + - name: notice.connection_id + type: keyword + description: > + Identifier of the related connection session + + - name: notice.icmp_id + type: keyword + description: > + Identifier of the related ICMP session + + - name: notice.file.id + type: keyword + description: > + An identifier associated with a single file that is related to this notice + + - name: notice.file.parent_id + type: keyword + description: > + Identifier associated with a container file from which this one was extracted + + - name: notice.file.source + type: keyword + description: | + An identification of the source of the file data. E.g. it may be a network protocol + over which it was transferred, or a local file path which was read, or some other + input source + + - name: notice.file.mime_type + type: keyword + description: > + A mime type if the notice is related to a file + + - name: notice.file.is_orig + type: boolean + description: | + If the source of this file is a network connection, this field indicates if the file is + being sent by the originator of the connection or the responder + + - name: notice.file.seen_bytes + type: long + description: > + Number of bytes provided to the file analysis engine for the file + + - name: fnotice.file.total_bytes + type: long + description: > + Total number of bytes that are supposed to comprise the full file + + - name: notice.file.missing_bytes + type: long + description: | + The number of bytes in the file stream that were completely missed during the process + of analysis + + - name: notice.file.overflow_bytes + type: long + description: | + The number of bytes in the file stream that were not delivered to stream file analyzers. + This could be overlapping bytes or bytes that couldn't be reassembled + + - name: notice.fuid + type: keyword + description: > + A file unique ID if this notice is related to a file + + - name: notice.note + type: keyword + description: > + The type of the notice + + - name: notice.msg + type: keyword + description: > + The human readable message for the notice. + + - name: notice.sub + type: keyword + description: > + The human readable sub-message + + - name: notice.n + type: long + description: > + Associated count, or a status code + + - name: notice.peer_name + type: keyword + description: > + Name of remote peer that raised this notice + + - name: notice.peer_descr + type: text + description: > + Textual description for the peer that raised this notice + + - name: notice.actions + type: keyword + description: > + The actions which have been applied to this notice + + - name: notice.email_body_sections + type: text + description: | + By adding chunks of text into this element, other scripts can expand on notices + that are being emailed + + - name: notice.email_delay_tokens + type: keyword + description: | + Adding a string token to this set will cause the built-in emailing functionality + to delay sending the email either the token has been removed or the email + has been delayed for the specified time duration + + - name: notice.identifier + type: keyword + description: > + This field is provided when a notice is generated for the purpose of deduplicating notices + + - name: notice.suppress_for + type: double + description: > + This field indicates the length of time that this unique notice should be suppressed + + - name: notice.dropped + type: boolean + description: > + Indicate if the source IP address was dropped and denied network access + + diff --git a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json index 5fd7816eb98..8e12b26cb26 100644 --- a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json +++ b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json @@ -3,17 +3,194 @@ { "attributes": { "description": "", + "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Destination Geo [SIEM Zeek] ECS", + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 20, + "i": "1", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "2", + "w": 16, + "x": 0, + "y": 20 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "3", + "w": 16, + "x": 16, + "y": 20 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "4", + "w": 16, + "x": 32, + "y": 20 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "5", + "w": 16, + "x": 0, + "y": 32 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "6", + "w": 16, + "x": 16, + "y": 32 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "7", + "w": 16, + "x": 32, + "y": 32 + }, + "panelIndex": "7", + "panelRefName": "panel_6", + "version": "7.0.0-beta1" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "8", + "w": 48, + "x": 0, + "y": 44 + }, + "panelIndex": "8", + "panelRefName": "panel_7", + "version": "7.0.0-beta1" + } + ], + "timeRestore": false, + "title": "Zeek Overview Dashboard", + "version": 1 + }, + "id": "7cbb5410-3700-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "f469f230-370c-11e9-aa6d-ff445a78330c", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "1df7ea80-370d-11e9-aa6d-ff445a78330c", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "466e5850-370d-11e9-aa6d-ff445a78330c", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "649acd40-370d-11e9-aa6d-ff445a78330c", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "9436c270-370d-11e9-aa6d-ff445a78330c", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "bec2f0e0-370d-11e9-aa6d-ff445a78330c", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "e042fda0-370d-11e9-aa6d-ff445a78330c", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "f8c40810-370d-11e9-aa6d-ff445a78330c", + "name": "panel_7", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-02-23T05:05:18.205Z", + "version": "WzMxMTYsNF0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Destination Geo [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -47,6 +224,18 @@ "params": { "addTooltip": true, "colorSchema": "Yellow to Red", + "dimensions": { + "geocentroid": null, + "geohash": null, + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "heatClusterSize": 1.5, "isDesaturated": true, "legendPosition": "bottomright", @@ -61,25 +250,27 @@ "options": { "format": "image/png", "transparent": true - }, - "selectedTmsLayer": { - "attribution": "\u003cp\u003e\u0026#169; \u003ca href=\"http://www.openstreetmap.org/copyright\"\u003eOpenStreetMap\u003c/a\u003e contributors | \u003ca href=\"https://www.elastic.co/elastic-maps-service\"\u003eElastic Maps Service\u003c/a\u003e\u003c/p\u003e\u0026#10;", - "id": "road_map", - "maxZoom": 18, - "minZoom": 0, - "subdomains": [], - "url": "https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree\u0026my_app_name=kibana\u0026my_app_version=6.5.4\u0026license=decdfd78-7d5b-47b7-9627-603d9b789d29" } } }, - "title": "Destination Geo [SIEM Zeek] ECS", + "title": "Destination Geo [Zeek]", "type": "tile_map" } }, - "id": "5d95a3e0-1a29-11e9-84b1-a12c578fa9e8-ecs", + "id": "f469f230-370c-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:27:37.758Z", - "version": 1 + "updated_at": "2019-02-26T00:06:27.634Z", + "version": "WzMyNzUsNV0=" }, { "attributes": { @@ -87,14 +278,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Network Transport [SIEM Zeek] ECS", + "title": "Network Transport [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -126,6 +317,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -136,14 +337,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Transport [SIEM Zeek] ECS", + "title": "Network Transport [Zeek]", "type": "pie" } }, - "id": "c337dbf0-1a29-11e9-84b1-a12c578fa9e8-ecs", + "id": "1df7ea80-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:30:28.271Z", - "version": 1 + "updated_at": "2019-02-26T00:07:08.521Z", + "version": "WzMyNzgsNV0=" }, { "attributes": { @@ -151,14 +362,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Network Application [SIEM Zeek] ECS", + "title": "Network Application [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -181,7 +392,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 10 + "size": 5 }, "schema": "segment", "type": "terms" @@ -190,6 +401,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -200,14 +421,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Application [SIEM Zeek] ECS", + "title": "Network Application [Zeek]", "type": "pie" } }, - "id": "f054ee70-1a29-11e9-84b1-a12c578fa9e8-ecs", + "id": "466e5850-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:31:43.959Z", - "version": 1 + "updated_at": "2019-02-26T00:06:41.868Z", + "version": "WzMyNzYsNV0=" }, { "attributes": { @@ -215,14 +446,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Network Traffic Direction [SIEM Zeek] ECS", + "title": "Network Traffic Direction [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -254,6 +485,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -264,14 +505,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Network Traffic Direction [SIEM Zeek] ECS", + "title": "Network Traffic Direction [Zeek]", "type": "pie" } }, - "id": "15922a40-1a2a-11e9-84b1-a12c578fa9e8-ecs", + "id": "649acd40-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:32:46.436Z", - "version": 1 + "updated_at": "2019-02-26T00:06:55.885Z", + "version": "WzMyNzcsNV0=" }, { "attributes": { @@ -279,14 +530,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Top DNS Domains [SIEM Zeek] ECS", + "title": "Top DNS Domains [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -309,7 +560,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 8 + "size": 10 }, "schema": "segment", "type": "terms" @@ -318,6 +569,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -328,14 +589,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top DNS Domains [SIEM Zeek] ECS", + "title": "Top DNS Domains [Zeek]", "type": "pie" } }, - "id": "b3705f00-1a2c-11e9-84b1-a12c578fa9e8-ecs", + "id": "9436c270-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:51:30.288Z", - "version": 1 + "updated_at": "2019-02-26T00:07:23.763Z", + "version": "WzMyNzksNV0=" }, { "attributes": { @@ -343,14 +614,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Top URL Domain [SIEM Zeek] ECS", + "title": "Top URL Domains [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -373,7 +644,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 8 + "size": 10 }, "schema": "segment", "type": "terms" @@ -382,6 +653,31 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "buckets": [ + { + "accessor": 0, + "aggType": "terms", + "format": { + "id": "terms", + "params": { + "id": "string", + "missingBucketLabel": "Missing", + "otherBucketLabel": "Other" + } + }, + "params": {} + } + ], + "metric": { + "accessor": 1, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -392,14 +688,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top URL Domain [SIEM Zeek] ECS", + "title": "Top URL Domains [Zeek]", "type": "pie" } }, - "id": "ef0cfdc0-1a2c-11e9-84b1-a12c578fa9e8-ecs", + "id": "bec2f0e0-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:53:10.300Z", - "version": 1 + "updated_at": "2019-02-26T00:07:49.910Z", + "version": "WzMyODEsNV0=" }, { "attributes": { @@ -407,14 +713,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "7f83fe80-1947-11e9-84dc-b9c00e3e5a85", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Top SSL Server [SIEM Zeek] ECS", + "title": "Top SSL Servers [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -437,7 +743,7 @@ "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 8 + "size": 10 }, "schema": "segment", "type": "terms" @@ -446,6 +752,16 @@ "params": { "addLegend": true, "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, "isDonut": true, "labels": { "last_level": true, @@ -456,14 +772,24 @@ "legendPosition": "right", "type": "pie" }, - "title": "Top SSL Server [SIEM Zeek] ECS", + "title": "Top SSL Servers [Zeek]", "type": "pie" } }, - "id": "13454cb0-1a2d-11e9-84b1-a12c578fa9e8-ecs", + "id": "e042fda0-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-17T07:54:11.067Z", - "version": 1 + "updated_at": "2019-02-26T00:07:36.653Z", + "version": "WzMyODAsNV0=" }, { "attributes": { @@ -472,12 +798,12 @@ "searchSourceJSON": { "filter": [], "query": { - "language": "lucene", + "language": "kuery", "query": "" } } }, - "title": "Time Series Count [SIEM Zeek] ECS", + "title": "Number of Sessions Overtime [Zeek]", "uiStateJSON": {}, "version": 1, "visState": { @@ -486,26 +812,8 @@ "axis_formatter": "number", "axis_position": "left", "axis_scale": "normal", - "background_color_rules": [ - { - "id": "3716ea90-1a2d-11e9-b2af-13b289f0bf65" - } - ], - "bar_color_rules": [ - { - "id": "3822dc50-1a2d-11e9-b2af-13b289f0bf65" - } - ], - "gauge_color_rules": [ - { - "id": "4c1a3ff0-1a2d-11e9-b2af-13b289f0bf65" - } - ], - "gauge_inner_width": 10, - "gauge_style": "half", - "gauge_width": 10, "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "filebeat-*", + "index_pattern": "", "interval": "auto", "series": [ { @@ -513,7 +821,6 @@ "chart_type": "line", "color": "#68BC00", "fill": 0.5, - "filter": "tags:zeek", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "line_width": 1, @@ -525,183 +832,28 @@ ], "point_size": 1, "separate_axis": 0, - "split_mode": "filter", + "split_mode": "everything", "stacked": "none" } ], "show_grid": 1, - "show_legend": 0, + "show_legend": 1, "time_field": "@timestamp", "type": "timeseries" }, - "title": "Time Series Count [SIEM Zeek] ECS", + "title": "Number of Sessions Overtime [Zeek]", "type": "metrics" } }, - "id": "fad258c0-1078-11e9-b27a-69e6e8b80a25-ecs", - "type": "visualization", - "updated_at": "2019-01-17T07:56:26.486Z", - "version": 74 - }, - { - "attributes": { - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "lucene", - "query": "" - } - } - }, - "optionsJSON": { - "darkTheme": false, - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": { - "mapCenter": [ - 20.3034175184893, - -5.537109375000001 - ], - "mapZoom": 2 - }, - "gridData": { - "h": 18, - "i": "1", - "w": 48, - "x": 0, - "y": 0 - }, - "id": "5d95a3e0-1a29-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "1", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true - } - }, - "gridData": { - "h": 10, - "i": "2", - "w": 16, - "x": 0, - "y": 18 - }, - "id": "c337dbf0-1a29-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "2", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true - } - }, - "gridData": { - "h": 10, - "i": "3", - "w": 17, - "x": 16, - "y": 18 - }, - "id": "f054ee70-1a29-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "3", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true - } - }, - "gridData": { - "h": 10, - "i": "4", - "w": 15, - "x": 33, - "y": 18 - }, - "id": "15922a40-1a2a-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "4", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "5", - "w": 16, - "x": 0, - "y": 28 - }, - "id": "b3705f00-1a2c-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "5", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "6", - "w": 17, - "x": 16, - "y": 28 - }, - "id": "ef0cfdc0-1a2c-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "6", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "7", - "w": 15, - "x": 33, - "y": 28 - }, - "id": "13454cb0-1a2d-11e9-84b1-a12c578fa9e8-ecs", - "panelIndex": "7", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 9, - "i": "8", - "w": 48, - "x": 0, - "y": 39 - }, - "id": "fad258c0-1078-11e9-b27a-69e6e8b80a25-ecs", - "panelIndex": "8", - "type": "visualization", - "version": "6.5.4" - } - ], - "timeRestore": false, - "title": "Zeek Overview Dashboard [SIEM] ECS", - "version": 1 + "id": "f8c40810-370d-11e9-aa6d-ff445a78330c", + "migrationVersion": { + "visualization": "7.0.0" }, - "id": "87b0c430-1a2d-11e9-84b1-a12c578fa9e8-ecs", - "type": "dashboard", - "updated_at": "2019-01-17T07:57:50.613Z", - "version": 2 + "references": [], + "type": "visualization", + "updated_at": "2019-02-26T00:05:56.379Z", + "version": "WzMyNzQsNV0=" } ], - "version": "6.5.4" + "version": "7.0.0-beta1" } \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml index b925dc01aec..47fb2906642 100644 --- a/x-pack/filebeat/module/zeek/connection/config/connection.yml +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -17,7 +17,7 @@ processors: to: "zeek.connection" - from: "zeek.connection.duration" - to: "event.duration" + to: "temp.duration" - from: "zeek.connection.id.orig_h" to: "source.address" diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json index 862787cd0f7..1ca5eadc409 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.json @@ -4,27 +4,44 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['connection']['ts'] * params.multiplier; ctx.zeek.connection.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['connection']['ts'] * 1000; ctx.zeek.connection.remove('ts');" } }, { "script": { "lang": "painless", - "source": "ctx.event.duration = (long)ctx.event.duration * params.multiplier", + "source": "ctx.event.duration = (long)ctx.temp.duration * params.scale", "params": { - "multiplier": 1000000000 + "scale": 1000000000 }, - "ignore_failure": true + "if": "ctx.temp?.duration != null" + } + }, + { + "remove": { + "field": "temp.duration", + "ignore_missing": true + } + }, + { + "set": { + "field": "event.id", + "value": "{{zeek.session_id}}", + "if": "ctx.zeek.session_id != null" + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.zeek.connection.local_orig) ctx.tags.add(\"local_orig\");", + "if": "ctx.zeek.connection.local_orig != null" } }, { "script": { "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-connection\"", - "ignore_failure": true + "source": "if (ctx.zeek.connection.local_resp) ctx.tags.add(\"local_resp\");", + "if": "ctx.zeek.connection.local_resp != null" } }, { diff --git a/x-pack/filebeat/module/zeek/connection/manifest.yml b/x-pack/filebeat/module/zeek/connection/manifest.yml index 53e7f507cd6..fc71598ebdd 100644 --- a/x-pack/filebeat/module/zeek/connection/manifest.yml +++ b/x-pack/filebeat/module/zeek/connection/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/conn.log - name: tags - default: [zeek] + default: [zeek.connection] ingest_pipeline: ingest/pipeline.json input: config/connection.yml diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log b/x-pack/filebeat/module/zeek/connection/test/connection-json.log index 9e4b15b535a..4e47ebedcec 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log @@ -1 +1,3 @@ {"ts":1547188415.857497,"uid":"CAcJw21BbVedgFnYH3","id.orig_h":"192.168.86.167","id.orig_p":38339,"id.resp_h":"192.168.86.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":true,"local_resp":true,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} +{"ts":1547188416.857497,"uid":"CAcJw21BbVedgFnYH4","id.orig_h":"192.168.86.167","id.orig_p":38340,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} +{"ts":1547188417.857497,"uid":"CAcJw21BbVedgFnYH5","id.orig_h":"4.4.2.2","id.orig_p":383341,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.076967,"orig_bytes":75,"resp_bytes":178,"conn_state":"SF","local_orig":false,"local_resp":false,"missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":206,"tunnel_parents":[]} diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json index 89b37e6e83e..becb63faad1 100644 --- a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json @@ -9,7 +9,7 @@ "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.connection", "event.duration": 0.0, - "event.id": "CAcJw21BbVedgFnYH3-connection", + "event.id": "CAcJw21BbVedgFnYH3", "event.module": "zeek", "fileset.name": "connection", "input.type": "log", @@ -24,7 +24,9 @@ "source.packets": 1, "source.port": 38339, "tags": [ - "zeek" + "zeek.connection", + "local_orig", + "local_resp" ], "zeek.connection.history": "Dd", "zeek.connection.local_orig": true, @@ -32,5 +34,86 @@ "zeek.connection.missed_bytes": 0, "zeek.connection.state": "SF", "zeek.session_id": "CAcJw21BbVedgFnYH3" + }, + { + "@timestamp": 1547188416000, + "destination.address": "8.8.8.8", + "destination.bytes": 206, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.packets": 1, + "destination.port": 53, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.connection", + "event.duration": 0.0, + "event.id": "CAcJw21BbVedgFnYH4", + "event.module": "zeek", + "fileset.name": "connection", + "input.type": "log", + "log.offset": 398, + "network.application": "dns", + "network.direction": "outbound", + "network.transport": "udp", + "service.type": "zeek", + "source.address": "192.168.86.167", + "source.bytes": 103, + "source.ip": "192.168.86.167", + "source.packets": 1, + "source.port": 38340, + "tags": [ + "zeek.connection", + "local_orig" + ], + "zeek.connection.history": "Dd", + "zeek.connection.local_orig": true, + "zeek.connection.local_resp": false, + "zeek.connection.missed_bytes": 0, + "zeek.connection.state": "SF", + "zeek.session_id": "CAcJw21BbVedgFnYH4" + }, + { + "@timestamp": 1547188417000, + "destination.address": "8.8.8.8", + "destination.bytes": 206, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.packets": 1, + "destination.port": 53, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.connection", + "event.duration": 0.0, + "event.id": "CAcJw21BbVedgFnYH5", + "event.module": "zeek", + "fileset.name": "connection", + "input.type": "log", + "log.offset": 792, + "network.application": "dns", + "network.direction": "external", + "network.transport": "udp", + "service.type": "zeek", + "source.address": "4.4.2.2", + "source.bytes": 103, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "4.4.2.2", + "source.packets": 1, + "source.port": 383341, + "tags": [ + "zeek.connection" + ], + "zeek.connection.history": "Dd", + "zeek.connection.local_orig": false, + "zeek.connection.local_resp": false, + "zeek.connection.missed_bytes": 0, + "zeek.connection.state": "SF", + "zeek.session_id": "CAcJw21BbVedgFnYH5" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json index 28f4adb5f41..bea3798a7bb 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json @@ -4,17 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['dns']['ts'] * params.multiplier; ctx.zeek.dns.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['dns']['ts'] * 1000; ctx.zeek.dns.remove('ts');" } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-dns\"", - "ignore_failure": true + "set": { + "field": "event.id", + "value": "{{zeek.session_id}}", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/dns/manifest.yml b/x-pack/filebeat/module/zeek/dns/manifest.yml index da306cc5cfe..71032e045d8 100644 --- a/x-pack/filebeat/module/zeek/dns/manifest.yml +++ b/x-pack/filebeat/module/zeek/dns/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/dns.log - name: tags - default: [zeek] + default: [zeek.dns] ingest_pipeline: ingest/pipeline.json input: config/dns.yml diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json index f30c13cfaf6..acc66d7e044 100644 --- a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -6,7 +6,7 @@ "destination.port": 53, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.dns", - "event.id": "CAcJw21BbVedgFnYH3-dns", + "event.id": "CAcJw21BbVedgFnYH3", "event.module": "zeek", "fileset.name": "dns", "input.type": "log", @@ -17,7 +17,7 @@ "source.ip": "192.168.86.167", "source.port": 38339, "tags": [ - "zeek" + "zeek.dns" ], "zeek.dns.AA": false, "zeek.dns.RA": true, diff --git a/x-pack/filebeat/module/zeek/fields.go b/x-pack/filebeat/module/zeek/fields.go index fe6e78a484e..4ecc23bb6dc 100644 --- a/x-pack/filebeat/module/zeek/fields.go +++ b/x-pack/filebeat/module/zeek/fields.go @@ -19,5 +19,5 @@ func init() { // AssetZeek returns asset data. // This is the base64 encoded gzipped contents of module/zeek. func AssetZeek() string { - return "eJycl7GOIzcMhns/hV7gNsgBl8JFgEuCVEkTbJVGkCXOjLIaUUdK3vU+fSCNY2M99g2V0vb3kxRFivQn9QKnvXoHeNkplX0OsFd/L58csCWfsse4Vz/vlFLqT3QlgBqQ1GSiCz6OKuDIKhG6YsGpw6nJf/iFcKfU4CE43u+U+qSimeHiSal8SrBXI2FJ7fMdb0r93vRqIJwvZheHZshAKiLNJvh3U2VNc/V49cnA7DFq785mF98vcHpFcrsPrMUYwVZzTwGtCRrJjx9kB8QAJm7ICDh1yWbPDE4fThn4gzBgHB+qOJsMXaeaPGekU5em5kCHz9o4R9vCenQ5fQwmblM+RiAtY13kp0wm8u2F+5hhBFrTlPMH0GE5BFhz3wpIMtdQGwxvXeQV1PULoeX6u8Rw/bHDLll0EruN67D79et2H1Tu+VcZ99dvQk7o10R+BWLZYZ6f/2BRqRD8AzaDuxtD/WZdsphN0PeieVi3i4QgBQ9CCZtXvS7jh8mpeLUvwKec07nvHKQ83Y1ndfKmqq9YYT3zePcW7ot8HFCvavb7jprmkZs1PvgAskJfDm9GQRk11JqUC4HTyTBfDrmd30T45kEwxxpNJo6gCb4V4Cz0YIOHmPUExgG1NpeeiYGOQP9H2ebLULzrEsx+Bl3JPjfnO5WK2jTria0JumNb3HTGdk3D4477XoBCVQ2Ln2oOtuNa2PymJ7wpOZ/ukSQmr1ucID1nCRayoKT4Oh23E5AvqKdlN5LaNtGE07toziyCSwVJBfLHauF71ttF4bkLd4XMKkX35uV/9wtRtP6eS6zNPjlf92sfxw4FHoGGgK89QfkZHBbBW7vwyVB9bHtaa3ZfxOU/mR872M9ffpLS8JbJrNYbiUDbknEYpAm66ti/b22nZ0nMhOm0WXXM4ekIxKImrrD1aQLBP5rGFjoKOrGi57Epa90qIOAyP9gsV3CEt6wTYUaLQWYfOJtD8DxJfVigrO1kvDSRF146WJtqWUy6nd3Kenx65iK+8sVRj+RognftjdTLCiysmHKofy66wurSBMNZmwD0gN/9GwAA///PiHUF" + return "eJzkW99vG7eyfvdfMW/3JVHaAClw83ABN05RA3Zuahu995wXgVqOJNZccktyZSvoH38wQ3J/SCtZu3GMnNOXALF25/uGHM58HHJfwz1u38MXxPszgKCCxvfwz/g/ib5wqgrKmvfwP2cAANdW1hphaR2shZFamRVou/JQOSvrAiUstvz6m5+dPQNYKtTSvz8DeA1GlNggAYRthe9h5Wxd8f8H0AB+4fdh6WzZmI2AYhnQgbGuFFp9EfQav9MitpgevVfWzJVMZiP2PW4frMt/G8QHOIfaqD9rBCXRBLVU6MAuIawxm4WzHlhhjcGCzMy0LYSeW6dWPdyFtRqFOY57aaQqREAPD2sMa3Q9SOWBzCojAkpgHL19godDX30DHmTWGnkCjVJ5j3K+2Ab0PSLamtVxFtf8KvCrHH0dEgfxfBABx8/4L1qsPKjoNwU4Y5GtnYk/CLxWPli3fSboZO1UcIqLuX47F1K68QyulLl/rcUWHZAB9D7j5niz7hWoJYiNUFosNB4kQoHxLYjkgDuVx0Y3MR7xlQm4Qncc//er80+dNX/QujIG3fwbYEjjZ8EJ4yelrYtPt8BvC6Z5FMWF0AOQtqbxPGr/xtZGQnCqgqBKbNbknzW6LQgj0zR53MfjZ8a7dLdGkLYUyrAtCGsRKAHxoqgXf2ARcoiQ9xFlH7zQwo9NPwT924er89tb2AhdI/gKC7Xc5iXKNjP4UeA5/WFKFWp+2GB0P4/4SdgENdLnc/jt7h+fPw47TEZOwXx+d09AdoWVY729axKLR6D3k9/KcDA1P5XovVihP4D6/P4OkNrHPj8/tbD/1fP4vA5r61QQjHtu/AM6WKjA6Hs+pyBQ6OPa6yRjiovGMvP36DboaH0KAyLhbBu3ugtZmTybnnOVj6l13827D+P1C7l552pTsERk5wbcSC7Cg/AQ4tMo9wncXEwb5xssaseC6QK9cqRkVCC/BThkvxsGqtFbDblCKzShHd8HYQLpLra5iUOtihwyyh9aFzcTo6Rlf56LbZd/P0qGHOBZbkymyPB1VVnXc4R4K/SzfeqCQ9NPKxoeuTA49LZ2BXYf9Z3g47pFMAOBd3flR5dIgi5Esaa1QXLAbYRuErXw3haK1fvNjU8mFnH/xL9Hh+N2ZiDZIJU7lM+l55uCySsgW89k4oyxiX2JYoPQ86H5OUkC0SCxCTB1uYh7q2aiHBbWyWaOHFZ6CwMqiSnQrwpfhEIK9X0mXjzM99XNSfPyf53ZWNZad6ZkLTwsECkx4kBWJFDm9Uyg0ccjoOsQqiRNJVZhPX7Ib7By6JHSGEFXqkKtDEpge2TDxszX6Oy4bni7yfnyTZN3Oip3P0KZKu3daj8v/Wp8/rjld5vk5jDUzjy9MhhXmaWd72mRkwboSvjAIw8/Pj4CGXIlFzCh0/ywHBimMzvGZ9IoPE3n+ADNhgJIrE7O5391lFLK5qnMWMcpdSOcsrUHEYJTi5rSm1S+sBukYhu3I5qzbbAgoBIuqKLWwrVlaS+wKqGI+D7zSnjfYTpiHD+nN2njuhBeFa9JGZFIqtDRoKLsCD/mM4BfiCrUDuV8kMhJC/8CA7pSGfREhddgpvagtIYFla6I0sqKw4wqZx8VTumuaZ0r4hqFpIrHqqEU20ZIZIa/3t19buQSlamMus/HCbPCeeY7enTaGqn6WQcKErPe1yXC2x9+inEkNOWpQBJNmQPb3jhvrOPm0VHeKUxYAFStNlgEy3WKxyQaZCQPlFXz4ouAM/hk8zO8f/Bt0AtHgq3QtUQJa3T4Cv6ofehMSDQ7tIDj0n5Bd1IueQZ3BrzhttmyVnKCyDw3YJ3kVNP6slQac+/48iL1sPuNtEM0SlXinPCfiwsZZCOjaJAHo6b1hBGJs3oaCW4gvsiUNC3FQyxeZEaeZPECE3KMQxubE2Xfh9o5Ws+tzr6+vP4IaIIKtB1PIruX6LOwWFi5PTo7L8JpZ6+7T4qG1M8oaCfMUS9AD3VsI0J4nK/tTm1T1dMbHXopbcxJNy/RUUiw/le7PfSI5L4ayWGBavMETHtGN3mXzwaSgFiLDcbSfRAwbvAmSNC2m56aSvlkJlpM/+O5lCKIGXycrWagoqpZIAgwGB6suycFE2xhdVvBSLHCw1oVa3ohtqOaeXoF1oGI52zRfiXCOj0eN+4iPuRtiWB5b9eYVqaqQyI5NCD7C+hQ27Qrx2ObNFjaB8Q9Xezv8UYub9qYK4sjHUcsWFDBJzIzuDRwe333+RX53PL1u5Ywrk0RgijWJUFZ023fsSFaqWxo6P1dtQ8PKqzTEr/78Lmz4RwaIWGE3n6Z1Idq9i1swxM3Tv7SGgRZu9xY55HKzwxxaCrReA7XueZ0Q3QwfaW6MB7ikyj7C+DwCV2EGn9E3gbf5f6ya2LNd1ZZO6mv8iOoZadRmTYYtFg7K7E9X2+qY1x52a51YGwY8kr578Sl/GYDsECKtK6qbiVYnrhu18UdUwUpb9SuvX4xtkGaX27ZUh5LC433nqT3U07Jz/ZPwmfD5QTNpJsGnxodEC8bVM5ulIyNg70FCmhWyrSHNYcWVOxQTqFzt9OYjKS4xNFmh5voPrIrbFk55bHt5h2iUyrvlVmNINTfsO3SUZ3p88GhKCPDB3TItDQG1FuIF0C66a5ytkDfic9OhhxiTuVxqe3Dy1A3NoBErWIXKdj8RBsDVAxm0EFQHgpba0l1nshqUVXkbESzrjuB/KD5r0DPOhTeY7nQu/2MFECqRGnrCb2MXou3F7tsE2wdQATQKDyV0+LpYK4ECeb5GI3bSXDtLarOEQhVYRDcRhHKoItMOe1GccM5jkolZQd8DE7w+YTw3IKh6WwQTqui8t2kPdT1xTuQakXKoVvlUv9nEMqvxY9T9MLtr+c/TsB6++6niWhv3/10DG8wzTZTMR7zqlGxJomGdl4PhV7zxLyog10uxy+H3NrrnX6xn7Eythwo0oqaxmIJa6SFQemg/b3Na8eZevVlymWE3UTVAgcLUvn7QVQTnK22kypxp6sPEo1XYdsRBBwCh3Wj93q2QT4kHh8Jt7dXb+6ubiEZSAlYeNB2tdrNh4RUqGqNE+6UZaD4PvhaBTwBrXabCVL4o9b0xwL4/V0Yij8DNZVh+Pjh4tc39M/HffDUZB2jxttk+zvvzdKk3cZjd1bol/nwpBkSfAw059bM4DL0zvCbqxz9U/zd2wlx+xnvcJjVviMOfV2efmDdOvGLFiveL+4cB+QrqBGYjYPDOKL0+z1uoRQBnRIa8LFYC0PjrgwI07qBwmkqRoc2fkTd4GOY5636hD0RPoZ2p98Z0GJtPULLWFSVzi2FePPR9F5t5mifIvogFlr59ZQrAcMjrDxZboa5OQ/uQIGvC5JvJDa3AwsHXZgXa6Em5IQP9BqFLhmJnRbST8slS7HegQBR3witJFMPvsnM4NXK0OBGDkf4PWuXucM47b8azdPeXX7arQG+8fzomw9rWtGTh3WX5vc1upHePm3lfT2pqPSvntLgtJ8I/P/s3Q//3eM8bq6fidSTNA4NSgoAZc083qIYT+UGfa3D7tS1htvz5WM5uENk70bFN6PxClZqg4a2Gf9bobm9veq+wDz2C3Yc+W8/ZU9EzovxOBQ6WvgwFxrdBA5834TfbXUTXz/ptA4ORYuxQRU4a3+edH/+cu9rn3yBpNMdG/wUI+GroqyeGfnyw/Xno5iUD2eTTp66HwgMbMxJouiUb/O1+86FGl42kcNhXqlt8JVD8vVNg8MM09HIifT+kw+EuqPyFQcO5+0xd1bt0fJO/IiBHWUvpP8e7fReLH4v/esuqe+mjd2Pz3+/ZnaX/9+vpZ29f4ZLEpcXzWZ1bGYxdsrHoXc7nyAdqXqTbtryzYm6FIZTN39qkS+b5EWazA9i+nrxLJi+XrwevOWeB2/k+j9vy3ZhaxNS4Yp7igElnWAqHNn72j+JdljagECW0m0UoTjTPKFZGJpt97ADPoYnRhMfQy109+dm6kaziPfaJ96ISS8nOcA3YriDw32m05QblkLp+cLK7Tx9leVPGo42Q/28BSFlbBfU5j72kPEx5Ev+VIQ0lsgxwfokmvF81xYfK2EkWJMI+Z0GJBWP1LgnpgcyTfRCohbbebD3ePqAdkRedIJiNiZ8stMMoccQL04Xok6la1ErHV4rE5nRK8vaFPHSvArbjiMWmBrJCJlrCb8EqJoziojXdOEorDcos6KIjzcmm8fYcOdSef70Tcavdpuj+8EdTHsBbUL4tSKqo0G46S06qXqFBl28Y5GXSO1IClCYSJR16oiaVQ6AA2mvqhx6P1/aPtnTDj8GBB/f9kCzSleHVP7emGc7FZ/khl/n6phpHAhD6WxVfc03W03rO+rZy8/Np/Gk9ZN5/tpBoqFJzvpWcJs2kjo7O/tXAAAA//+iJmrR" } diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json index 42b6aae2c32..84e96dbd912 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json @@ -4,16 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * params.multiplier; ctx.zeek.files.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * 1000; ctx.zeek.files.remove('ts');" } }, { "script": { "lang": "painless", "source": "ctx.zeek.session_id = ctx.zeek.files.session_ids[0];", + "if": "ctx.zeek.files.session_ids != null", "ignore_failure": true } }, @@ -32,10 +30,10 @@ } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-files\"", - "ignore_failure": true + "set": { + "field": "event.id", + "value": "{{zeek.session_id}}", + "if": "ctx.zeek.session_id != null" } } ] diff --git a/x-pack/filebeat/module/zeek/files/manifest.yml b/x-pack/filebeat/module/zeek/files/manifest.yml index 9da593ea2ed..1d9ac220761 100644 --- a/x-pack/filebeat/module/zeek/files/manifest.yml +++ b/x-pack/filebeat/module/zeek/files/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/files.log - name: tags - default: [zeek] + default: [zeek.files] ingest_pipeline: ingest/pipeline.json input: config/files.yml diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json index c5d2d872e2f..c1f0c949f21 100644 --- a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json @@ -3,14 +3,14 @@ "@timestamp": 1547688796000, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.files", - "event.id": "C8I0zn3r9EPbfLgta6-files", + "event.id": "C8I0zn3r9EPbfLgta6", "event.module": "zeek", "fileset.name": "files", "input.type": "log", "log.offset": 0, "service.type": "zeek", "tags": [ - "zeek" + "zeek.files" ], "zeek.files.analyzers": [ "X509", @@ -41,14 +41,14 @@ "@timestamp": 1547688801000, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.files", - "event.id": "C6sjVo23iNApLnlAt6-files", + "event.id": "C6sjVo23iNApLnlAt6", "event.module": "zeek", "fileset.name": "files", "input.type": "log", "log.offset": 452, "service.type": "zeek", "tags": [ - "zeek" + "zeek.files" ], "zeek.files.analyzers": [ "X509", diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json index 93222421916..a892d959ce5 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json @@ -4,17 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * params.multiplier; ctx.zeek.http.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * 1000; ctx.zeek.http.remove('ts');" } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-http\"", - "ignore_failure": true + "set": { + "field": "event.id", + "value": "{{zeek.session_id}}", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/http/manifest.yml b/x-pack/filebeat/module/zeek/http/manifest.yml index 6ee2cadec4c..e98068206ee 100644 --- a/x-pack/filebeat/module/zeek/http/manifest.yml +++ b/x-pack/filebeat/module/zeek/http/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/http.log - name: tags - default: [zeek] + default: [zeek.http] ingest_pipeline: ingest/pipeline.json input: config/http.yml diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json index 9d99db4f00f..075b2e2cd02 100644 --- a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json @@ -10,7 +10,7 @@ "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.http", - "event.id": "CCNp8v1SNzY7v9d1Ih-http", + "event.id": "CCNp8v1SNzY7v9d1Ih", "event.module": "zeek", "fileset.name": "http", "http.request.body.bytes": 0, @@ -25,7 +25,7 @@ "source.ip": "10.178.98.102", "source.port": 62995, "tags": [ - "zeek" + "zeek.http" ], "url.domain": "ocsp.apple.com", "url.original": "/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=", diff --git a/x-pack/filebeat/module/zeek/module.yml b/x-pack/filebeat/module/zeek/module.yml index 0db59890087..4d55536c0c6 100644 --- a/x-pack/filebeat/module/zeek/module.yml +++ b/x-pack/filebeat/module/zeek/module.yml @@ -1,3 +1,3 @@ dashboards: -- id: 87b0c430-1a2d-11e9-84b1-a12c578fa9e8 +- id: 7cbb5410-3700-11e9-aa6d-ff445a78330c file: Filebeat-Zeek-Overview.json diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml new file mode 100644 index 00000000000..c722a1b8c2f --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -0,0 +1,80 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +tags: {{.tags}} + +json.keys_under_root: false + +processors: + - drop_fields: + fields: ["json.actions"] + - rename: + fields: + - from: "json" + to: "zeek.notice" + + - from: "zeek.notice.src" + to: "source.address" + + - from: "zeek.notice.dest" + to: "destination.address" + + - from: "zeek.notice.uid" + to: "zeek.session_id" + + - from: "zeek.notice.p" + to: "destination.port" + + - from: "zeek.notice.conn" + to: "zeek.notice.connnection_id" + + - from: "zeek.notice.iconn" + to: "zeek.notice.icmp_id" + + - from: "zeek.notice.id.orig_h" + to: "source.address" + + - from: "zeek.notice.id.orig_p" + to: "source.port" + + - from: "zeek.notice.id.resp_h" + to: "destination.address" + + - from: "zeek.notice.id.resp_p" + to: "destination.port" + + - from: "zeek.notice.proto" + to: "network.transport" + + - from: "zeek.notice.id.orig_p" + to: "source.port" + + - from: "zeek.notice.f.id" + to: "zeek.notice.file.id" + + - from: "zeek.notice.f.parent_id" + to: "dzeek.notice.file.parent_id" + + - from: "zeek.notice.f.source" + to: "zeek.notice.file.source" + + - from: "zeek.notice.f.is_orig" + to: "zeek.notice.file.is_orig" + + - from: "zeek.notice.f.seen_bytes" + to: "zeek.notice.file.seen_bytes" + + - from: "zeek.notice.f.total_bytes" + to: "zeek.notice.file.total_bytes" + + - from: "zzeek.notice.file_mime_type" + to: "zeek.notice.file.mime_type" + + ignore_missing: true + fail_on_error: false + + - drop_fields: + fields: ["zeek.notice.remote_location", "zeek.notice.f"] diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json new file mode 100644 index 00000000000..1b1bf8b49af --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json @@ -0,0 +1,46 @@ +{ + "description": "Pipeline for normalizing Zeek notice.log", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * 1000; ctx.zeek.notice.remove('ts');" + } + }, + { + "set": { + "field": "destination.ip", + "value": "{{destination.address}}", + "if": "ctx.destination?.address != null" + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}", + "if": "ctx.source?.address != null" + } + }, + { + "set": { + "field": "event.id", + "value": "{{zeek.session_id}}", + "if": "ctx.zeek.session_id != null" + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + } + ] +} diff --git a/x-pack/filebeat/module/zeek/notice/manifest.yml b/x-pack/filebeat/module/zeek/notice/manifest.yml new file mode 100644 index 00000000000..b806ac04e1d --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/bro/current/notice.log + os.linux: + - /var/log/bro/current/notice.log + os.darwin: + - /usr/local/var/logs/current/notice.log + - name: tags + default: [zeek.notice] + +ingest_pipeline: ingest/pipeline.json +input: config/notice.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log b/x-pack/filebeat/module/zeek/notice/test/notice-json.log new file mode 100644 index 00000000000..8c20486cb79 --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log @@ -0,0 +1 @@ +{"ts":1320435875.879278,"note":"SSH::Password_Guessing","msg":"172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).","sub":"Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136","src":"172.16.238.1","peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false} diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json new file mode 100644 index 00000000000..aab984d1d36 --- /dev/null +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json @@ -0,0 +1,23 @@ +[ + { + "@timestamp": 1320435875000, + "ecs.version": "1.0.0-beta2", + "event.dataset": "zeek.notice", + "event.module": "zeek", + "fileset.name": "notice", + "input.type": "log", + "log.offset": 0, + "service.type": "zeek", + "source.address": "172.16.238.1", + "source.ip": "172.16.238.1", + "tags": [ + "zeek.notice" + ], + "zeek.notice.dropped": false, + "zeek.notice.msg": "172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).", + "zeek.notice.note": "SSH::Password_Guessing", + "zeek.notice.peer_descr": "bro", + "zeek.notice.sub": "Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136", + "zeek.notice.suppress_for": 3600 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json index de32cf75099..54d068b19f9 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.json @@ -4,17 +4,14 @@ { "script": { "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['ssl']['ts'] * params.multiplier; ctx.zeek.ssl.remove('ts');", - "params": { - "multiplier": 1000 - } + "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['ssl']['ts'] * 1000; ctx.zeek.ssl.remove('ts');" } }, { - "script": { - "lang": "painless", - "source": "ctx.event.id = ctx.zeek.session_id + \"-ssl\"", - "ignore_failure": true + "set": { + "field": "event.id", + "value": "{{zeek.session_id}}", + "if": "ctx.zeek.session_id != null" } }, { diff --git a/x-pack/filebeat/module/zeek/ssl/manifest.yml b/x-pack/filebeat/module/zeek/ssl/manifest.yml index d403fa97311..74d9c46134f 100644 --- a/x-pack/filebeat/module/zeek/ssl/manifest.yml +++ b/x-pack/filebeat/module/zeek/ssl/manifest.yml @@ -9,7 +9,7 @@ var: os.darwin: - /usr/local/var/logs/current/ssl.log - name: tags - default: [zeek] + default: [zeek.ssl] ingest_pipeline: ingest/pipeline.json input: config/ssl.yml diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json index 3ef9fd2bb8d..6a034c1d938 100644 --- a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json @@ -13,7 +13,7 @@ "destination.port": 9243, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.ssl", - "event.id": "CAOvs1BMFCX2Eh0Y3-ssl", + "event.id": "CAOvs1BMFCX2Eh0Y3", "event.module": "zeek", "fileset.name": "ssl", "input.type": "log", @@ -23,7 +23,7 @@ "source.ip": "10.178.98.102", "source.port": 63199, "tags": [ - "zeek" + "zeek.ssl" ], "zeek.session_id": "CAOvs1BMFCX2Eh0Y3", "zeek.ssl.cert_chain_fuids": [ @@ -56,7 +56,7 @@ "destination.port": 9243, "ecs.version": "1.0.0-beta2", "event.dataset": "zeek.ssl", - "event.id": "C3mki91FnnNtm0u1ok-ssl", + "event.id": "C3mki91FnnNtm0u1ok", "event.module": "zeek", "fileset.name": "ssl", "input.type": "log", @@ -66,7 +66,7 @@ "source.ip": "10.178.98.102", "source.port": 63198, "tags": [ - "zeek" + "zeek.ssl" ], "zeek.session_id": "C3mki91FnnNtm0u1ok", "zeek.ssl.cert_chain_fuids": [ diff --git a/x-pack/filebeat/modules.d/zeek.yml.disabled b/x-pack/filebeat/modules.d/zeek.yml.disabled index d296ebc3d84..eb2a01e2eaa 100644 --- a/x-pack/filebeat/modules.d/zeek.yml.disabled +++ b/x-pack/filebeat/modules.d/zeek.yml.disabled @@ -13,6 +13,8 @@ enabled: true ssl: enabled: true + notice: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS.