Add system tests for CEF

packages/cef/_dev/deploy/docker/docker-compose.yml

@@ -1,8 +1,13 @@
 version: '2.3'
 services:
-  cef:
-    tty: true
-    build: .
+  cef-log-logfile:
+    image: alpine
     volumes:
+      - ./sample_logs:/sample_logs:ro
       - ${SERVICE_LOGS_DIR}:/var/log
-    command: -c "cp /cef.log /var/log/"
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+  cef-log-syslog:
+    image: akroh/stream:v0.0.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9515 -p=udp /sample_logs/cef.log

packages/cef/_dev/deploy/docker/sample_logs/cef.log

@@ -0,0 +1 @@
+CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=6.7.8.9 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart

packages/cef/_dev/deploy/docker/sample_logs/checkpoint.log

@@ -0,0 +1,3 @@
+CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\=R80,O\=R80_M..6u6bdo sequencenum=1 version=5 dst=52.173.84.157 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 & FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up
+CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration
+CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds

packages/cef/_dev/deploy/docker/sample_logs/forcepoint-ngfw-smc.log

@@ -0,0 +1,10 @@
+CEF:0|FORCEPOINT|Firewall|6.6.1|0|Generic|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=log server connection established deviceFacility=Logging System rt=Jan 17 2020 08:52:10
+CEF:0|FORCEPOINT|Firewall|6.6.1|9005|FW_Communication-Communication-Error|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=Communication error: No route to host (-3, 5, 0) deviceFacility=Management rt=Jan 17 2020 08:52:09
+CEF:0|FORCEPOINT|Firewall|6.6.1|70018|Connection_Allowed|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 src=10.37.205.252 dst=10.1.1.40 proto=1 deviceOutboundInterface=255 act=Allow msg=Referred connection: 10.1.1.40 -> 10.37.133.35 frag\=0x4000 TCP 47413->3020 deviceFacility=Packet Filtering rt=Jan 17 2020 08:52:09 app=Dest. Unreachable (Host Unreachable) cs1Label=RuleID cs1=2097157.1
+CEF:0|FORCEPOINT|Firewall|unknown|70019|Connection_Discarded|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=172.16.1.1 dst=255.255.255.255 spt=68 dpt=67 proto=17 deviceOutboundInterface=255 deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:21 app=BOOTPS (UDP) cs1Label=RuleID cs1=605.0
+CEF:0|FORCEPOINT|Firewall|unknown|70020|Connection_Refused|0|deviceExternalId=Firewall-1 node 1 dvc=10.1.1.1 dvchost=10.1.1.1 src=172.16.1.1 dst=192.168.1.1 proto=1 deviceOutboundInterface=255 act=Refuse deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:23 app=Echo Request (No Code) cs1Label=RuleID cs1=601.0
+CEF:0|FORCEPOINT|Firewall|unknown|70021|Connection_Closed|0|deviceExternalId=Firewall-6 node 1 dvc=10.1.1.6 dvchost=10.1.1.6 proto=6 deviceOutboundInterface=255 destinationServiceName=YouTube suser=alice deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:20 app=TCP in=32526 out=27366
+CEF:0|FORCEPOINT|Firewall|unknown|72714|ECA_Metadata_login|0|deviceExternalId=Firewall-3 node 1 dvc=10.1.1.3 dvchost=10.1.1.3 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:33
+CEF:0|FORCEPOINT|Firewall|unknown|72715|ECA_Metadata_logout|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:31
+CEF:0|FORCEPOINT|Firewall|unknown|72716|ECA_Metadata_system_metadata_received|0|deviceExternalId=Firewall-8 node 1 dvc=10.1.1.8 dvchost=10.1.1.8 src=172.16.2.1 suser=alice deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:26
+CEF:0|FORCEPOINT|Firewall|6.6.1|78002|TLS connection state|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=TLS: Couldn't establish TLS connection (11, N/A) deviceFacility=Management rt=Jan 17 2020 08:52:09

packages/cef/data_stream/log/_dev/test/pipeline/test-cef-event.json-expected.json

@@ -586,7 +586,7 @@
         {
             "checkpoint": {
                 "severity": "Very-High",
-                "event_count": "12",
+                "event_count": 12,
                 "app_risk": "High"
             },
             "agent": {

packages/cef/data_stream/log/_dev/test/system/test-logfile-config.yml

@@ -0,0 +1,6 @@
+service: cef-log-logfile
+input: logfile
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/*.log"

packages/cef/data_stream/log/_dev/test/system/test-syslog-config.yml

@@ -0,0 +1,7 @@
+service: cef-log-syslog
+service_notify_signal: SIGHUP
+input: syslog
+data_stream:
+  vars:
+    syslog_host: 0.0.0.0
+    syslog_port: 9515

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml

@@ -337,3 +337,8 @@ processors:
       field: event.category
       value: intrusion_detection
       if: 'ctx.event?.category != "malware" && (ctx.checkpoint?.protection_type != null || ctx.cef.extensions?.flexString2Label == "Attack Information")'
+
+  - convert:
+      field: checkpoint.event_count
+      type: long
+      ignore_missing: true

packages/cef/data_stream/log/fields/agent.yml

@@ -196,3 +196,6 @@
       description: >
         OS codename, if any.
 
+- name: log.source.address
+  type: keyword
+  description: Source address from which the log event was read / sent from.

packages/cef/data_stream/log/fields/ecs.yml

@@ -386,3 +386,15 @@
       norms: false
       default_field: false
   description: 'Unmodified original url as seen in the event source.'
+- name: file.group
+  type: keyword
+  description: Primary group name of the file.
+- name: file.inode
+  type: keyword
+  description: Inode representing the file in the filesystem.
+- name: file.type
+  type: keyword
+  description: File type (file, dir, or symlink).
+- name: user_agent.original
+  type: keyword
+  description: Unparsed user_agent string.

packages/cef/data_stream/log/fields/fields.yml

@@ -18,16 +18,6 @@
 - name: checkpoint
   type: group
   fields:
-    - name: app_risk
-      type: keyword
-    - name: email_control
-      type: keyword
-    - name: event_count
-      type: keyword
-    - name: severity
-      type: keyword
-    - name: subs_exp
-      type: keyword
     - name: app_risk
       type: keyword
       description: Application risk.

packages/cef/docs/README.md

@@ -310,7 +310,7 @@ An example event for `log` looks as following:
 | cef.name |  | keyword |
 | cef.severity |  | keyword |
 | cef.version |  | keyword |
-| checkpoint.app_risk |  | keyword |
+| checkpoint.app_risk | Application risk. | keyword |
 | checkpoint.app_severity | Application threat severity. | keyword |
 | checkpoint.app_sig_id | The signature ID which the application was detected by. | keyword |
 | checkpoint.auth_method | Password authentication protocol used. | keyword |
@@ -319,7 +319,7 @@ An example event for `log` looks as following:
 | checkpoint.connectivity_state | Connectivity state. | keyword |
 | checkpoint.cookie | IKE cookie. | keyword |
 | checkpoint.dst_phone_number | Destination IP-Phone. | keyword |
-| checkpoint.email_control |  | keyword |
+| checkpoint.email_control | Engine name. | keyword |
 | checkpoint.email_id | Internal email ID. | keyword |
 | checkpoint.email_recipients_num | Number of recipients. | long |
 | checkpoint.email_session_id | Internal email session ID. | keyword |
@@ -340,7 +340,7 @@ An example event for `log` looks as following:
 | checkpoint.protection_type | Type of protection used to detect the attack. | keyword |
 | checkpoint.scan_result | Scan result. | keyword |
 | checkpoint.sensor_mode | Sensor mode. | keyword |
-| checkpoint.severity |  | keyword |
+| checkpoint.severity | Threat severity. | keyword |
 | checkpoint.spyware_name | Spyware name. | keyword |
 | checkpoint.spyware_status | Spyware status. | keyword |
 | checkpoint.subs_exp | The expiration date of the subscription. | date |
@@ -389,8 +389,11 @@ An example event for `log` looks as following:
 | destination.user.name | Short name or login of the user. | keyword |
 | ecs.version | ECS version | keyword |
 | event.ingested | Timestamp when an event arrived in the central data store. | date |
+| file.group | Primary group name of the file. | keyword |
 | file.hash.md5 | MD5 hash. | keyword |
 | file.hash.sha1 | SHA1 hash. | keyword |
+| file.inode | Inode representing the file in the filesystem. | keyword |
+| file.type | File type (file, dir, or symlink). | keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
 | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword |
@@ -412,6 +415,7 @@ An example event for `log` looks as following:
 | input.type | Input type | keyword |
 | log.file.path | Log path | keyword |
 | log.offset | Log offset | long |
+| log.source.address | Source address from which the log event was read / sent from. | keyword |
 | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text |
 | network.application | A name given to an application level protocol. | keyword |
 | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. | keyword |
@@ -455,4 +459,5 @@ An example event for `log` looks as following:
 | source.user.name | Short name or login of the user. | keyword |
 | tags | List of keywords used to tag each event. | keyword |
 | url.original | Unmodified original url as seen in the event source. | keyword |
+| user_agent.original | Unparsed user_agent string. | keyword |
 

packages/cef/manifest.yml

@@ -1,6 +1,6 @@
 name: cef
 title: CEF
-version: 0.0.1
+version: 0.0.2
 release: experimental
 description: CEF Integration
 type: integration