[conf] Use Common Event Format (CEF) standard

* Use CEF stardard to define normalized types.

conf/types.json

@@ -1,65 +1,69 @@
 {
   "carbonblack":{
-    "username": ["username"],
-    "domain": ["domain"],
-    "path": ["parent_path", "process_path", "path"],
-    "protocol": ["protocol"],
-    "vend": ["feed_name"],
-    "process": ["parent_name", "process_name"],
-    "name": ["observed_filename", "file_path"],
-    "cmd": ["cmdline"],
-    "hashmd5": ["process_md5", "parent_md5", "expect_followon_w_md5", "md5"],
-    "score": ["report_score"],
-    "os": ["host_type", "os_type"],
-    "ipv4": ["ipv4", "comms_ip", "interface_ip", "remote_ip", "local_ip"],
-    "port": ["port", "remote_port", "local_port"],
-    "host": ["other_hostnames", "server_name", "hostname", "computer_name"]
+    "userName": ["username"],
+    "destinationDomain": ["domain"],
+    "processPath": ["parent_path", "process_path", "path"],
+    "filePath": ["path"],
+    "transportProtocol": ["protocol"],
+    "processName": ["parent_name", "process_name"],
+    "fileName": ["observed_filename", "file_path"],
+    "command": ["cmdline"],
+    "fileHash": ["process_md5", "parent_md5", "expect_followon_w_md5", "md5"],
+    "deviceAddress": ["interface_ip", "comms_ip"],
+    "sourceAddress": ["ipv4", "local_ip"],
+    "destinationAddress": ["remote_ip"],
+    "sourcePort": ["port", "local_port"],
+    "destinationPort": ["remote_port"]
   },
   "cloudwatch":{
-    "username": ["userName", "owner", "invokedBy"],
-    "account": ["account", "recipientAccountId"],
-    "protocol": ["protocol"],
-    "event_type": ["eventType"],
-    "event_name": ["eventName"],
+    "userName": ["userName", "owner", "invokedBy"],
+    "sourceAccount": ["account"],
+    "destinationAccount": ["recipientAccountId"],
+    "transportProtocol": ["protocol"],
+    "eventType": ["eventType"],
+    "eventName": ["eventName"],
     "region": ["region"],
-    "agent": ["userAgent"],
-    "ipv4": ["destination", "source", "sourceIPAddress"],
-    "port": ["srcport", "destport"]
+    "userAgent": ["userAgent"],
+    "sourceAddress": ["source", "sourceIPAddress"],
+    "destinationAddress": ["destination"],
+    "sourcePort": ["srcport"],
+    "destinationPort": ["destport"]
   },
   "cloudtrail": {
-    "account": ["account", "recipientAccountId", "accountId"],
-    "event_type": ["eventType"],
-    "event_name": ["eventName"],
+    "sourceAccount": ["account", "accountId"],
+    "destinationAccount": ["recipientAccountId"],
+    "eventType": ["eventType"],
+    "eventName": ["eventName"],
     "region": ["region", "awsRegion"],
-    "user_type": ["type"],
-    "agent": ["userAgent"],
-    "ipv4": ["sourceIPAddress"]
+    "userAgent": ["userAgent"],
+    "sourceAddress": ["sourceIPAddress"]
   },
   "ghe": {
-    "process": ["program"],
-    "username": ["current_user"],
-    "ipv4": ["remote_address"],
-    "port": ["port"],
-    "host": ["host"]
+    "processName": ["program"],
+    "userName": ["current_user"],
+    "destinationAddress": ["remote_address"],
+    "sourcePort": ["port"]
   },
   "osquery": {
-    "username": ["username", "user"],
-    "path": ["path"],
-    "protocol": ["protocol"],
-    "sev": ["severity"],
-    "cluster": ["envIdentifier"],
-    "role": ["roleIdentifier"],
-    "cmd": ["cmdline", "command"],
-    "msg": ["message"],
-    "ipv4": ["destination", "remote_address", "host", "source", "local_address", "gateway", "address"],
-    "port": ["local_port", "remote_port", "port"],
-    "host": ["hostIdentifier"]
+    "userName": ["username", "user"],
+    "filePath": ["path"],
+    "transportProtocol": ["protocol"],
+    "severity": ["severity"],
+    "environmentIdentifier": ["envIdentifier"],
+    "roleIdentifier": ["roleIdentifier"],
+    "command": ["cmdline", "command"],
+    "message": ["message"],
+    "sourceAddress": ["host", "source", "local_address", "address"],
+    "destinationAddress": ["destination", "remote_address", "gateway"],
+    "sourcePort": ["local_port", "port"],
+    "destinationPort": ["remote_port"]
   },
   "pan": {
-    "username": ["srcuser", "dstuser"],
-    "protocol": ["proto"],
-    "ipv4": ["src", "natsrc", "dst", "natdst"],
-    "port": ["dport", "sport", "natsport", "natdport"],
-    "host": ["sourceName"]
+    "userName": ["srcuser", "dstuser"],
+    "transportProtocol": ["proto"],
+    "sourceAddress": ["src", "natsrc"],
+    "destinationAddress": ["dst", "natdst"],
+    "sourcePort": ["sport", "natsport"],
+    "destinationPort": ["dport", "natdport"]
   }
 }

tests/unit/conf/types.json

@@ -1,13 +1,15 @@
 {
   "cloudwatch":{
-    "username": ["userName", "owner", "invokedBy"],
-    "account": ["account", "recipientAccountId"],
-    "protocol": ["protocol"],
-    "event_type": ["eventType"],
-    "event_name": ["eventName"],
+    "userName": ["userName", "owner", "invokedBy"],
+    "sourceAccount": ["account", "recipientAccountId"],
+    "transportProtocol": ["protocol"],
+    "eventType": ["eventType"],
+    "eventName": ["eventName"],
     "region": ["region"],
-    "agent": ["userAgent"],
-    "ipv4": ["destination", "source", "sourceIPAddress"],
-    "port": ["srcport", "destport"]
+    "userAgent": ["userAgent"],
+    "sourceAddress": ["source", "sourceIPAddress"],
+    "destinationAddress": ["destination"],
+    "sourcePort": ["srcport"],
+    "destinationPort": ["destport"]
   }
 }

tests/unit/stream_alert_rule_processor/test_rules_engine.py

@@ -476,10 +476,10 @@ def test_match_types(self):
         """Rules Engine - Match normalized types against record"""
         @rule(logs=['cloudwatch:test_match_types'],
               outputs=['s3:sample_bucket'],
-              datatypes=['ipv4'])
+              datatypes=['sourceAddress'])
         def match_ipaddress(rec): # pylint: disable=unused-variable
             """Testing rule to detect matching IP address"""
-            results = fetch_values_by_datatype(rec, 'ipv4')
+            results = fetch_values_by_datatype(rec, 'sourceAddress')
 
             for result in results:
                 if result == '1.1.1.2':
@@ -488,12 +488,12 @@ def match_ipaddress(rec): # pylint: disable=unused-variable
 
         @rule(logs=['cloudwatch:test_match_types'],
               outputs=['s3:sample_bucket'],
-              datatypes=['ipv4', 'cmd'])
+              datatypes=['sourceAddress', 'command'])
         def mismatch_types(rec): # pylint: disable=unused-variable
             """Testing rule with non-existing normalized type in the record. It
             should not trigger alert.
             """
-            results = fetch_values_by_datatype(rec, 'ipv4')
+            results = fetch_values_by_datatype(rec, 'sourceAddress')
 
             for result in results:
                 if result == '2.2.2.2':