enhancement(remap transform): add parse_aws_alb_log function (#5489)

Signed-off-by: Kirill Fomichev <fanatid@ya.ru>
vectordotdev · Dec 18, 2020 · 6389e88 · 6389e88
1 parent 9f760f2
commit 6389e88
Show file tree

Hide file tree

Showing 7 changed files with 403 additions and 0 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -75,6 +75,7 @@
 /docs/reference/remap/log.cue @FungusHumungus
 /docs/reference/remap/merge.cue @FungusHumungus
 /docs/reference/remap/parse_grok.cue @FungusHumungus
+/docs/reference/remap/parse_aws_alb_log.cue @fanatid
 
 /distribution/ @hoverbear @jamtur01
 /distribution/docker/ @vector-kubernetes
@@ -101,6 +102,7 @@
 /lib/remap-functions/src/ipv6_to_ipv4.rs @FungusHumungus
 /lib/remap-functions/src/log.rs @FungusHumungus
 /lib/remap-functions/src/merge.rs @FungusHumungus
+/src/remap/function/parse_aws_alb_log.rs @fanatid
 /lib/remap-functions/src/parse_grok.rs @FungusHumungus
 
 /proto/ @lukesteensen

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/docs/reference/remap/parse_aws_alb_log.cue b/docs/reference/remap/parse_aws_alb_log.cue
@@ -0,0 +1,76 @@
+package metadata
+
+remap: functions: parse_aws_alb_log: {
+	arguments: [
+		{
+			name:        "value"
+			description: "Access log of the Application Load Balancer."
+			required:    true
+			type: ["string"]
+		},
+	]
+	return: ["map"]
+	category: "parse"
+	description: #"""
+		Parses a Elastic Load Balancer Access log into it's constituent components.
+		"""#
+	examples: [
+		{
+			title: "Success"
+			input: {
+				log: #"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""#
+			}
+			source: #"""
+				.parsed = parse_aws_alb_log(.log)
+				"""#
+			output: {
+				log: #"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""#
+				parsed: {
+					"type":                     "http"
+					"timestamp":                "2018-11-30T22:23:00.186641Z"
+					"elb":                      "app/my-loadbalancer/50dc6c495c0c9188"
+					"client_host":              "192.168.131.39:2817"
+					"target_host":              null
+					"request_processing_time":  0.0
+					"target_processing_time":   0.001
+					"response_processing_time": 0.0
+					"elb_status_code":          "200"
+					"target_status_code":       "200"
+					"received_bytes":           34
+					"sent_bytes":               366
+					"request_method":           "GET"
+					"request_url":              "http://www.example.com:80/"
+					"request_protocol":         "HTTP/1.1"
+					"user_agent":               "curl/7.46.0"
+					"ssl_cipher":               null
+					"ssl_protocol":             null
+					"target_group_arn":         "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067"
+					"trace_id":                 "Root=1-58337364-23a8c76965a2ef7629b185e3"
+					"domain_name":              null
+					"chosen_cert_arn":          null
+					"matched_rule_priority":    "0"
+					"request_creation_time":    "2018-11-30T22:22:48.364000Z"
+					"actions_executed":         "forward"
+					"redirect_url":             null
+					"error_reason":             null
+					"target_port_list": []
+					"target_status_code_list": []
+					"classification":        null
+					"classification_reason": null
+				}
+			}
+		},
+		{
+			title: "Error"
+			input: {
+				log: "I am not a log"
+			}
+			source: #"""
+				.parsed = parse_aws_alb_log(.log)
+				"""#
+			output: {
+				error: remap.errors.ParseError
+			}
+		},
+	]
+}
diff --git a/lib/remap-functions/Cargo.toml b/lib/remap-functions/Cargo.toml
@@ -15,6 +15,7 @@ grok = { version = "1", optional = true }
 hex = { version = "0.4", optional = true }
 lazy_static = { version = "1", optional = true }
 md-5 = { version = "0.9", optional = true }
+nom = { version = "6.0.1", optional = true }
 regex = { version = "1", optional = true }
 rust_decimal = { version = "1", optional = true }
 serde_json = { version = "1", optional = true }
@@ -57,6 +58,7 @@ default = [
     "now",
     "ok",
     "only_fields",
+    "parse_aws_alb_log",
     "parse_duration",
     "parse_grok",
     "parse_json",
@@ -108,6 +110,7 @@ merge = []
 now = []
 ok = []
 only_fields = []
+parse_aws_alb_log = ["nom"]
 parse_duration = []
 parse_grok = ["grok"]
 parse_json = ["serde_json"]

diff --git a/lib/remap-functions/src/lib.rs b/lib/remap-functions/src/lib.rs
@@ -46,6 +46,8 @@ mod now;
 mod ok;
 #[cfg(feature = "only_fields")]
 mod only_fields;
+#[cfg(feature = "parse_aws_alb_log")]
+mod parse_aws_alb_log;
 #[cfg(feature = "parse_duration")]
 mod parse_duration;
 #[cfg(feature = "parse_grok")]
@@ -147,6 +149,8 @@ pub use now::Now;
 pub use ok::OK;
 #[cfg(feature = "only_fields")]
 pub use only_fields::OnlyFields;
+#[cfg(feature = "parse_aws_alb_log")]
+pub use parse_aws_alb_log::ParseAwsAlbLog;
 #[cfg(feature = "parse_duration")]
 pub use parse_duration::ParseDuration;
 #[cfg(feature = "parse_grok")]
@@ -248,6 +252,8 @@ pub fn all() -> Vec<Box<dyn remap::Function>> {
         Box::new(OK),
         #[cfg(feature = "only_fields")]
         Box::new(OnlyFields),
+        #[cfg(feature = "parse_aws_alb_log")]
+        Box::new(ParseAwsAlbLog),
         #[cfg(feature = "parse_duration")]
         Box::new(ParseDuration),
         #[cfg(feature = "parse_grok")]