From 9dd4ca63045388e2ed07e534f9bfc78dac05c89b Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Fri, 11 Dec 2020 10:59:38 +0300 Subject: [PATCH 1/7] enhancement(remap transform): add `parse_aws_elb` function Signed-off-by: Kirill Fomichev --- docs/reference/remap/parse_aws_elb.cue | 76 +++++++ src/remap/function.rs | 2 + src/remap/function/parse_aws_elb.rs | 263 +++++++++++++++++++++++++ src/remap/mod.rs | 1 + 4 files changed, 342 insertions(+) create mode 100644 docs/reference/remap/parse_aws_elb.cue create mode 100644 src/remap/function/parse_aws_elb.rs diff --git a/docs/reference/remap/parse_aws_elb.cue b/docs/reference/remap/parse_aws_elb.cue new file mode 100644 index 0000000000000..75adf965bf04b --- /dev/null +++ b/docs/reference/remap/parse_aws_elb.cue @@ -0,0 +1,76 @@ +package metadata + +remap: functions: parse_aws_elb: { + arguments: [ + { + name: "value" + description: "Access log of the Application Load Balancer." + required: true + type: ["string"] + }, + ] + return: ["map"] + category: "parse" + description: #""" + Parses a Elastic Load Balancer Access log into it's constituent components. + """# + examples: [ + { + title: "Success" + input: { + log: #"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""# + } + source: #""" + .parsed = parse_aws_elb(.log) + """# + output: { + log: #"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""# + parsed: { + "type": "http" + "timestamp": "2018-11-30T22:23:00.186641Z" + "elb": "app/my-loadbalancer/50dc6c495c0c9188" + "client_host": "192.168.131.39:2817" + "target_host": "-" + "request_processing_time": 0.0 + "target_processing_time": 0.001 + "response_processing_time": 0.0 + "elb_status_code": "200" + "target_status_code": "200" + "received_bytes": 34 + "sent_bytes": 366 + "request_method": "GET" + "request_url": "http://www.example.com:80/" + "request_protocol": "HTTP/1.1" + "user_agent": "curl/7.46.0" + "ssl_cipher": "-" + "ssl_protocol": "-" + "target_group_arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067" + "trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3" + "domain_name": "-" + "chosen_cert_arn": "-" + "matched_rule_priority": "0" + "request_creation_time": "2018-11-30T22:22:48.364000Z" + "actions_executed": "forward" + "redirect_url": "-" + "error_reason": "-" + "target_port_list": [] + "target_status_code_list": [] + "classification": "-" + "classification_reason": "-" + } + } + }, + { + title: "Error" + input: { + log: "I am not a log" + } + source: #""" + .parsed = parse_aws_elb(.log) + """# + output: { + error: remap.errors.ParseError + } + }, + ] +} diff --git a/src/remap/function.rs b/src/remap/function.rs index 260ef2f789afa..1121a922c05e2 100644 --- a/src/remap/function.rs +++ b/src/remap/function.rs @@ -22,6 +22,7 @@ mod md5; mod merge; mod now; mod only_fields; +mod parse_aws_elb; mod parse_duration; mod parse_grok; mod parse_json; @@ -73,6 +74,7 @@ pub use log::Log; pub use merge::Merge; pub use now::Now; pub use only_fields::OnlyFields; +pub use parse_aws_elb::ParseAwsElb; pub use parse_duration::ParseDuration; pub use parse_grok::ParseGrok; pub use parse_json::ParseJson; diff --git a/src/remap/function/parse_aws_elb.rs b/src/remap/function/parse_aws_elb.rs new file mode 100644 index 0000000000000..8107d4eb0af83 --- /dev/null +++ b/src/remap/function/parse_aws_elb.rs @@ -0,0 +1,263 @@ +use nom::{ + branch::alt, + bytes::complete::{tag, take_while1}, + character::complete::char, + combinator::map_res, + sequence::{delimited, preceded}, + IResult, +}; +use remap::prelude::*; +use std::collections::BTreeMap; + +#[derive(Clone, Copy, Debug)] +pub struct ParseAwsElb; + +impl Function for ParseAwsElb { + fn identifier(&self) -> &'static str { + "parse_aws_elb" + } + + fn parameters(&self) -> &'static [Parameter] { + &[Parameter { + keyword: "value", + accepts: |v| matches!(v, Value::Bytes(_)), + required: true, + }] + } + + fn compile(&self, mut arguments: ArgumentList) -> Result> { + let value = arguments.required("value")?.boxed(); + + Ok(Box::new(ParseAwsElbFn::new(value))) + } +} + +#[derive(Debug, Clone)] +struct ParseAwsElbFn { + value: Box, +} + +impl ParseAwsElbFn { + fn new(value: Box) -> Self { + Self { value } + } +} + +impl Expression for ParseAwsElbFn { + fn execute(&self, state: &mut state::Program, object: &mut dyn Object) -> Result { + let bytes = self.value.execute(state, object)?.try_bytes()?; + + parse_log(&String::from_utf8_lossy(&bytes)) + } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::Bytes) + .into_fallible(true) // Log parsing error + .with_constraint(value::Kind::Map) + } +} + +fn parse_log(mut input: &str) -> Result { + let mut log = BTreeMap::new(); + + macro_rules! get_value { + ($name:expr, $parser:expr) => {{ + let result: IResult<&str, _, (&str, nom::error::ErrorKind)> = $parser(input); + match result { + Ok((rest, value)) => { + input = rest; + value + } + Err(error) => { + return Err(format!("failed to get field `{}`: {}", $name, error).into()) + } + } + }}; + } + macro_rules! field_raw { + ($name:expr, $parser:expr) => { + log.insert($name.into(), get_value!($name, $parser).into()) + }; + } + macro_rules! field { + ($name:expr, $($pattern:pat)|+) => { + field_raw!($name, preceded(char(' '), take_while1(|c| matches!(c, $($pattern)|+)))) + }; + } + macro_rules! field_parse { + ($name:expr, $($pattern:pat)|+, $type:ty) => { + field_raw!($name, map_res(preceded(char(' '), take_while1(|c| matches!(c, $($pattern)|+))), |s: &str| s.parse::<$type>())) + }; + } + + field_raw!("type", take_while1(|c| matches!(c, 'a'..='z' | '0'..='9'))); + field!("timestamp", '0'..='9' | '.' | '-' | ':' | 'T' | 'Z'); + field_raw!("elb", take_anything); + field!("client_host", '0'..='9' | '.' | ':' | '-'); + field!("target_host", '0'..='9' | '.' | ':' | '-'); + field_parse!("request_processing_time", '0'..='9' | '.' | '-', f64); + field_parse!("target_processing_time", '0'..='9' | '.' | '-', f64); + field_parse!("response_processing_time", '0'..='9' | '.' | '-', f64); + field!("elb_status_code", '0'..='9' | '-'); + field!("target_status_code", '0'..='9' | '-'); + field_parse!("received_bytes", '0'..='9' | '-', i64); + field_parse!("sent_bytes", '0'..='9' | '-', i64); + let request = get_value!("request", take_quoted1); + let mut iter = request.splitn(2, ' '); + log.insert("request_method".to_owned(), iter.next().unwrap().into()); // split always have at least 1 item + match iter.next() { + Some(value) => { + let mut iter = value.rsplitn(2, ' '); + log.insert("request_protocol".into(), iter.next().unwrap().into()); // same as previous one + match iter.next() { + Some(value) => log.insert("request_url".into(), value.into()), + None => return Err("failed to get field `request_url`".into()), + } + } + None => return Err("failed to get field `request_url`".into()), + }; + field_raw!("user_agent", take_quoted1); + field_raw!("ssl_cipher", take_anything); + field_raw!("ssl_protocol", take_anything); + field_raw!("target_group_arn", take_anything); + field_raw!("trace_id", take_quoted1); + field_raw!("domain_name", take_quoted1); + field_raw!("chosen_cert_arn", take_quoted1); + field!("matched_rule_priority", '0'..='9' | '-'); + field!( + "request_creation_time", + '0'..='9' | '.' | '-' | ':' | 'T' | 'Z' + ); + field_raw!("actions_executed", take_quoted1); + field_raw!("redirect_url", take_quoted1); + field_raw!("error_reason", take_quoted1); + field_raw!( + "target_port_list", + take_list(|c| matches!(c, '0'..='9' | '.' | ':' | '-')) + ); + field_raw!( + "target_status_code_list", + take_list(|c| matches!(c, '0'..='9')) + ); + field_raw!("classification", take_quoted1); + field_raw!("classification_reason", take_quoted1); + + match input.is_empty() { + true => Ok(log.into()), + false => Err(format!(r#"Log should be fully consumed: "{}""#, input).into()), + } +} + +type SResult<'a, O> = IResult<&'a str, O, (&'a str, nom::error::ErrorKind)>; + +fn take_anything(input: &str) -> SResult<&str> { + preceded(char(' '), take_while1(|c| c != ' '))(input) +} + +fn take_quoted1(input: &str) -> SResult { + delimited(tag(" \""), until_quote, char('"'))(input) +} + +fn until_quote(input: &str) -> SResult { + let mut ret = String::new(); + let mut skip_delimiter = false; + for (i, ch) in input.char_indices() { + if ch == '\\' && !skip_delimiter { + skip_delimiter = true; + } else if ch == '"' && !skip_delimiter { + return Ok((&input[i..], ret)); + } else { + ret.push(ch); + skip_delimiter = false; + } + } + Err(nom::Err::Incomplete(nom::Needed::Unknown)) +} + +fn take_list(cond: impl Fn(char) -> bool) -> impl FnOnce(&str) -> SResult> { + move |input: &str| { + alt(( + map_res(tag(r#" "-""#), |_| { + Ok::<_, std::convert::Infallible>(vec![]) + }), + map_res(preceded(char(' '), take_while1(cond)), |v: &str| { + Ok::<_, std::convert::Infallible>(vec![v]) + }), + ))(input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + remap::test_type_def![ + value_string { + expr: |_| ParseAwsElbFn { value: Literal::from("foo").boxed() }, + def: TypeDef { fallible: true, kind: value::Kind::Map }, + } + + value_optional { + expr: |_| ParseAwsElbFn { value: Box::new(Noop) }, + def: TypeDef { fallible: true, kind: value::Kind::Map }, + } + ]; + + #[test] + fn parse_aws_elb() { + let logs = vec![ + r#"http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 +"GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - +arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" +0 2018-07-02T22:22:48.364000Z "forward" "-" "-" 10.0.0.1:80 200 "-" "-""#, + r#"https 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 +"GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 +arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337281-1d84f3d73c47ec4e58577259" "www.example.com" "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012" +1 2018-07-02T22:22:48.364000Z "authenticate,forward" "-" "-" 10.0.0.1:80 200 "-" "-""#, + r#"h2 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +10.0.1.252:48160 10.0.0.66:9000 0.000 0.002 0.000 200 200 5 257 +"GET https://10.0.2.105:773/ HTTP/2.0" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 +arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337327-72bd00b0343d75b906739c42" "-" "-" +1 2018-07-02T22:22:48.364000Z "redirect" "https://example.com:80/" "-" 10.0.0.66:9000 200 "-" "-""#, + r#"ws 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +10.0.0.140:40914 10.0.1.192:8010 0.001 0.003 0.000 101 101 218 587 +"GET http://10.0.0.30:80/ HTTP/1.1" "-" - - +arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" +1 2018-07-02T22:22:48.364000Z "forward" "-" "-" 10.0.1.192:8010 101 "-" "-""#, + r#"wss 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +10.0.0.140:44244 10.0.0.171:8010 0.000 0.001 0.000 101 101 218 786 +"GET https://10.0.0.30:443/ HTTP/1.1" "-" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 +arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" +1 2018-07-02T22:22:48.364000Z "forward" "-" "-" 10.0.0.171:8010 101 "-" "-""#, + r#"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 +"GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - +arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" +0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""#, + r#"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 +192.168.131.39:2817 - 0.000 0.001 0.000 502 - 34 366 +"GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - +arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 +"Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" +0 2018-11-30T22:22:48.364000Z "forward" "-" "LambdaInvalidResponse" "-" "-" "-" "-""#, + ]; + let logs = logs + .into_iter() + .map(|s| s.replace('\n', " ")) + .collect::>(); + + for log in logs { + assert!(parse_log(&log).is_ok()) + } + } +} diff --git a/src/remap/mod.rs b/src/remap/mod.rs index a91518165ef83..43fb90cf03b8c 100644 --- a/src/remap/mod.rs +++ b/src/remap/mod.rs @@ -27,6 +27,7 @@ lazy_static! { Box::new(Tokenize), Box::new(Sha2), Box::new(Sha3), + Box::new(ParseAwsElb), Box::new(ParseDuration), Box::new(FormatNumber), Box::new(ParseUrl), From da0074d8c17ce86ef3dbe09cd0707141565f67b1 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 16 Dec 2020 18:25:23 +0300 Subject: [PATCH 2/7] rename to parse_aws_alb_log Signed-off-by: Kirill Fomichev --- ...arse_aws_elb.cue => parse_aws_alb_log.cue} | 6 +++--- src/remap/function.rs | 4 ++-- ...{parse_aws_elb.rs => parse_aws_alb_log.rs} | 20 +++++++++---------- src/remap/mod.rs | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) rename docs/reference/remap/{parse_aws_elb.cue => parse_aws_alb_log.cue} (96%) rename src/remap/function/{parse_aws_elb.rs => parse_aws_alb_log.rs} (96%) diff --git a/docs/reference/remap/parse_aws_elb.cue b/docs/reference/remap/parse_aws_alb_log.cue similarity index 96% rename from docs/reference/remap/parse_aws_elb.cue rename to docs/reference/remap/parse_aws_alb_log.cue index 75adf965bf04b..8a85f1aee881b 100644 --- a/docs/reference/remap/parse_aws_elb.cue +++ b/docs/reference/remap/parse_aws_alb_log.cue @@ -1,6 +1,6 @@ package metadata -remap: functions: parse_aws_elb: { +remap: functions: parse_aws_alb_log: { arguments: [ { name: "value" @@ -21,7 +21,7 @@ remap: functions: parse_aws_elb: { log: #"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""# } source: #""" - .parsed = parse_aws_elb(.log) + .parsed = parse_aws_alb_log(.log) """# output: { log: #"http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-""# @@ -66,7 +66,7 @@ remap: functions: parse_aws_elb: { log: "I am not a log" } source: #""" - .parsed = parse_aws_elb(.log) + .parsed = parse_aws_alb_log(.log) """# output: { error: remap.errors.ParseError diff --git a/src/remap/function.rs b/src/remap/function.rs index 1121a922c05e2..b16fc78fdda80 100644 --- a/src/remap/function.rs +++ b/src/remap/function.rs @@ -22,7 +22,7 @@ mod md5; mod merge; mod now; mod only_fields; -mod parse_aws_elb; +mod parse_aws_alb_log; mod parse_duration; mod parse_grok; mod parse_json; @@ -74,7 +74,7 @@ pub use log::Log; pub use merge::Merge; pub use now::Now; pub use only_fields::OnlyFields; -pub use parse_aws_elb::ParseAwsElb; +pub use parse_aws_alb_log::ParseAwsAlbLog; pub use parse_duration::ParseDuration; pub use parse_grok::ParseGrok; pub use parse_json::ParseJson; diff --git a/src/remap/function/parse_aws_elb.rs b/src/remap/function/parse_aws_alb_log.rs similarity index 96% rename from src/remap/function/parse_aws_elb.rs rename to src/remap/function/parse_aws_alb_log.rs index 8107d4eb0af83..7e6d17a5de110 100644 --- a/src/remap/function/parse_aws_elb.rs +++ b/src/remap/function/parse_aws_alb_log.rs @@ -10,11 +10,11 @@ use remap::prelude::*; use std::collections::BTreeMap; #[derive(Clone, Copy, Debug)] -pub struct ParseAwsElb; +pub struct ParseAwsAlbLog; -impl Function for ParseAwsElb { +impl Function for ParseAwsAlbLog { fn identifier(&self) -> &'static str { - "parse_aws_elb" + "parse_aws_alb_log" } fn parameters(&self) -> &'static [Parameter] { @@ -28,22 +28,22 @@ impl Function for ParseAwsElb { fn compile(&self, mut arguments: ArgumentList) -> Result> { let value = arguments.required("value")?.boxed(); - Ok(Box::new(ParseAwsElbFn::new(value))) + Ok(Box::new(ParseAwsAlbLogFn::new(value))) } } #[derive(Debug, Clone)] -struct ParseAwsElbFn { +struct ParseAwsAlbLogFn { value: Box, } -impl ParseAwsElbFn { +impl ParseAwsAlbLogFn { fn new(value: Box) -> Self { Self { value } } } -impl Expression for ParseAwsElbFn { +impl Expression for ParseAwsAlbLogFn { fn execute(&self, state: &mut state::Program, object: &mut dyn Object) -> Result { let bytes = self.value.execute(state, object)?.try_bytes()?; @@ -195,18 +195,18 @@ mod tests { remap::test_type_def![ value_string { - expr: |_| ParseAwsElbFn { value: Literal::from("foo").boxed() }, + expr: |_| ParseAwsAlbLogFn { value: Literal::from("foo").boxed() }, def: TypeDef { fallible: true, kind: value::Kind::Map }, } value_optional { - expr: |_| ParseAwsElbFn { value: Box::new(Noop) }, + expr: |_| ParseAwsAlbLogFn { value: Box::new(Noop) }, def: TypeDef { fallible: true, kind: value::Kind::Map }, } ]; #[test] - fn parse_aws_elb() { + fn parse_aws_alb_log() { let logs = vec![ r#"http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 diff --git a/src/remap/mod.rs b/src/remap/mod.rs index 43fb90cf03b8c..3589abe886db8 100644 --- a/src/remap/mod.rs +++ b/src/remap/mod.rs @@ -27,7 +27,7 @@ lazy_static! { Box::new(Tokenize), Box::new(Sha2), Box::new(Sha3), - Box::new(ParseAwsElb), + Box::new(ParseAwsAlbLog), Box::new(ParseDuration), Box::new(FormatNumber), Box::new(ParseUrl), From cc42f7f1f6e1818036e9e0f0f83ee07b684a7ecf Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 16 Dec 2020 18:26:19 +0300 Subject: [PATCH 3/7] add to CODEOWNERS Signed-off-by: Kirill Fomichev --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1d8242ed05d95..cb58a31492088 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -75,6 +75,7 @@ /docs/reference/remap/log.cue @FungusHumungus /docs/reference/remap/merge.cue @FungusHumungus /docs/reference/remap/parse_grok.cue @FungusHumungus +/docs/reference/remap/parse_aws_alb_log.cue @fanatid /distribution/ @hoverbear @jamtur01 /distribution/docker/ @vector-kubernetes @@ -202,6 +203,7 @@ /src/remap/function/log.rs @FungusHumungus /src/remap/function/merge.rs @FungusHumungus /src/remap/function/parse_grok.rs @FungusHumungus +/src/remap/function/parse_aws_alb_log.rs @fanatid /src/wasm/ @hoverbear @JeanMertz From 734b6a443672b6d751a8fb40c5d8e2ad88687869 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 16 Dec 2020 21:46:36 +0300 Subject: [PATCH 4/7] null instead "-" Signed-off-by: Kirill Fomichev --- docs/reference/remap/parse_aws_alb_log.cue | 18 +++++++++--------- src/remap/function/parse_aws_alb_log.rs | 8 +++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/reference/remap/parse_aws_alb_log.cue b/docs/reference/remap/parse_aws_alb_log.cue index 8a85f1aee881b..2ef81f8d2dc68 100644 --- a/docs/reference/remap/parse_aws_alb_log.cue +++ b/docs/reference/remap/parse_aws_alb_log.cue @@ -30,7 +30,7 @@ remap: functions: parse_aws_alb_log: { "timestamp": "2018-11-30T22:23:00.186641Z" "elb": "app/my-loadbalancer/50dc6c495c0c9188" "client_host": "192.168.131.39:2817" - "target_host": "-" + "target_host": null "request_processing_time": 0.0 "target_processing_time": 0.001 "response_processing_time": 0.0 @@ -42,21 +42,21 @@ remap: functions: parse_aws_alb_log: { "request_url": "http://www.example.com:80/" "request_protocol": "HTTP/1.1" "user_agent": "curl/7.46.0" - "ssl_cipher": "-" - "ssl_protocol": "-" + "ssl_cipher": null + "ssl_protocol": null "target_group_arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067" "trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3" - "domain_name": "-" - "chosen_cert_arn": "-" + "domain_name": null + "chosen_cert_arn": null "matched_rule_priority": "0" "request_creation_time": "2018-11-30T22:22:48.364000Z" "actions_executed": "forward" - "redirect_url": "-" - "error_reason": "-" + "redirect_url": null + "error_reason": null "target_port_list": [] "target_status_code_list": [] - "classification": "-" - "classification_reason": "-" + "classification": null + "classification_reason": null } } }, diff --git a/src/remap/function/parse_aws_alb_log.rs b/src/remap/function/parse_aws_alb_log.rs index 7e6d17a5de110..fe227091030bb 100644 --- a/src/remap/function/parse_aws_alb_log.rs +++ b/src/remap/function/parse_aws_alb_log.rs @@ -78,7 +78,13 @@ fn parse_log(mut input: &str) -> Result { } macro_rules! field_raw { ($name:expr, $parser:expr) => { - log.insert($name.into(), get_value!($name, $parser).into()) + log.insert( + $name.into(), + match get_value!($name, $parser).into() { + Value::Bytes(bytes) if bytes == &"-" => Value::Null, + value => value, + }, + ) }; } macro_rules! field { From 60ac1efa30d3b6a0b96589f7274b3defb633fdb4 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 17 Dec 2020 14:12:51 +0300 Subject: [PATCH 5/7] add behavior test Signed-off-by: Kirill Fomichev --- tests/behavior/transforms/remap.toml | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/behavior/transforms/remap.toml b/tests/behavior/transforms/remap.toml index d038b92a74a3a..1dc10944b0c75 100644 --- a/tests/behavior/transforms/remap.toml +++ b/tests/behavior/transforms/remap.toml @@ -1230,3 +1230,51 @@ [[tests.outputs.conditions]] "a.equals" = "fbaro" "b.equals" = "fbarbar" + +[transforms.remap_function_parse_aws_alb_log] + inputs = [] + type = "remap" + source = """ + .parts = parse_aws_alb_log(.log) + """ +[[tests]] + name = "remap_function_parse_aws_alb_log" + [tests.input] + insert_at = "remap_function_parse_aws_alb_log" + type = "log" + [tests.input.log_fields] + log = 'http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "-" "-" "-" "-" "-"' + [[tests.outputs]] + extract_from = "remap_function_parse_aws_alb_log" + [[tests.outputs.conditions]] + "parts.type.equals" = "http" + "parts.timestamp.equals" = "2018-11-30T22:23:00.186641Z" + "parts.elb.equals" = "app/my-loadbalancer/50dc6c495c0c9188" + "parts.client_host.equals" = "192.168.131.39:2817" + "parts.target_host.equals" = "" + "parts.request_processing_time.equals" = 0.0 + "parts.target_processing_time.equals" = 0.001 + "parts.response_processing_time.equals" = 0.0 + "parts.elb_status_code.equals" = "200" + "parts.target_status_code.equals" = "200" + "parts.received_bytes.equals" = 34 + "parts.sent_bytes.equals" = 366 + "parts.request_method.equals" = "GET" + "parts.request_url.equals" = "http://www.example.com:80/" + "parts.request_protocol.equals" = "HTTP/1.1" + "parts.user_agent.equals" = "curl/7.46.0" + "parts.ssl_cipher.equals" = "" + "parts.ssl_protocol.equals" = "" + "parts.target_group_arn.equals" = "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067" + "parts.trace_id.equals" = "Root=1-58337364-23a8c76965a2ef7629b185e3" + "parts.domain_name.equals" = "" + "parts.chosen_cert_arn.equals" = "" + "parts.matched_rule_priority.equals" = "0" + "parts.request_creation_time.equals" = "2018-11-30T22:22:48.364000Z" + "parts.actions_executed.equals" = "forward" + "parts.redirect_url.equals" = "" + "parts.error_reason.equals" = "" + "parts.target_port_list.equals" = [] + "parts.target_status_code_list.equals" = [] + "parts.classification.equals" = "" + "parts.classification_reason.equals" = "" From 4fd4c1c64162aa6ca3ca57281520c4fa9ab76a59 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 17 Dec 2020 16:54:42 +0300 Subject: [PATCH 6/7] fix behavior test Signed-off-by: Kirill Fomichev --- tests/behavior/transforms/remap.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/behavior/transforms/remap.toml b/tests/behavior/transforms/remap.toml index 1dc10944b0c75..bc0ae7f9dfb84 100644 --- a/tests/behavior/transforms/remap.toml +++ b/tests/behavior/transforms/remap.toml @@ -1274,7 +1274,5 @@ "parts.actions_executed.equals" = "forward" "parts.redirect_url.equals" = "" "parts.error_reason.equals" = "" - "parts.target_port_list.equals" = [] - "parts.target_status_code_list.equals" = [] "parts.classification.equals" = "" "parts.classification_reason.equals" = "" From f3777cf232157f6f18701f113d885dbd6fa1a1c8 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 17 Dec 2020 19:44:16 +0300 Subject: [PATCH 7/7] fix tests Signed-off-by: Kirill Fomichev --- lib/remap-functions/src/parse_aws_alb_log.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/remap-functions/src/parse_aws_alb_log.rs b/lib/remap-functions/src/parse_aws_alb_log.rs index fe227091030bb..b66fbda199165 100644 --- a/lib/remap-functions/src/parse_aws_alb_log.rs +++ b/lib/remap-functions/src/parse_aws_alb_log.rs @@ -202,12 +202,12 @@ mod tests { remap::test_type_def![ value_string { expr: |_| ParseAwsAlbLogFn { value: Literal::from("foo").boxed() }, - def: TypeDef { fallible: true, kind: value::Kind::Map }, + def: TypeDef { fallible: true, kind: value::Kind::Map, ..Default::default() }, } value_optional { expr: |_| ParseAwsAlbLogFn { value: Box::new(Noop) }, - def: TypeDef { fallible: true, kind: value::Kind::Map }, + def: TypeDef { fallible: true, kind: value::Kind::Map, ..Default::default() }, } ];