From a10c1f1dded570f99c4972ef9f730cec79218b75 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 1 Sep 2022 16:02:05 +0200 Subject: [PATCH] smtp: use rust for mime parsing Ticket: #3487 --- rust/src/mime/mime.rs | 569 ++++ rust/src/mime/mod.rs | 599 +--- rust/src/mime/smtp.rs | 854 ++++++ rust/src/mime/smtp_log.rs | 241 ++ src/Makefile.am | 2 - src/app-layer-htp.c | 11 +- src/app-layer-smtp.c | 398 +-- src/app-layer-smtp.h | 14 +- src/output-json-email-common.c | 200 +- src/runmode-unittests.c | 2 +- src/tests/fuzz/fuzz_mimedecparseline.c | 33 +- src/util-decode-mime.c | 3590 ------------------------ src/util-decode-mime.h | 243 -- src/util-lua-smtp.c | 35 +- src/util-spm-bs.c | 10 + src/util-spm-bs.h | 1 + 16 files changed, 1861 insertions(+), 4941 deletions(-) create mode 100644 rust/src/mime/mime.rs create mode 100644 rust/src/mime/smtp.rs create mode 100644 rust/src/mime/smtp_log.rs delete mode 100644 src/util-decode-mime.c delete mode 100644 src/util-decode-mime.h diff --git a/rust/src/mime/mime.rs b/rust/src/mime/mime.rs new file mode 100644 index 000000000000..cebf268d1cc7 --- /dev/null +++ b/rust/src/mime/mime.rs @@ -0,0 +1,569 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use crate::common::nom7::take_until_and_consume; +use nom7::branch::alt; +use nom7::bytes::complete::{tag, take, take_till, take_until, take_while}; +use nom7::character::complete::char; +use nom7::combinator::{complete, opt, rest, value}; +use nom7::error::{make_error, ErrorKind}; +use nom7::{Err, IResult}; +use std; +use std::collections::HashMap; + +#[derive(Clone)] +pub struct HeaderTokens<'a> { + pub tokens: HashMap<&'a [u8], &'a [u8]>, +} + +fn mime_parse_value_delimited(input: &[u8]) -> IResult<&[u8], &[u8]> { + let (input, _) = char('"')(input)?; + let mut escaping = false; + for i in 0..input.len() { + if input[i] == b'\\' { + escaping = true; + } else { + if input[i] == b'"' && !escaping { + return Ok((&input[i + 1..], &input[..i])); + } + // unescape can be processed later + escaping = false; + } + } + // should fail + let (input, value) = take_until("\"")(input)?; + let (input, _) = char('"')(input)?; + return Ok((input, value)); +} + +fn mime_parse_value_until_semicolon(input: &[u8]) -> IResult<&[u8], &[u8]> { + let (input, value) = alt((take_till(|ch: u8| ch == b';'), rest))(input)?; + for i in 0..value.len() { + if !is_mime_space(value[value.len() - i - 1]) { + return Ok((input, &value[..value.len() - i])); + } + } + return Ok((input, value)); +} + +#[inline] +fn is_mime_space(ch: u8) -> bool { + ch == 0x20 || ch == 0x09 || ch == 0x0a || ch == 0x0d +} + +pub fn mime_parse_header_token(input: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> { + // from RFC2047 : like ch.is_ascii_whitespace but without 0x0c FORM-FEED + let (input, _) = take_while(is_mime_space)(input)?; + let (input, name) = take_until("=")(input)?; + let (input, _) = char('=')(input)?; + let (input, value) = + alt((mime_parse_value_delimited, mime_parse_value_until_semicolon))(input)?; + let (input, _) = take_while(is_mime_space)(input)?; + let (input, _) = opt(complete(char(';')))(input)?; + return Ok((input, (name, value))); +} + +fn mime_parse_header_tokens(input: &[u8]) -> IResult<&[u8], HeaderTokens> { + let (mut input, _) = take_until_and_consume(b";")(input)?; + let mut tokens = HashMap::new(); + while !input.is_empty() { + match mime_parse_header_token(input) { + Ok((rem, t)) => { + tokens.insert(t.0, t.1); + // should never happen + debug_validate_bug_on!(input.len() == rem.len()); + if input.len() == rem.len() { + //infinite loop + return Err(Err::Error(make_error(input, ErrorKind::Eof))); + } + input = rem; + } + Err(_) => { + // keep first tokens is error in remaining buffer + break; + } + } + } + return Ok((input, HeaderTokens { tokens })); +} + +pub fn mime_find_header_token<'a>( + header: &'a [u8], token: &[u8], sections_values: &'a mut Vec, +) -> Option<&'a [u8]> { + match mime_parse_header_tokens(header) { + Ok((_rem, t)) => { + // in case of multiple sections for the parameter cf RFC2231 + let mut current_section_slice = Vec::new(); + + // look for the specific token + match t.tokens.get(token) { + // easy nominal case + Some(value) => return Some(value), + None => { + // check for initial section of a parameter + current_section_slice.extend_from_slice(token); + current_section_slice.extend_from_slice(b"*0"); + match t.tokens.get(¤t_section_slice[..]) { + Some(value) => { + sections_values.extend_from_slice(value); + let l = current_section_slice.len(); + current_section_slice[l - 1] = b'1'; + } + None => return None, + } + } + } + + let mut current_section_seen = 1; + // we have at least the initial section + // try looping until we do not find anymore a next section + loop { + match t.tokens.get(¤t_section_slice[..]) { + Some(value) => { + sections_values.extend_from_slice(value); + current_section_seen += 1; + let nbdigits = current_section_slice.len() - token.len() - 1; + current_section_slice.truncate(current_section_slice.len() - nbdigits); + current_section_slice + .extend_from_slice(current_section_seen.to_string().as_bytes()); + } + None => return Some(sections_values), + } + } + } + Err(_) => { + return None; + } + } +} + +pub(crate) const RS_MIME_MAX_TOKEN_LEN: usize = 255; + +#[derive(Debug)] +enum MimeParserState { + Start, + Header, + HeaderEnd, + Chunk, + BoundaryWaitingForEol, +} + +impl Default for MimeParserState { + fn default() -> Self { + MimeParserState::Start + } +} + +#[derive(Debug, Default)] +pub struct MimeStateHTTP { + boundary: Vec, + filename: Vec, + state: MimeParserState, +} + +#[repr(u8)] +#[derive(Copy, Clone, PartialOrd, PartialEq, Eq)] +pub enum MimeParserResult { + MimeNeedsMore = 0, + MimeFileOpen = 1, + MimeFileChunk = 2, + MimeFileClose = 3, +} + +fn mime_parse_skip_line(input: &[u8]) -> IResult<&[u8], MimeParserState> { + let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?; + let (input, _) = char('\n')(input)?; + return Ok((input, MimeParserState::Start)); +} + +fn mime_parse_boundary_regular<'a>( + boundary: &[u8], input: &'a [u8], +) -> IResult<&'a [u8], MimeParserState> { + let (input, _) = tag(boundary)(input)?; + let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?; + let (input, _) = char('\n')(input)?; + return Ok((input, MimeParserState::Header)); +} + +// Number of characters after boundary, without end of line, before changing state to streaming +const MIME_BOUNDARY_MAX_BEFORE_EOL: usize = 128; +const MIME_HEADER_MAX_LINE: usize = 4096; + +fn mime_parse_boundary_missing_eol<'a>( + boundary: &[u8], input: &'a [u8], +) -> IResult<&'a [u8], MimeParserState> { + let (input, _) = tag(boundary)(input)?; + let (input, _) = take(MIME_BOUNDARY_MAX_BEFORE_EOL)(input)?; + return Ok((input, MimeParserState::BoundaryWaitingForEol)); +} + +fn mime_parse_boundary<'a>(boundary: &[u8], input: &'a [u8]) -> IResult<&'a [u8], MimeParserState> { + let r = mime_parse_boundary_regular(boundary, input); + if r.is_ok() { + return r; + } + let r2 = mime_parse_skip_line(input); + if r2.is_ok() { + return r2; + } + return mime_parse_boundary_missing_eol(boundary, input); +} + +fn mime_consume_until_eol(input: &[u8]) -> IResult<&[u8], bool> { + return alt((value(true, mime_parse_skip_line), value(false, rest)))(input); +} + +pub fn mime_parse_header_line(input: &[u8]) -> IResult<&[u8], &[u8]> { + let (input, name) = take_till(|ch: u8| ch == b':')(input)?; + let (input, _) = char(':')(input)?; + let (input, _) = take_while(is_mime_space)(input)?; + return Ok((input, name)); +} + +// s2 is already lower case +pub fn slice_equals_lowercase(s1: &[u8], s2: &[u8]) -> bool { + if s1.len() == s2.len() { + for i in 0..s1.len() { + if s1[i].to_ascii_lowercase() != s2[i] { + return false; + } + } + return true; + } + return false; +} + +fn mime_parse_headers<'a>( + ctx: &mut MimeStateHTTP, i: &'a [u8], +) -> IResult<&'a [u8], (MimeParserState, bool, bool)> { + let mut fileopen = false; + let mut errored = false; + let mut input = i; + while !input.is_empty() { + if let Ok((input2, line)) = take_until::<_, &[u8], nom7::error::Error<&[u8]>>("\r\n")(input) + { + if let Ok((value, name)) = mime_parse_header_line(line) { + if slice_equals_lowercase(name, "content-disposition".as_bytes()) { + let mut sections_values = Vec::new(); + if let Some(filename) = + mime_find_header_token(value, "filename".as_bytes(), &mut sections_values) + { + if !filename.is_empty() { + ctx.filename = Vec::with_capacity(filename.len()); + fileopen = true; + for c in filename { + // unescape + if *c != b'\\' { + ctx.filename.push(*c); + } + } + } + } + } + if value.is_empty() { + errored = true; + } + } else if !line.is_empty() { + errored = true; + } + let (input3, _) = tag("\r\n")(input2)?; + input = input3; + if line.is_empty() || (line.len() == 1 && line[0] == b'\r') { + return Ok((input, (MimeParserState::HeaderEnd, fileopen, errored))); + } + } else { + // guard against too long header lines + if input.len() > MIME_HEADER_MAX_LINE { + return Ok(( + input, + (MimeParserState::BoundaryWaitingForEol, fileopen, errored), + )); + } + if input.len() < i.len() { + return Ok((input, (MimeParserState::Header, fileopen, errored))); + } // else only an incomplete line, ask for more + return Err(Err::Error(make_error(input, ErrorKind::Eof))); + } + } + return Ok((input, (MimeParserState::Header, fileopen, errored))); +} + +type NomTakeError<'a> = Err>; + +fn mime_consume_chunk<'a>(boundary: &[u8], input: &'a [u8]) -> IResult<&'a [u8], bool> { + let r: Result<(&[u8], &[u8]), NomTakeError> = take_until("\r\n")(input); + if let Ok((input, line)) = r { + let (next_line, _) = tag("\r\n")(input)?; + if next_line.len() < boundary.len() { + if next_line == &boundary[..next_line.len()] { + if !line.is_empty() { + // consume as chunk up to eol (not consuming eol) + return Ok((input, false)); + } + // new line beignning like boundary, with nothin to consume as chunk : request more + return Err(Err::Error(make_error(input, ErrorKind::Eof))); + } + // not like boundary : consume everything as chunk + return Ok((&input[input.len()..], false)); + } // else + if &next_line[..boundary.len()] == boundary { + // end of file with boundary, consume eol but do not consume boundary + return Ok((next_line, true)); + } + // not like boundary : consume everything as chunk + return Ok((next_line, false)); + } else { + return Ok((&input[input.len()..], false)); + } +} + +pub const MIME_EVENT_FLAG_INVALID_HEADER: u32 = 0x01; +pub const MIME_EVENT_FLAG_NO_FILEDATA: u32 = 0x02; + +fn mime_process(ctx: &mut MimeStateHTTP, i: &[u8]) -> (MimeParserResult, u32, u32) { + let mut input = i; + let mut consumed = 0; + let mut warnings = 0; + while !input.is_empty() { + match ctx.state { + MimeParserState::Start => { + if let Ok((rem, next)) = mime_parse_boundary(&ctx.boundary, input) { + ctx.state = next; + consumed += (input.len() - rem.len()) as u32; + input = rem; + } else { + return (MimeParserResult::MimeNeedsMore, consumed, warnings); + } + } + MimeParserState::BoundaryWaitingForEol => { + if let Ok((rem, found)) = mime_consume_until_eol(input) { + if found { + ctx.state = MimeParserState::Header; + } + consumed += (input.len() - rem.len()) as u32; + input = rem; + } else { + // should never happen + return (MimeParserResult::MimeNeedsMore, consumed, warnings); + } + } + MimeParserState::Header => { + if let Ok((rem, (next, fileopen, err))) = mime_parse_headers(ctx, input) { + ctx.state = next; + consumed += (input.len() - rem.len()) as u32; + input = rem; + if err { + warnings |= MIME_EVENT_FLAG_INVALID_HEADER; + } + if fileopen { + return (MimeParserResult::MimeFileOpen, consumed, warnings); + } + } else { + return (MimeParserResult::MimeNeedsMore, consumed, warnings); + } + } + MimeParserState::HeaderEnd => { + // check if we start with the boundary + // and transition to chunk, or empty file and back to start + if input.len() < ctx.boundary.len() { + if input == &ctx.boundary[..input.len()] { + return (MimeParserResult::MimeNeedsMore, consumed, warnings); + } + ctx.state = MimeParserState::Chunk; + } else if input[..ctx.boundary.len()] == ctx.boundary { + ctx.state = MimeParserState::Start; + if !ctx.filename.is_empty() { + warnings |= MIME_EVENT_FLAG_NO_FILEDATA; + } + ctx.filename.clear(); + return (MimeParserResult::MimeFileClose, consumed, warnings); + } else { + ctx.state = MimeParserState::Chunk; + } + } + MimeParserState::Chunk => { + if let Ok((rem, eof)) = mime_consume_chunk(&ctx.boundary, input) { + consumed += (input.len() - rem.len()) as u32; + if eof { + ctx.state = MimeParserState::Start; + ctx.filename.clear(); + return (MimeParserResult::MimeFileClose, consumed, warnings); + } else { + // + 2 for \r\n + if rem.len() < ctx.boundary.len() + 2 { + return (MimeParserResult::MimeFileChunk, consumed, warnings); + } + input = rem; + } + } else { + return (MimeParserResult::MimeNeedsMore, consumed, warnings); + } + } + } + } + return (MimeParserResult::MimeNeedsMore, consumed, warnings); +} + +pub fn mime_state_init(i: &[u8]) -> Option { + let mut sections_values = Vec::new(); + if let Some(value) = mime_find_header_token(i, "boundary".as_bytes(), &mut sections_values) { + if value.len() <= RS_MIME_MAX_TOKEN_LEN { + let mut r = MimeStateHTTP { + boundary: Vec::with_capacity(2 + value.len()), + ..Default::default() + }; + // start wih 2 additional hyphens + r.boundary.push(b'-'); + r.boundary.push(b'-'); + for c in value { + // unescape + if *c != b'\\' { + r.boundary.push(*c); + } + } + return Some(r); + } + } + return None; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeStateInit(input: *const u8, input_len: u32) -> *mut MimeStateHTTP { + let slice = build_slice!(input, input_len as usize); + + if let Some(ctx) = mime_state_init(slice) { + let boxed = Box::new(ctx); + return Box::into_raw(boxed) as *mut _; + } + return std::ptr::null_mut(); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeParse( + ctx: &mut MimeStateHTTP, input: *const u8, input_len: u32, consumed: *mut u32, + warnings: *mut u32, +) -> MimeParserResult { + let slice = build_slice!(input, input_len as usize); + let (r, c, w) = mime_process(ctx, slice); + *consumed = c; + *warnings = w; + return r; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeStateGetFilename( + ctx: &mut MimeStateHTTP, buffer: *mut *const u8, filename_len: *mut u16, +) { + if !ctx.filename.is_empty() { + *buffer = ctx.filename.as_ptr(); + if ctx.filename.len() < u16::MAX.into() { + *filename_len = ctx.filename.len() as u16; + } else { + *filename_len = u16::MAX; + } + } else { + *buffer = std::ptr::null_mut(); + *filename_len = 0; + } +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeStateFree(ctx: &mut MimeStateHTTP) { + std::mem::drop(Box::from_raw(ctx)); +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_mime_find_header_token() { + let mut outvec = Vec::new(); + let undelimok = mime_find_header_token( + "attachment; filename=test;".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(undelimok, Some("test".as_bytes())); + + let delimok = mime_find_header_token( + "attachment; filename=\"test2\";".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(delimok, Some("test2".as_bytes())); + + let escaped = mime_find_header_token( + "attachment; filename=\"test\\\"2\";".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(escaped, Some("test\\\"2".as_bytes())); + + let evasion_othertoken = mime_find_header_token( + "attachment; dummy=\"filename=wrong\"; filename=real;".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(evasion_othertoken, Some("real".as_bytes())); + + let evasion_suffixtoken = mime_find_header_token( + "attachment; notafilename=wrong; filename=good;".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(evasion_suffixtoken, Some("good".as_bytes())); + + let badending = mime_find_header_token( + "attachment; filename=oksofar; badending".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(badending, Some("oksofar".as_bytes())); + + let missend = mime_find_header_token( + "attachment; filename=test".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(missend, Some("test".as_bytes())); + + let spaces = mime_find_header_token( + "attachment; filename=test me wrong".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(spaces, Some("test me wrong".as_bytes())); + + assert_eq!(outvec.len(), 0); + let multi = mime_find_header_token( + "attachment; filename*0=abc; filename*1=\"def\";".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(multi, Some("abcdef".as_bytes())); + outvec.clear(); + + let multi = mime_find_header_token( + "attachment; filename*1=456; filename*0=\"123\"".as_bytes(), + "filename".as_bytes(), + &mut outvec, + ); + assert_eq!(multi, Some("123456".as_bytes())); + outvec.clear(); + } +} diff --git a/rust/src/mime/mod.rs b/rust/src/mime/mod.rs index 5899da415fea..2eac4d1bd442 100644 --- a/rust/src/mime/mod.rs +++ b/rust/src/mime/mod.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2021 Open Information Security Foundation +/* Copyright (C) 2021-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -17,597 +17,6 @@ //! MIME protocol parser module. -use crate::common::nom7::take_until_and_consume; -use nom7::branch::alt; -use nom7::bytes::complete::{tag, take, take_till, take_until, take_while}; -use nom7::character::complete::char; -use nom7::combinator::{complete, opt, rest, value}; -use nom7::error::{make_error, ErrorKind}; -use nom7::{Err, IResult}; -use std; -use std::collections::HashMap; - -#[derive(Clone)] -pub struct MIMEHeaderTokens<'a> { - pub tokens: HashMap<&'a [u8], &'a [u8]>, -} - -fn mime_parse_value_delimited(input: &[u8]) -> IResult<&[u8], &[u8]> { - let (input, _) = char('"')(input)?; - let mut escaping = false; - for i in 0..input.len() { - if input[i] == b'\\' { - escaping = true; - } else { - if input[i] == b'"' && !escaping { - return Ok((&input[i + 1..], &input[..i])); - } - // unescape can be processed later - escaping = false; - } - } - // should fail - let (input, value) = take_until("\"")(input)?; - let (input, _) = char('"')(input)?; - return Ok((input, value)); -} - -fn mime_parse_value_until(input: &[u8]) -> IResult<&[u8], &[u8]> { - let (input, value) = alt((take_till(|ch: u8| ch == b';'), rest))(input)?; - for i in 0..value.len() { - if !is_mime_space(value[value.len() - i - 1]) { - return Ok((input, &value[..value.len() - i])); - } - } - return Ok((input, value)); -} - -#[inline] -fn is_mime_space(ch: u8) -> bool { - ch == 0x20 || ch == 0x09 || ch == 0x0a || ch == 0x0d -} - -pub fn mime_parse_header_token(input: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> { - // from RFC2047 : like ch.is_ascii_whitespace but without 0x0c FORM-FEED - let (input, _) = take_while(|ch: u8| is_mime_space(ch))(input)?; - let (input, name) = take_until("=")(input)?; - let (input, _) = char('=')(input)?; - let (input, value) = alt((mime_parse_value_delimited, mime_parse_value_until))(input)?; - let (input, _) = take_while(|ch: u8| is_mime_space(ch))(input)?; - let (input, _) = opt(complete(char(';')))(input)?; - return Ok((input, (name, value))); -} - -fn mime_parse_header_tokens(input: &[u8]) -> IResult<&[u8], MIMEHeaderTokens> { - let (mut input, _) = take_until_and_consume(b";")(input)?; - let mut tokens = HashMap::new(); - while !input.is_empty() { - match mime_parse_header_token(input) { - Ok((rem, t)) => { - tokens.insert(t.0, t.1); - // should never happen - debug_validate_bug_on!(input.len() == rem.len()); - if input.len() == rem.len() { - //infinite loop - return Err(Err::Error(make_error(input, ErrorKind::Eof))); - } - input = rem; - } - Err(_) => { - // keep first tokens is error in remaining buffer - break; - } - } - } - return Ok((input, MIMEHeaderTokens { tokens })); -} - -fn mime_find_header_token<'a>( - header: &'a [u8], token: &[u8], sections_values: &'a mut Vec, -) -> Result<&'a [u8], ()> { - match mime_parse_header_tokens(header) { - Ok((_rem, t)) => { - // in case of multiple sections for the parameter cf RFC2231 - let mut current_section_slice = Vec::new(); - - // look for the specific token - match t.tokens.get(token) { - // easy nominal case - Some(value) => return Ok(value), - None => { - // check for initial section of a parameter - current_section_slice.extend_from_slice(token); - current_section_slice.extend_from_slice(b"*0"); - match t.tokens.get(¤t_section_slice[..]) { - Some(value) => { - sections_values.extend_from_slice(value); - let l = current_section_slice.len(); - current_section_slice[l - 1] = b'1'; - } - None => return Err(()), - } - } - } - - let mut current_section_seen = 1; - // we have at least the initial section - // try looping until we do not find anymore a next section - loop { - match t.tokens.get(¤t_section_slice[..]) { - Some(value) => { - sections_values.extend_from_slice(value); - current_section_seen += 1; - let nbdigits = current_section_slice.len() - token.len() - 1; - current_section_slice.truncate(current_section_slice.len() - nbdigits); - current_section_slice - .extend_from_slice(current_section_seen.to_string().as_bytes()); - } - None => return Ok(sections_values), - } - } - } - Err(_) => { - return Err(()); - } - } -} - -// used on the C side -pub const RS_MIME_MAX_TOKEN_LEN: usize = 255; - -#[no_mangle] -pub unsafe extern "C" fn rs_mime_find_header_token( - hinput: *const u8, hlen: u32, tinput: *const u8, tlen: u32, outbuf: &mut [u8; 255], - outlen: *mut u32, -) -> bool { - let hbuf = build_slice!(hinput, hlen as usize); - let tbuf = build_slice!(tinput, tlen as usize); - let mut sections_values = Vec::new(); - if let Ok(value) = mime_find_header_token(hbuf, tbuf, &mut sections_values) { - // limit the copy to the supplied buffer size - if value.len() <= RS_MIME_MAX_TOKEN_LEN { - outbuf[..value.len()].clone_from_slice(value); - } else { - outbuf.clone_from_slice(&value[..RS_MIME_MAX_TOKEN_LEN]); - } - *outlen = value.len() as u32; - return true; - } - return false; -} - -#[derive(Debug)] -enum MimeParserState { - MimeStart = 0, - MimeHeader = 1, - MimeHeaderEnd = 2, - MimeChunk = 3, - MimeBoundaryWaitingForEol = 4, -} - -impl Default for MimeParserState { - fn default() -> Self { - MimeParserState::MimeStart - } -} - -#[derive(Debug, Default)] -pub struct MimeStateHTTP { - boundary: Vec, - filename: Vec, - state: MimeParserState, -} - -#[repr(u8)] -#[derive(Copy, Clone, PartialOrd, PartialEq)] -pub enum MimeParserResult { - MimeNeedsMore = 0, - MimeFileOpen = 1, - MimeFileChunk = 2, - MimeFileClose = 3, -} - -fn mime_parse_skip_line(input: &[u8]) -> IResult<&[u8], MimeParserState> { - let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?; - let (input, _) = char('\n')(input)?; - return Ok((input, MimeParserState::MimeStart)); -} - -fn mime_parse_boundary_regular<'a, 'b>( - boundary: &'b [u8], input: &'a [u8], -) -> IResult<&'a [u8], MimeParserState> { - let (input, _) = tag(boundary)(input)?; - let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?; - let (input, _) = char('\n')(input)?; - return Ok((input, MimeParserState::MimeHeader)); -} - -// Number of characters after boundary, without end of line, before changing state to streaming -const MIME_BOUNDARY_MAX_BEFORE_EOL: usize = 128; -const MIME_HEADER_MAX_LINE: usize = 4096; - -fn mime_parse_boundary_missing_eol<'a, 'b>( - boundary: &'b [u8], input: &'a [u8], -) -> IResult<&'a [u8], MimeParserState> { - let (input, _) = tag(boundary)(input)?; - let (input, _) = take(MIME_BOUNDARY_MAX_BEFORE_EOL)(input)?; - return Ok((input, MimeParserState::MimeBoundaryWaitingForEol)); -} - -fn mime_parse_boundary<'a, 'b>( - boundary: &'b [u8], input: &'a [u8], -) -> IResult<&'a [u8], MimeParserState> { - let r = mime_parse_boundary_regular(boundary, input); - if r.is_ok() { - return r; - } - let r2 = mime_parse_skip_line(input); - if r2.is_ok() { - return r2; - } - return mime_parse_boundary_missing_eol(boundary, input); -} - -fn mime_consume_until_eol(input: &[u8]) -> IResult<&[u8], bool> { - return alt((value(true, mime_parse_skip_line), value(false, rest)))(input); -} - -fn mime_parse_header_line(input: &[u8]) -> IResult<&[u8], &[u8]> { - let (input, name) = take_till(|ch: u8| ch == b':')(input)?; - let (input, _) = char(':')(input)?; - return Ok((input, name)); -} - -// s2 is already lower case -fn rs_equals_lowercase(s1: &[u8], s2: &[u8]) -> bool { - if s1.len() == s2.len() { - for i in 0..s1.len() { - if s1[i].to_ascii_lowercase() != s2[i] { - return false; - } - } - return true; - } - return false; -} - -fn mime_parse_headers<'a, 'b>( - ctx: &'b mut MimeStateHTTP, i: &'a [u8], -) -> IResult<&'a [u8], (MimeParserState, bool, bool)> { - let mut fileopen = false; - let mut errored = false; - let mut input = i; - while input.len() > 0 { - match take_until::<_, &[u8], nom7::error::Error<&[u8]>>("\r\n")(input) { - Ok((input2, line)) => { - match mime_parse_header_line(line) { - Ok((value, name)) => { - if rs_equals_lowercase(name, "content-disposition".as_bytes()) { - let mut sections_values = Vec::new(); - if let Ok(filename) = mime_find_header_token( - value, - "filename".as_bytes(), - &mut sections_values, - ) { - if filename.len() > 0 { - ctx.filename = Vec::with_capacity(filename.len()); - fileopen = true; - for c in filename { - // unescape - if *c != b'\\' { - ctx.filename.push(*c); - } - } - } - } - } - if value.len() == 0 { - errored = true; - } - } - _ => { - if line.len() > 0 { - errored = true; - } - } - } - let (input3, _) = tag("\r\n")(input2)?; - input = input3; - if line.len() == 0 || (line.len() == 1 && line[0] == b'\r') { - return Ok((input, (MimeParserState::MimeHeaderEnd, fileopen, errored))); - } - } - _ => { - // guard against too long header lines - if input.len() > MIME_HEADER_MAX_LINE { - return Ok(( - input, - ( - MimeParserState::MimeBoundaryWaitingForEol, - fileopen, - errored, - ), - )); - } - if input.len() < i.len() { - return Ok((input, (MimeParserState::MimeHeader, fileopen, errored))); - } // else only an incomplete line, ask for more - return Err(Err::Error(make_error(input, ErrorKind::Eof))); - } - } - } - return Ok((input, (MimeParserState::MimeHeader, fileopen, errored))); -} - -fn mime_consume_chunk<'a, 'b>(boundary: &'b [u8], input: &'a [u8]) -> IResult<&'a [u8], bool> { - let r: Result<(&[u8], &[u8]), Err>> = take_until("\r\n")(input); - match r { - Ok((input, line)) => { - let (input2, _) = tag("\r\n")(input)?; - if input2.len() < boundary.len() { - if input2 == &boundary[..input2.len()] { - if line.len() > 0 { - // consume as chunk up to eol (not consuming eol) - return Ok((input, false)); - } - // new line beignning like boundary, with nothin to consume as chunk : request more - return Err(Err::Error(make_error(input, ErrorKind::Eof))); - } - // not like boundary : consume everything as chunk - return Ok((&input[input.len()..], false)); - } // else - if &input2[..boundary.len()] == boundary { - // end of file with boundary, consume eol but do not consume boundary - return Ok((input2, true)); - } - // not like boundary : consume everything as chunk - return Ok((input2, false)); - } - _ => { - return Ok((&input[input.len()..], false)); - } - } -} - -pub const MIME_EVENT_FLAG_INVALID_HEADER: u32 = 0x01; -pub const MIME_EVENT_FLAG_NO_FILEDATA: u32 = 0x02; - -fn mime_process(ctx: &mut MimeStateHTTP, i: &[u8]) -> (MimeParserResult, u32, u32) { - let mut input = i; - let mut consumed = 0; - let mut warnings = 0; - while input.len() > 0 { - match ctx.state { - MimeParserState::MimeStart => { - if let Ok((rem, next)) = mime_parse_boundary(&ctx.boundary, input) { - ctx.state = next; - consumed += (input.len() - rem.len()) as u32; - input = rem; - } else { - return (MimeParserResult::MimeNeedsMore, consumed, warnings); - } - } - MimeParserState::MimeBoundaryWaitingForEol => { - if let Ok((rem, found)) = mime_consume_until_eol(input) { - if found { - ctx.state = MimeParserState::MimeHeader; - } - consumed += (input.len() - rem.len()) as u32; - input = rem; - } else { - // should never happen - return (MimeParserResult::MimeNeedsMore, consumed, warnings); - } - } - MimeParserState::MimeHeader => { - if let Ok((rem, (next, fileopen, err))) = mime_parse_headers(ctx, input) { - ctx.state = next; - consumed += (input.len() - rem.len()) as u32; - input = rem; - if err { - warnings |= MIME_EVENT_FLAG_INVALID_HEADER; - } - if fileopen { - return (MimeParserResult::MimeFileOpen, consumed, warnings); - } - } else { - return (MimeParserResult::MimeNeedsMore, consumed, warnings); - } - } - MimeParserState::MimeHeaderEnd => { - // check if we start with the boundary - // and transition to chunk, or empty file and back to start - if input.len() < ctx.boundary.len() { - if input == &ctx.boundary[..input.len()] { - return (MimeParserResult::MimeNeedsMore, consumed, warnings); - } - ctx.state = MimeParserState::MimeChunk; - } else { - if &input[..ctx.boundary.len()] == ctx.boundary { - ctx.state = MimeParserState::MimeStart; - if ctx.filename.len() > 0 { - warnings |= MIME_EVENT_FLAG_NO_FILEDATA; - } - ctx.filename.clear(); - return (MimeParserResult::MimeFileClose, consumed, warnings); - } else { - ctx.state = MimeParserState::MimeChunk; - } - } - } - MimeParserState::MimeChunk => { - if let Ok((rem, eof)) = mime_consume_chunk(&ctx.boundary, input) { - consumed += (input.len() - rem.len()) as u32; - if eof { - ctx.state = MimeParserState::MimeStart; - ctx.filename.clear(); - return (MimeParserResult::MimeFileClose, consumed, warnings); - } else { - // + 2 for \r\n - if rem.len() < ctx.boundary.len() + 2 { - return (MimeParserResult::MimeFileChunk, consumed, warnings); - } - input = rem; - } - } else { - return (MimeParserResult::MimeNeedsMore, consumed, warnings); - } - } - } - } - return (MimeParserResult::MimeNeedsMore, consumed, warnings); -} - -pub fn mime_state_init(i: &[u8]) -> Option { - let mut sections_values = Vec::new(); - match mime_find_header_token(i, "boundary".as_bytes(), &mut sections_values) { - Ok(value) => { - if value.len() <= RS_MIME_MAX_TOKEN_LEN { - let mut r = MimeStateHTTP::default(); - r.boundary = Vec::with_capacity(2 + value.len()); - // start wih 2 additional hyphens - r.boundary.push(b'-'); - r.boundary.push(b'-'); - for c in value { - // unescape - if *c != b'\\' { - r.boundary.push(*c); - } - } - return Some(r); - } - } - _ => {} - } - return None; -} - -#[no_mangle] -pub unsafe extern "C" fn rs_mime_state_init( - input: *const u8, input_len: u32, -) -> *mut MimeStateHTTP { - let slice = build_slice!(input, input_len as usize); - - if let Some(ctx) = mime_state_init(slice) { - let boxed = Box::new(ctx); - return Box::into_raw(boxed) as *mut _; - } - return std::ptr::null_mut(); -} - -#[no_mangle] -pub unsafe extern "C" fn rs_mime_parse( - ctx: &mut MimeStateHTTP, input: *const u8, input_len: u32, consumed: *mut u32, - warnings: *mut u32, -) -> MimeParserResult { - let slice = build_slice!(input, input_len as usize); - let (r, c, w) = mime_process(ctx, slice); - *consumed = c; - *warnings = w; - return r; -} - -#[no_mangle] -pub unsafe extern "C" fn rs_mime_state_get_filename( - ctx: &mut MimeStateHTTP, buffer: *mut *const u8, filename_len: *mut u16, -) { - if ctx.filename.len() > 0 { - *buffer = ctx.filename.as_ptr(); - if ctx.filename.len() < u16::MAX.into() { - *filename_len = ctx.filename.len() as u16; - } else { - *filename_len = u16::MAX; - } - } else { - *buffer = std::ptr::null_mut(); - *filename_len = 0; - } -} - -#[no_mangle] -pub unsafe extern "C" fn rs_mime_state_free(ctx: &mut MimeStateHTTP) { - // Just unbox... - std::mem::drop(Box::from_raw(ctx)); -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_mime_find_header_token() { - let mut outvec = Vec::new(); - let undelimok = mime_find_header_token( - "attachment; filename=test;".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(undelimok, Ok("test".as_bytes())); - - let delimok = mime_find_header_token( - "attachment; filename=\"test2\";".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(delimok, Ok("test2".as_bytes())); - - let escaped = mime_find_header_token( - "attachment; filename=\"test\\\"2\";".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(escaped, Ok("test\\\"2".as_bytes())); - - let evasion_othertoken = mime_find_header_token( - "attachment; dummy=\"filename=wrong\"; filename=real;".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(evasion_othertoken, Ok("real".as_bytes())); - - let evasion_suffixtoken = mime_find_header_token( - "attachment; notafilename=wrong; filename=good;".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(evasion_suffixtoken, Ok("good".as_bytes())); - - let badending = mime_find_header_token( - "attachment; filename=oksofar; badending".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(badending, Ok("oksofar".as_bytes())); - - let missend = mime_find_header_token( - "attachment; filename=test".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(missend, Ok("test".as_bytes())); - - let spaces = mime_find_header_token( - "attachment; filename=test me wrong".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(spaces, Ok("test me wrong".as_bytes())); - - assert_eq!(outvec.len(), 0); - let multi = mime_find_header_token( - "attachment; filename*0=abc; filename*1=\"def\";".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(multi, Ok("abcdef".as_bytes())); - outvec.clear(); - - let multi = mime_find_header_token( - "attachment; filename*1=456; filename*0=\"123\"".as_bytes(), - "filename".as_bytes(), - &mut outvec, - ); - assert_eq!(multi, Ok("123456".as_bytes())); - outvec.clear(); - } -} +pub mod mime; +pub mod smtp; +pub mod smtp_log; diff --git a/rust/src/mime/smtp.rs b/rust/src/mime/smtp.rs new file mode 100644 index 000000000000..947ea58a74e8 --- /dev/null +++ b/rust/src/mime/smtp.rs @@ -0,0 +1,854 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use super::mime; +use crate::core::StreamingBufferConfig; +use crate::filecontainer::FileContainer; +use digest::generic_array::{typenum::U16, GenericArray}; +use digest::Digest; +use digest::Update; +use md5::Md5; +use std::ffi::CStr; +use std::io; +use std::os::raw::c_uchar; + +#[repr(u8)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)] +pub enum MimeSmtpParserState { + MimeSmtpStart = 0, + MimeSmtpHeader = 1, + MimeSmtpBody = 2, + MimeSmtpParserError = 3, +} + +impl Default for MimeSmtpParserState { + fn default() -> Self { + MimeSmtpParserState::MimeSmtpStart + } +} + +#[derive(Debug, Default)] +pub struct MimeHeader { + pub name: Vec, + pub value: Vec, +} + +#[repr(u8)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)] +pub enum MimeSmtpMd5State { + MimeSmtpMd5Disabled = 0, + MimeSmtpMd5Inited = 1, + MimeSmtpMd5Started = 2, + MimeSmtpMd5Completed = 3, +} + +#[repr(u8)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)] +enum MimeSmtpContentType { + Message = 0, + PlainText = 1, + Html = 2, + Unknown = 3, +} + +impl Default for MimeSmtpContentType { + fn default() -> Self { + MimeSmtpContentType::Message + } +} + +#[derive(Debug)] +pub struct MimeStateSMTP<'a> { + pub(crate) state_flag: MimeSmtpParserState, + pub(crate) headers: Vec, + pub(crate) main_headers_nb: usize, + filename: Vec, + pub(crate) attachments: Vec>, + pub(crate) urls: Vec>, + boundaries: Vec>, + encoding: MimeSmtpEncoding, + decoder: Option, + content_type: MimeSmtpContentType, + decoded_line: Vec, + // small buffer for end of line + // waiting to see if it is part of the boundary + bufeol: [u8; 2], + bufeolen: u8, + files: &'a mut FileContainer, + sbcfg: *const StreamingBufferConfig, + md5: md5::Md5, + pub(crate) md5_state: MimeSmtpMd5State, + pub(crate) md5_result: GenericArray, +} + +#[derive(Debug)] +pub struct MimeBase64Decoder { + tmp: [u8; 4], + nb: u8, +} + +impl MimeBase64Decoder { + pub fn new() -> MimeBase64Decoder { + MimeBase64Decoder { tmp: [0; 4], nb: 0 } + } +} + +impl Default for MimeBase64Decoder { + fn default() -> Self { + Self::new() + } +} + +pub fn mime_smtp_state_init( + files: &mut FileContainer, sbcfg: *const StreamingBufferConfig, +) -> Option { + let r = MimeStateSMTP { + state_flag: MimeSmtpParserState::MimeSmtpStart, + headers: Vec::new(), + main_headers_nb: 0, + filename: Vec::new(), + attachments: Vec::new(), + urls: Vec::new(), + boundaries: Vec::new(), + decoded_line: Vec::new(), + encoding: MimeSmtpEncoding::Plain, + decoder: None, + content_type: MimeSmtpContentType::Message, + bufeol: [0; 2], + bufeolen: 0, + files, + sbcfg, + md5: Md5::new(), + md5_state: MimeSmtpMd5State::MimeSmtpMd5Disabled, + md5_result: [0; 16].into(), + }; + return Some(r); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpStateInit( + files: &mut FileContainer, sbcfg: *const StreamingBufferConfig, +) -> *mut MimeStateSMTP { + if let Some(ctx) = mime_smtp_state_init(files, sbcfg) { + let boxed = Box::new(ctx); + return Box::into_raw(boxed) as *mut _; + } + return std::ptr::null_mut(); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpStateFree(ctx: &mut MimeStateSMTP) { + // Just unbox... + std::mem::drop(Box::from_raw(ctx)); +} + +#[repr(u8)] +#[derive(Copy, Clone, PartialOrd, PartialEq, Eq)] +pub enum MimeSmtpParserResult { + MimeSmtpNeedsMore = 0, + MimeSmtpFileOpen = 1, + MimeSmtpFileClose = 2, + MimeSmtpFileChunk = 3, +} + +#[repr(u8)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)] +pub enum MimeSmtpEncoding { + Plain = 0, + Base64 = 1, + QuotedPrintable = 2, +} + +impl Default for MimeSmtpEncoding { + fn default() -> Self { + MimeSmtpEncoding::Plain + } +} + +// Cannot use BIT_U32 macros as they do not get exported by cbindgen :-/ +pub const MIME_ANOM_INVALID_BASE64: u32 = 0x1; +pub const MIME_ANOM_INVALID_QP: u32 = 0x2; +pub const MIME_ANOM_LONG_LINE: u32 = 0x4; +pub const MIME_ANOM_LONG_ENC_LINE: u32 = 0x8; +pub const MIME_ANOM_LONG_HEADER_NAME: u32 = 0x10; +pub const MIME_ANOM_LONG_HEADER_VALUE: u32 = 0x20; +//unused pub const MIME_ANOM_MALFORMED_MSG: u32 = 0x40; +pub const MIME_ANOM_LONG_BOUNDARY: u32 = 0x80; +pub const MIME_ANOM_LONG_FILENAME: u32 = 0x100; + +fn mime_smtp_process_headers(ctx: &mut MimeStateSMTP) -> (u32, bool) { + let mut sections_values = Vec::new(); + let mut warnings = 0; + let mut encap = false; + for h in &ctx.headers[ctx.main_headers_nb..] { + if mime::slice_equals_lowercase(&h.name, b"content-disposition") { + if ctx.filename.is_empty() { + if let Some(value) = + mime::mime_find_header_token(&h.value, b"filename", &mut sections_values) + { + let value = if value.len() > mime::RS_MIME_MAX_TOKEN_LEN { + warnings |= MIME_ANOM_LONG_FILENAME; + &value[..mime::RS_MIME_MAX_TOKEN_LEN] + } else { + value + }; + ctx.filename.extend_from_slice(value); + let mut newname = Vec::new(); + newname.extend_from_slice(value); + ctx.attachments.push(newname); + sections_values.clear(); + } + } + } else if mime::slice_equals_lowercase(&h.name, b"content-transfer-encoding") { + if mime::slice_equals_lowercase(&h.value, b"base64") { + ctx.encoding = MimeSmtpEncoding::Base64; + ctx.decoder = Some(MimeBase64Decoder::new()); + } else if mime::slice_equals_lowercase(&h.value, b"quoted-printable") { + ctx.encoding = MimeSmtpEncoding::QuotedPrintable; + } + } else if mime::slice_equals_lowercase(&h.name, b"content-type") { + if ctx.filename.is_empty() { + if let Some(value) = + mime::mime_find_header_token(&h.value, b"name", &mut sections_values) + { + let value = if value.len() > mime::RS_MIME_MAX_TOKEN_LEN { + warnings |= MIME_ANOM_LONG_FILENAME; + &value[..mime::RS_MIME_MAX_TOKEN_LEN] + } else { + value + }; + ctx.filename.extend_from_slice(value); + let mut newname = Vec::new(); + newname.extend_from_slice(value); + ctx.attachments.push(newname); + sections_values.clear(); + } + } + if let Some(value) = + mime::mime_find_header_token(&h.value, b"boundary", &mut sections_values) + { + // start wih 2 additional hyphens + let mut boundary = Vec::new(); + boundary.push(b'-'); + boundary.push(b'-'); + boundary.extend_from_slice(value); + ctx.boundaries.push(boundary); + if value.len() > MAX_BOUNDARY_LEN { + warnings |= MIME_ANOM_LONG_BOUNDARY; + } + sections_values.clear(); + } + let ct = if let Some(x) = h.value.iter().position(|&x| x == b';') { + &h.value[..x] + } else { + &h.value + }; + match ct { + b"text/plain" => { + ctx.content_type = MimeSmtpContentType::PlainText; + } + b"text/html" => { + ctx.content_type = MimeSmtpContentType::Html; + } + _ => { + if ct.starts_with(b"message/") { + encap = true; + } + ctx.content_type = MimeSmtpContentType::Unknown; + } + } + } + } + return (warnings, encap); +} + +extern "C" { + // Defined in util-file.h + pub fn FileAppendData( + c: *mut FileContainer, sbcfg: *const StreamingBufferConfig, data: *const c_uchar, + data_len: u32, + ) -> std::os::raw::c_int; + // Defined in util-spm-bs.h + pub fn BasicSearchNocaseIndex( + data: *const c_uchar, data_len: u32, needle: *const c_uchar, needle_len: u16, + ) -> u32; +} + +fn hex(i: u8) -> Option { + if i.is_ascii_digit() { + return Some(i - b'0'); + } + if (b'A'..=b'F').contains(&i) { + return Some(i - b'A' + 10); + } + return None; +} + +const SMTP_MIME_MAX_DECODED_LINE_LENGTH: usize = 8192; + +fn mime_smtp_finish_url(input: &[u8]) -> &[u8] { + if let Some(x) = input.iter().position(|&x| { + x == b' ' || x == b'"' || x == b'\'' || x == b'<' || x == b'>' || x == b']' || x == b'\t' + }) { + return &input[..x]; + } + return input; +} + +fn mime_smtp_extract_urls(urls: &mut Vec>, input_start: &[u8]) { + //TODO optimize later : use mpm + for s in unsafe { MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES.iter() } { + let mut input = input_start; + let mut start = unsafe { + BasicSearchNocaseIndex( + input.as_ptr(), + input.len() as u32, + s.as_ptr(), + s.len() as u16, + ) + }; + while (start as usize) < input.len() { + let url = mime_smtp_finish_url(&input[start as usize..]); + let mut urlv = Vec::with_capacity(url.len()); + if unsafe { !MIME_SMTP_CONFIG_LOG_URL_SCHEME } { + urlv.extend_from_slice(&url[s.len()..]); + } else { + urlv.extend_from_slice(url); + } + urls.push(urlv); + input = &input[start as usize + url.len()..]; + start = unsafe { + BasicSearchNocaseIndex( + input.as_ptr(), + input.len() as u32, + s.as_ptr(), + s.len() as u16, + ) + }; + } + } +} + +fn mime_smtp_find_url_strings(ctx: &mut MimeStateSMTP, input_new: &[u8]) { + if unsafe { !MIME_SMTP_CONFIG_EXTRACT_URLS } { + return; + } + + let mut input = input_new; + // use previosly buffered beginning of line if any + if !ctx.decoded_line.is_empty() { + ctx.decoded_line.extend_from_slice(input_new); + input = &ctx.decoded_line; + } + // no input, no url + if input.is_empty() { + return; + } + + if input[input.len() - 1] == b'\n' || input.len() > SMTP_MIME_MAX_DECODED_LINE_LENGTH { + // easy case, no buffering to do + mime_smtp_extract_urls(&mut ctx.urls, input); + if !ctx.decoded_line.is_empty() { + ctx.decoded_line.clear() + } + } else if let Some(x) = input.iter().rev().position(|&x| x == b'\n') { + input = &input[..x]; + mime_smtp_extract_urls(&mut ctx.urls, input); + if !ctx.decoded_line.is_empty() { + ctx.decoded_line.drain(0..x); + } else { + ctx.decoded_line.extend_from_slice(&input_new[x..]); + } + } // else no end of line, already buffered for next input... +} + +fn mime_base64_map(input: u8) -> io::Result { + match input { + 43 => Ok(62), // + + 47 => Ok(63), // / + 48 => Ok(52), // 0 + 49 => Ok(53), // 1 + 50 => Ok(54), // 2 + 51 => Ok(55), // 3 + 52 => Ok(56), // 4 + 53 => Ok(57), // 5 + 54 => Ok(58), // 6 + 55 => Ok(59), // 7 + 56 => Ok(60), // 8 + 57 => Ok(61), // 9 + 65 => Ok(0), // A + 66 => Ok(1), // B + 67 => Ok(2), // C + 68 => Ok(3), // D + 69 => Ok(4), // E + 70 => Ok(5), // F + 71 => Ok(6), // G + 72 => Ok(7), // H + 73 => Ok(8), // I + 74 => Ok(9), // J + 75 => Ok(10), // K + 76 => Ok(11), // L + 77 => Ok(12), // M + 78 => Ok(13), // N + 79 => Ok(14), // O + 80 => Ok(15), // P + 81 => Ok(16), // Q + 82 => Ok(17), // R + 83 => Ok(18), // S + 84 => Ok(19), // T + 85 => Ok(20), // U + 86 => Ok(21), // V + 87 => Ok(22), // W + 88 => Ok(23), // X + 89 => Ok(24), // Y + 90 => Ok(25), // Z + 97 => Ok(26), // a + 98 => Ok(27), // b + 99 => Ok(28), // c + 100 => Ok(29), // d + 101 => Ok(30), // e + 102 => Ok(31), // f + 103 => Ok(32), // g + 104 => Ok(33), // h + 105 => Ok(34), // i + 106 => Ok(35), // j + 107 => Ok(36), // k + 108 => Ok(37), // l + 109 => Ok(38), // m + 110 => Ok(39), // n + 111 => Ok(40), // o + 112 => Ok(41), // p + 113 => Ok(42), // q + 114 => Ok(43), // r + 115 => Ok(44), // s + 116 => Ok(45), // t + 117 => Ok(46), // u + 118 => Ok(47), // v + 119 => Ok(48), // w + 120 => Ok(49), // x + 121 => Ok(50), // y + 122 => Ok(51), // z + _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid base64")), + } +} + +fn mime_base64_decode(decoder: &mut MimeBase64Decoder, input: &[u8]) -> io::Result> { + let mut i = input; + let maxlen = ((decoder.nb as usize + i.len()) * 3) / 4; + let mut r = vec![0; maxlen]; + let mut offset = 0; + while !i.is_empty() { + while decoder.nb < 4 && !i.is_empty() { + if mime_base64_map(i[0]).is_ok() || i[0] == b'=' { + decoder.tmp[decoder.nb as usize] = i[0]; + decoder.nb += 1; + } + i = &i[1..]; + } + if decoder.nb == 4 { + decoder.tmp[0] = mime_base64_map(decoder.tmp[0])?; + decoder.tmp[1] = mime_base64_map(decoder.tmp[1])?; + if decoder.tmp[2] == b'=' { + r[offset] = (decoder.tmp[0] << 2) | (decoder.tmp[1] >> 4); + offset += 1; + } else { + decoder.tmp[2] = mime_base64_map(decoder.tmp[2])?; + if decoder.tmp[3] == b'=' { + r[offset] = (decoder.tmp[0] << 2) | (decoder.tmp[1] >> 4); + r[offset + 1] = (decoder.tmp[1] << 4) | (decoder.tmp[2] >> 2); + offset += 2; + } else { + decoder.tmp[3] = mime_base64_map(decoder.tmp[3])?; + r[offset] = (decoder.tmp[0] << 2) | (decoder.tmp[1] >> 4); + r[offset + 1] = (decoder.tmp[1] << 4) | (decoder.tmp[2] >> 2); + r[offset + 2] = (decoder.tmp[2] << 6) | decoder.tmp[3]; + offset += 3; + } + } + decoder.nb = 0; + } + } + r.truncate(offset); + return Ok(r); +} + +const MAX_LINE_LEN: u32 = 998; // Def in RFC 2045, excluding CRLF sequence +const MAX_ENC_LINE_LEN: usize = 76; /* Def in RFC 2045, excluding CRLF sequence */ +const MAX_HEADER_NAME: usize = 75; /* 75 + ":" = 76 */ +const MAX_HEADER_VALUE: usize = 2000; /* Default - arbitrary limit */ +const MAX_BOUNDARY_LEN: usize = 254; + +fn mime_smtp_parse_line( + ctx: &mut MimeStateSMTP, i: &[u8], full: &[u8], +) -> (MimeSmtpParserResult, u32) { + if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Started { + Update::update(&mut ctx.md5, full); + } + let mut warnings = 0; + match ctx.state_flag { + MimeSmtpParserState::MimeSmtpStart => { + if unsafe { MIME_SMTP_CONFIG_BODY_MD5 } + && ctx.md5_state != MimeSmtpMd5State::MimeSmtpMd5Started + { + ctx.md5 = Md5::new(); + ctx.md5_state = MimeSmtpMd5State::MimeSmtpMd5Inited; + } + if i.is_empty() { + let (w, encap_msg) = mime_smtp_process_headers(ctx); + warnings |= w; + if ctx.main_headers_nb == 0 { + ctx.main_headers_nb = ctx.headers.len(); + } + if encap_msg { + ctx.state_flag = MimeSmtpParserState::MimeSmtpStart; + ctx.headers.truncate(ctx.main_headers_nb); + return (MimeSmtpParserResult::MimeSmtpNeedsMore, warnings); + } + ctx.state_flag = MimeSmtpParserState::MimeSmtpBody; + return (MimeSmtpParserResult::MimeSmtpFileOpen, warnings); + } else if let Ok((value, name)) = mime::mime_parse_header_line(i) { + ctx.state_flag = MimeSmtpParserState::MimeSmtpHeader; + let mut h = MimeHeader::default(); + h.name.extend_from_slice(name); + h.value.extend_from_slice(value); + if h.name.len() > MAX_HEADER_NAME { + warnings |= MIME_ANOM_LONG_HEADER_NAME; + } + if h.value.len() > MAX_HEADER_VALUE { + warnings |= MIME_ANOM_LONG_HEADER_VALUE; + } + ctx.headers.push(h); + } // else event ? + } + MimeSmtpParserState::MimeSmtpHeader => { + if i.is_empty() { + let (w, encap_msg) = mime_smtp_process_headers(ctx); + warnings |= w; + if ctx.main_headers_nb == 0 { + ctx.main_headers_nb = ctx.headers.len(); + } + if encap_msg { + ctx.state_flag = MimeSmtpParserState::MimeSmtpStart; + ctx.headers.truncate(ctx.main_headers_nb); + return (MimeSmtpParserResult::MimeSmtpNeedsMore, warnings); + } + ctx.state_flag = MimeSmtpParserState::MimeSmtpBody; + return (MimeSmtpParserResult::MimeSmtpFileOpen, warnings); + } else if i[0] == b' ' || i[0] == b'\t' { + let last = ctx.headers.len() - 1; + ctx.headers[last].value.extend_from_slice(&i[1..]); + } else if let Ok((value, name)) = mime::mime_parse_header_line(i) { + let mut h = MimeHeader::default(); + h.name.extend_from_slice(name); + h.value.extend_from_slice(value); + if h.name.len() > MAX_HEADER_NAME { + warnings |= MIME_ANOM_LONG_HEADER_NAME; + } + if h.value.len() > MAX_HEADER_VALUE { + warnings |= MIME_ANOM_LONG_HEADER_VALUE; + } + ctx.headers.push(h); + } + } + MimeSmtpParserState::MimeSmtpBody => { + if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Inited { + ctx.md5_state = MimeSmtpMd5State::MimeSmtpMd5Started; + Update::update(&mut ctx.md5, full); + } + let boundary = ctx.boundaries.last(); + if let Some(b) = boundary { + if i.len() >= b.len() && &i[..b.len()] == b { + if ctx.encoding == MimeSmtpEncoding::Base64 + && unsafe { MIME_SMTP_CONFIG_DECODE_BASE64 } + { + if let Some(ref mut decoder) = &mut ctx.decoder { + if decoder.nb > 0 { + // flush the base64 buffer with padding + let mut v = Vec::new(); + for _i in 0..4 - decoder.nb { + v.push(b'='); + } + if let Ok(dec) = mime_base64_decode(decoder, &v) { + unsafe { + FileAppendData( + ctx.files, + ctx.sbcfg, + dec.as_ptr(), + dec.len() as u32, + ); + } + } + } + } + } + ctx.state_flag = MimeSmtpParserState::MimeSmtpStart; + let toclose = !ctx.filename.is_empty(); + ctx.filename.clear(); + ctx.headers.truncate(ctx.main_headers_nb); + ctx.encoding = MimeSmtpEncoding::Plain; + if i.len() >= b.len() + 2 && i[b.len()] == b'-' && i[b.len() + 1] == b'-' { + ctx.boundaries.pop(); + } + if toclose { + return (MimeSmtpParserResult::MimeSmtpFileClose, 0); + } + return (MimeSmtpParserResult::MimeSmtpNeedsMore, 0); + } + } + if ctx.filename.is_empty() { + if ctx.content_type == MimeSmtpContentType::PlainText + || ctx.content_type == MimeSmtpContentType::Html + || ctx.content_type == MimeSmtpContentType::Message + { + mime_smtp_find_url_strings(ctx, full); + } + return (MimeSmtpParserResult::MimeSmtpNeedsMore, 0); + } + match ctx.encoding { + MimeSmtpEncoding::Plain => { + mime_smtp_find_url_strings(ctx, full); + if ctx.bufeolen > 0 { + unsafe { + FileAppendData( + ctx.files, + ctx.sbcfg, + ctx.bufeol.as_ptr(), + ctx.bufeol.len() as u32, + ); + } + } + unsafe { + FileAppendData(ctx.files, ctx.sbcfg, i.as_ptr(), i.len() as u32); + } + ctx.bufeolen = (full.len() - i.len()) as u8; + if ctx.bufeolen > 0 { + ctx.bufeol[..ctx.bufeolen as usize].copy_from_slice(&full[i.len()..]); + } + } + MimeSmtpEncoding::Base64 => { + if unsafe { MIME_SMTP_CONFIG_DECODE_BASE64 } { + if let Some(ref mut decoder) = &mut ctx.decoder { + if i.len() > MAX_ENC_LINE_LEN { + warnings |= MIME_ANOM_LONG_ENC_LINE; + } + if let Ok(dec) = mime_base64_decode(decoder, i) { + mime_smtp_find_url_strings(ctx, &dec); + unsafe { + FileAppendData( + ctx.files, + ctx.sbcfg, + dec.as_ptr(), + dec.len() as u32, + ); + } + } else { + warnings |= MIME_ANOM_INVALID_BASE64; + } + } + } + } + MimeSmtpEncoding::QuotedPrintable => { + if unsafe { MIME_SMTP_CONFIG_DECODE_QUOTED } { + if i.len() > MAX_ENC_LINE_LEN { + warnings |= MIME_ANOM_LONG_ENC_LINE; + } + let mut c = 0; + let mut eol_equal = false; + let mut quoted_buffer = Vec::with_capacity(i.len()); + while c < i.len() { + if i[c] == b'=' { + if c == i.len() - 1 { + eol_equal = true; + break; + } else if c + 2 >= i.len() { + // log event ? + warnings |= MIME_ANOM_INVALID_QP; + break; + } + if let Some(v) = hex(i[c + 1]) { + if let Some(v2) = hex(i[c + 2]) { + quoted_buffer.push((v << 4) | v2); + } else { + warnings |= MIME_ANOM_INVALID_QP; + } + } else { + warnings |= MIME_ANOM_INVALID_QP; + } + c += 3; + } else { + quoted_buffer.push(i[c]); + c += 1; + } + } + if !eol_equal { + quoted_buffer.extend_from_slice(&full[i.len()..]); + } + mime_smtp_find_url_strings(ctx, "ed_buffer); + unsafe { + FileAppendData( + ctx.files, + ctx.sbcfg, + quoted_buffer.as_ptr(), + quoted_buffer.len() as u32, + ); + } + } + } + } + return (MimeSmtpParserResult::MimeSmtpFileChunk, warnings); + } + _ => {} + } + return (MimeSmtpParserResult::MimeSmtpNeedsMore, warnings); +} + +#[no_mangle] +pub unsafe extern "C" fn SCSmtpMimeParseLine( + input: *const u8, input_len: u32, delim_len: u8, warnings: *mut u32, ctx: &mut MimeStateSMTP, +) -> MimeSmtpParserResult { + let full_line = build_slice!(input, input_len as usize + delim_len as usize); + let line = &full_line[..input_len as usize]; + let (r, w) = mime_smtp_parse_line(ctx, line, full_line); + *warnings = w; + if input_len > MAX_LINE_LEN { + *warnings |= MIME_ANOM_LONG_LINE; + } + return r; +} + +fn mime_smtp_complete(ctx: &mut MimeStateSMTP) { + if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Started { + ctx.md5_state = MimeSmtpMd5State::MimeSmtpMd5Completed; + ctx.md5_result = ctx.md5.finalize_reset(); + } +} + +#[no_mangle] +pub unsafe extern "C" fn SCSmtpMimeComplete(ctx: &mut MimeStateSMTP) { + mime_smtp_complete(ctx); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpGetState(ctx: &mut MimeStateSMTP) -> MimeSmtpParserState { + return ctx.state_flag; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpGetFilename( + ctx: &mut MimeStateSMTP, buffer: *mut *const u8, filename_len: *mut u16, +) { + if !ctx.filename.is_empty() { + *buffer = ctx.filename.as_ptr(); + if ctx.filename.len() < u16::MAX.into() { + *filename_len = ctx.filename.len() as u16; + } else { + *filename_len = u16::MAX; + } + } else { + *buffer = std::ptr::null_mut(); + *filename_len = 0; + } +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpGetHeader( + ctx: &mut MimeStateSMTP, str: *const std::os::raw::c_char, buffer: *mut *const u8, + buffer_len: *mut u32, +) -> bool { + let name: &CStr = CStr::from_ptr(str); //unsafe + for h in &ctx.headers[ctx.main_headers_nb..] { + if mime::slice_equals_lowercase(&h.name, name.to_bytes()) { + *buffer = h.value.as_ptr(); + *buffer_len = h.value.len() as u32; + return true; + } + } + *buffer = std::ptr::null_mut(); + *buffer_len = 0; + return false; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpGetHeaderName( + ctx: &mut MimeStateSMTP, buffer: *mut *const u8, buffer_len: *mut u32, num: u32, +) -> bool { + if num as usize + ctx.main_headers_nb < ctx.headers.len() { + *buffer = ctx.headers[ctx.main_headers_nb + num as usize] + .name + .as_ptr(); + *buffer_len = ctx.headers[ctx.main_headers_nb + num as usize].name.len() as u32; + return true; + } + *buffer = std::ptr::null_mut(); + *buffer_len = 0; + return false; +} + +static mut MIME_SMTP_CONFIG_DECODE_BASE64: bool = true; +static mut MIME_SMTP_CONFIG_DECODE_QUOTED: bool = true; +static mut MIME_SMTP_CONFIG_BODY_MD5: bool = false; +static mut MIME_SMTP_CONFIG_HEADER_VALUE_DEPTH: u32 = 0; +static mut MIME_SMTP_CONFIG_EXTRACT_URLS: bool = true; +static mut MIME_SMTP_CONFIG_LOG_URL_SCHEME: bool = false; +static mut MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES: Vec<&str> = Vec::new(); + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigDecodeBase64(val: std::os::raw::c_int) { + MIME_SMTP_CONFIG_DECODE_BASE64 = val != 0; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigDecodeQuoted(val: std::os::raw::c_int) { + MIME_SMTP_CONFIG_DECODE_QUOTED = val != 0; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigExtractUrls(val: std::os::raw::c_int) { + MIME_SMTP_CONFIG_EXTRACT_URLS = val != 0; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigLogUrlScheme(val: std::os::raw::c_int) { + MIME_SMTP_CONFIG_LOG_URL_SCHEME = val != 0; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigBodyMd5(val: std::os::raw::c_int) { + MIME_SMTP_CONFIG_BODY_MD5 = val != 0; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigHeaderValueDepth(val: u32) { + MIME_SMTP_CONFIG_HEADER_VALUE_DEPTH = val; +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigExtractUrlsSchemeReset() { + MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES.clear(); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpConfigExtractUrlsSchemeAdd( + str: *const std::os::raw::c_char, +) -> std::os::raw::c_int { + let scheme: &CStr = CStr::from_ptr(str); //unsafe + if let Ok(s) = scheme.to_str() { + MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES.push(s); + return 0; + } + return -1; +} diff --git a/rust/src/mime/smtp_log.rs b/rust/src/mime/smtp_log.rs new file mode 100644 index 000000000000..de8ab6d2341c --- /dev/null +++ b/rust/src/mime/smtp_log.rs @@ -0,0 +1,241 @@ +/* Copyright (C) 2022 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use super::mime; +use crate::jsonbuilder::{JsonBuilder, JsonError}; +use crate::mime::smtp::{MimeSmtpMd5State, MimeStateSMTP}; +use digest::Digest; +use digest::Update; +use md5::Md5; +use std::ffi::CStr; + +fn log_subject_md5(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> Result<(), JsonError> { + for h in &ctx.headers[..ctx.main_headers_nb] { + if mime::slice_equals_lowercase(&h.name, b"subject") { + let hash = format!("{:x}", Md5::new().chain(&h.value).finalize()); + js.set_string("subject_md5", &hash)?; + break; + } + } + return Ok(()); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpLogSubjectMd5( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, +) -> bool { + return log_subject_md5(js, ctx).is_ok(); +} + +fn log_body_md5(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> Result<(), JsonError> { + if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Completed { + let hash = format!("{:x}", ctx.md5_result); + js.set_string("body_md5", &hash)?; + } + return Ok(()); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpLogBodyMd5( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, +) -> bool { + return log_body_md5(js, ctx).is_ok(); +} + +fn log_field_array( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, c: &str, e: &str, +) -> Result<(), JsonError> { + let mark = js.get_mark(); + let mut found = false; + js.open_array(c)?; + + for h in &ctx.headers[..ctx.main_headers_nb] { + if mime::slice_equals_lowercase(&h.name, e.as_bytes()) { + found = true; + js.append_string(&String::from_utf8_lossy(&h.value))?; + } + } + + if found { + js.close()?; + } else { + js.restore_mark(&mark)?; + } + + return Ok(()); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpLogFieldArray( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, email: *const std::os::raw::c_char, + config: *const std::os::raw::c_char, +) -> bool { + let e: &CStr = CStr::from_ptr(email); //unsafe + if let Ok(email_field) = e.to_str() { + let c: &CStr = CStr::from_ptr(config); //unsafe + if let Ok(config_field) = c.to_str() { + return log_field_array(js, ctx, config_field, email_field).is_ok(); + } + } + return false; +} + +enum FieldCommaState { + Start = 0, // skip leading spaces + Field = 1, + Quoted = 2, // do not take comma for split in quote +} + +fn log_field_comma( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, c: &str, e: &str, +) -> Result<(), JsonError> { + for h in &ctx.headers[..ctx.main_headers_nb] { + if mime::slice_equals_lowercase(&h.name, e.as_bytes()) { + let mark = js.get_mark(); + let mut has_not_empty_field = false; + js.open_array(c)?; + let mut start = 0; + let mut state = FieldCommaState::Start; + for i in 0..h.value.len() { + match state { + FieldCommaState::Start => { + if h.value[i] == b' ' || h.value[i] == b'\t' { + start += 1; + } else if h.value[i] == b'"' { + state = FieldCommaState::Quoted; + } else { + state = FieldCommaState::Field; + } + } + FieldCommaState::Field => { + if h.value[i] == b',' { + if i > start { + js.append_string(&String::from_utf8_lossy(&h.value[start..i]))?; + has_not_empty_field = true; + } + start = i + 1; + state = FieldCommaState::Start; + } else if h.value[i] == b'"' { + state = FieldCommaState::Quoted; + } + } + FieldCommaState::Quoted => { + if h.value[i] == b'"' { + state = FieldCommaState::Field; + } + } + } + } + if h.value.len() > start { + // do not log empty string + js.append_string(&String::from_utf8_lossy(&h.value[start..]))?; + has_not_empty_field = true; + } + if has_not_empty_field { + js.close()?; + } else { + js.restore_mark(&mark)?; + } + break; + } + } + return Ok(()); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpLogFieldComma( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, email: *const std::os::raw::c_char, + config: *const std::os::raw::c_char, +) -> bool { + let e: &CStr = CStr::from_ptr(email); //unsafe + if let Ok(email_field) = e.to_str() { + let c: &CStr = CStr::from_ptr(config); //unsafe + if let Ok(config_field) = c.to_str() { + return log_field_comma(js, ctx, config_field, email_field).is_ok(); + } + } + return false; +} + +fn log_field_string( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, c: &str, e: &str, +) -> Result<(), JsonError> { + for h in &ctx.headers[..ctx.main_headers_nb] { + if mime::slice_equals_lowercase(&h.name, e.as_bytes()) { + js.set_string(c, &String::from_utf8_lossy(&h.value))?; + break; + } + } + return Ok(()); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpLogFieldString( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, email: *const std::os::raw::c_char, + config: *const std::os::raw::c_char, +) -> bool { + let e: &CStr = CStr::from_ptr(email); //unsafe + if let Ok(email_field) = e.to_str() { + let c: &CStr = CStr::from_ptr(config); //unsafe + if let Ok(config_field) = c.to_str() { + return log_field_string(js, ctx, config_field, email_field).is_ok(); + } + } + return false; +} + +fn log_data_header( + js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, hname: &str, +) -> Result<(), JsonError> { + for h in &ctx.headers[..ctx.main_headers_nb] { + if mime::slice_equals_lowercase(&h.name, hname.as_bytes()) { + js.set_string(hname, &String::from_utf8_lossy(&h.value))?; + break; + } + } + return Ok(()); +} + +fn log_data(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> Result<(), JsonError> { + log_data_header(js, ctx, "from")?; + log_field_comma(js, ctx, "to", "to")?; + log_field_comma(js, ctx, "cc", "cc")?; + + js.set_string("status", "PARSE_DONE")?; + + if !ctx.attachments.is_empty() { + js.open_array("attachment")?; + for a in &ctx.attachments { + js.append_string(&String::from_utf8_lossy(a))?; + } + js.close()?; + } + if !ctx.urls.is_empty() { + js.open_array("url")?; + for a in ctx.urls.iter().rev() { + js.append_string(&String::from_utf8_lossy(a))?; + } + js.close()?; + } + + return Ok(()); +} + +#[no_mangle] +pub unsafe extern "C" fn SCMimeSmtpLogData(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> bool { + return log_data(js, ctx).is_ok(); +} diff --git a/src/Makefile.am b/src/Makefile.am index 2f4aa22d6a1d..9c0675d4b795 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -521,7 +521,6 @@ noinst_HEADERS = \ util-datalink.h \ util-debug-filters.h \ util-debug.h \ - util-decode-mime.h \ util-detect.h \ util-device.h \ util-dpdk.h \ @@ -1117,7 +1116,6 @@ libsuricata_c_a_SOURCES = \ util-datalink.c \ util-debug.c \ util-debug-filters.c \ - util-decode-mime.c \ util-detect.c \ util-device.c \ util-dpdk.c \ diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index acdb8739d923..3d8d17102874 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -373,7 +373,7 @@ static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud) HTPFree(htud->response_headers_raw, htud->response_headers_raw_len); AppLayerDecoderEventsFreeEvents(&htud->tx_data.events); if (htud->mime_state) - rs_mime_state_free(htud->mime_state); + SCMimeStateFree(htud->mime_state); if (htud->tx_data.de_state != NULL) { DetectEngineStateFree(htud->tx_data.de_state); } @@ -1136,7 +1136,7 @@ static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud) htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers, "Content-Type"); if (h != NULL && bstr_len(h->value) > 0) { - htud->mime_state = rs_mime_state_init(bstr_ptr(h->value), bstr_len(h->value)); + htud->mime_state = SCMimeStateInit(bstr_ptr(h->value), bstr_len(h->value)); if (htud->mime_state) { htud->tsflags |= HTP_BOUNDARY_SET; SCReturnInt(1); @@ -1211,7 +1211,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, // keep parsing mime and use callbacks when needed while (cur_buf_len > 0) { MimeParserResult r = - rs_mime_parse(htud->mime_state, cur_buf, cur_buf_len, &consumed, &warnings); + SCMimeParse(htud->mime_state, cur_buf, cur_buf_len, &consumed, &warnings); DEBUG_VALIDATE_BUG_ON(consumed > cur_buf_len); htud->request_body.body_parsed += consumed; if (warnings) { @@ -1230,7 +1230,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, goto end; case MimeFileOpen: // get filename owned by mime state - rs_mime_state_get_filename(htud->mime_state, &filename, &filename_len); + SCMimeStateGetFilename(htud->mime_state, &filename, &filename_len); if (filename_len > 0) { htud->tsflags |= HTP_FILENAME_SET; htud->tsflags &= ~HTP_DONTSTORE; @@ -1268,7 +1268,6 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, } htud->tsflags &= ~HTP_FILENAME_SET; break; - // TODO event on parsing error ? } cur_buf += consumed; cur_buf_len -= consumed; @@ -5609,7 +5608,7 @@ static int HTPBodyReassemblyTest01(void) printf("REASSCHUNK END: \n"); #endif - htud.mime_state = rs_mime_state_init((const uint8_t *)"multipart/form-data; boundary=toto", + htud.mime_state = SCMimeStateInit((const uint8_t *)"multipart/form-data; boundary=toto", strlen("multipart/form-data; boundary=toto")); FAIL_IF_NULL(htud.mime_state); htud.tsflags |= HTP_BOUNDARY_SET; diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 3558ff0009c5..b5eb04ba9e21 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -137,7 +137,6 @@ SCEnumCharMap smtp_decoder_event_table[] = { /* MIME Events */ { "MIME_PARSE_FAILED", SMTP_DECODER_EVENT_MIME_PARSE_FAILED }, - { "MIME_MALFORMED_MSG", SMTP_DECODER_EVENT_MIME_MALFORMED_MSG }, { "MIME_INVALID_BASE64", SMTP_DECODER_EVENT_MIME_INVALID_BASE64 }, { "MIME_INVALID_QP", SMTP_DECODER_EVENT_MIME_INVALID_QP }, { "MIME_LONG_LINE", SMTP_DECODER_EVENT_MIME_LONG_LINE }, @@ -233,15 +232,6 @@ SCEnumCharMap smtp_reply_map[ ] = { /* Create SMTP config structure */ SMTPConfig smtp_config = { .decode_mime = true, - { - .decode_base64 = true, - .decode_quoted_printable = true, - .extract_urls = true, - .extract_urls_schemes = NULL, - .log_url_scheme = false, - .body_md5 = false, - .header_value_depth = 0, - }, .content_limit = FILEDATA_CONTENT_LIMIT, .content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE, .content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW, @@ -251,6 +241,8 @@ SMTPConfig smtp_config = { static SMTPString *SMTPStringAlloc(void); +#define SCHEME_SUFFIX_LEN 3 + /** * \brief Configure SMTP Mime Decoder by parsing out mime section of YAML * config file @@ -277,22 +269,25 @@ static void SMTPConfigure(void) { ret = ConfGetChildValueBool(config, "decode-base64", &val); if (ret) { - smtp_config.mime_config.decode_base64 = val; + SCMimeSmtpConfigDecodeBase64(val); } ret = ConfGetChildValueBool(config, "decode-quoted-printable", &val); if (ret) { - smtp_config.mime_config.decode_quoted_printable = val; + SCMimeSmtpConfigDecodeQuoted(val); } ret = ConfGetChildValueInt(config, "header-value-depth", &imval); if (ret) { - smtp_config.mime_config.header_value_depth = (uint32_t) imval; + if (imval < 0 || imval > UINT32_MAX) { + FatalError("Invalid value for header-value-depth"); + } + SCMimeSmtpConfigHeaderValueDepth((uint32_t)imval); } ret = ConfGetChildValueBool(config, "extract-urls", &val); if (ret) { - smtp_config.mime_config.extract_urls = val; + SCMimeSmtpConfigExtractUrls(val); } /* Parse extract-urls-schemes from mime config, add '://' suffix to found schemes, @@ -302,75 +297,48 @@ static void SMTPConfigure(void) { if (extract_urls_schemes) { ConfNode *scheme = NULL; + SCMimeSmtpConfigExtractUrlsSchemeReset(); TAILQ_FOREACH (scheme, &extract_urls_schemes->head, next) { - /* new_val_len: scheme value from config e.g. 'http' + '://' + null terminator */ - size_t new_val_len = strlen(scheme->val) + 3 + 1; - if (new_val_len > UINT16_MAX) { + size_t scheme_len = strlen(scheme->val); + if (scheme_len > UINT16_MAX - SCHEME_SUFFIX_LEN) { FatalError("Too long value for extract-urls-schemes"); } - char *new_val = SCMalloc(new_val_len); - if (unlikely(new_val == NULL)) { - FatalError("SCMalloc failure."); + if (scheme->val[scheme_len - 1] != '/') { + scheme_len += SCHEME_SUFFIX_LEN; + char *new_val = SCMalloc(scheme_len + 1); + if (unlikely(new_val == NULL)) { + FatalError("SCMalloc failure."); + } + int r = snprintf(new_val, scheme_len + 1, "%s://", scheme->val); + if (r != (int)scheme_len) { + FatalError("snprintf failure for SMTP url extraction scheme."); + } + SCFree(scheme->val); + scheme->val = new_val; } - - int r = snprintf(new_val, new_val_len, "%s://", scheme->val); - if (r < 0 || r >= (int)new_val_len) { - FatalError("snprintf failure."); + int r = SCMimeSmtpConfigExtractUrlsSchemeAdd(scheme->val); + if (r < 0) { + FatalError("Failed to add smtp extract url scheme"); } - - /* replace existing scheme value stored on the linked list with new value including - * '://' suffix */ - SCFree(scheme->val); - scheme->val = new_val; } - - smtp_config.mime_config.extract_urls_schemes = extract_urls_schemes; } else { /* Add default extract url scheme 'http' since * extract-urls-schemes wasn't found in the config */ - ConfNode *seq_node = ConfNodeNew(); - if (unlikely(seq_node == NULL)) { - FatalError("ConfNodeNew failure."); - } - ConfNode *scheme = ConfNodeNew(); - if (unlikely(scheme == NULL)) { - FatalError("ConfNodeNew failure."); - } - - seq_node->name = SCStrdup("extract-urls-schemes"); - if (unlikely(seq_node->name == NULL)) { - FatalError("SCStrdup failure."); - } - scheme->name = SCStrdup("0"); - if (unlikely(scheme->name == NULL)) { - FatalError("SCStrdup failure."); - } - scheme->val = SCStrdup("http://"); - if (unlikely(scheme->val == NULL)) { - FatalError("SCStrdup failure."); - } - - seq_node->is_seq = 1; - TAILQ_INSERT_TAIL(&seq_node->head, scheme, next); - TAILQ_INSERT_TAIL(&config->head, seq_node, next); - - smtp_config.mime_config.extract_urls_schemes = seq_node; + SCMimeSmtpConfigExtractUrlsSchemeReset(); + SCMimeSmtpConfigExtractUrlsSchemeAdd("http://"); } ret = ConfGetChildValueBool(config, "log-url-scheme", &val); if (ret) { - smtp_config.mime_config.log_url_scheme = val; + SCMimeSmtpConfigLogUrlScheme(val); } ret = ConfGetChildValueBool(config, "body-md5", &val); if (ret) { - smtp_config.mime_config.body_md5 = val; + SCMimeSmtpConfigBodyMd5(val); } } - /* Pass mime config data to MimeDec API */ - MimeDecSetConfig(&smtp_config.mime_config); - ConfNode *t = ConfGetNode("app-layer.protocols.smtp.inspected-tracker"); ConfNode *p = NULL; @@ -453,7 +421,6 @@ static SMTPTransaction *SMTPTransactionCreate(SMTPState *state) } TAILQ_INIT(&tx->rcpt_to_list); - tx->mime_state = NULL; tx->tx_data.file_tx = STREAM_TOSERVER; // can xfer files return tx; } @@ -491,151 +458,6 @@ static void SMTPNewFile(SMTPTransaction *tx, File *file) smtp_config.content_inspect_min_size); } -int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, - MimeDecParseState *state) -{ - SCEnter(); - int ret = MIME_DEC_OK; - Flow *flow = (Flow *) state->data; - SMTPState *smtp_state = (SMTPState *) flow->alstate; - SMTPTransaction *tx = smtp_state->curr_tx; - MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; - FileContainer *files = NULL; - - DEBUG_VALIDATE_BUG_ON(tx == NULL); - - uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER); - - /* Find file */ - if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - files = &tx->files_ts; - - /* Open file if necessary */ - if (state->body_begin) { -#ifdef DEBUG - if (SCLogDebugEnabled()) { - SCLogDebug("Opening file...%u bytes", len); - printf("File - "); - for (uint32_t i = 0; i < entity->filename_len; i++) { - printf("%c", entity->filename[i]); - } - printf("\n"); - } -#endif - /* Set storage flag if applicable since only the first file in the - * flow seems to be processed by the 'filestore' detector */ - if (files->head != NULL && (files->head->flags & FILE_STORE)) { - flags |= FILE_STORE; - } - - uint32_t depth = smtp_config.content_inspect_min_size + - (smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp); - SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %"PRIu32, depth); - StreamTcpReassemblySetMinInspectDepth(flow->protoctx, STREAM_TOSERVER, depth); - - uint16_t flen = (uint16_t)entity->filename_len; - if (entity->filename_len > SC_FILENAME_MAX) { - flen = SC_FILENAME_MAX; - SMTPSetEvent(smtp_state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME); - } - if (FileOpenFileWithId(files, &smtp_config.sbcfg, smtp_state->file_track_id++, - (uint8_t *)entity->filename, flen, (uint8_t *)chunk, len, flags) != 0) { - ret = MIME_DEC_ERR_DATA; - SCLogDebug("FileOpenFile() failed"); - } else { - SMTPNewFile(tx, files->tail); - } - - /* If close in the same chunk, then pass in empty bytes */ - if (state->body_end) { - - SCLogDebug("Closing file...%u bytes", len); - - if (files->tail->state == FILE_STATE_OPENED) { - ret = FileCloseFile(files, &smtp_config.sbcfg, (uint8_t *)NULL, 0, flags); - if (ret != 0) { - SCLogDebug("FileCloseFile() failed: %d", ret); - ret = MIME_DEC_ERR_DATA; - } - } else { - SCLogDebug("File already closed"); - } - depth = smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp; - - AppLayerParserTriggerRawStreamReassembly(flow, STREAM_TOSERVER); - SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", - depth); - StreamTcpReassemblySetMinInspectDepth(flow->protoctx, STREAM_TOSERVER, - depth); - } - } else if (state->body_end) { - /* Close file */ - SCLogDebug("Closing file...%u bytes", len); - - if (files->tail && files->tail->state == FILE_STATE_OPENED) { - ret = FileCloseFile(files, &smtp_config.sbcfg, (uint8_t *)chunk, len, flags); - if (ret != 0) { - SCLogDebug("FileCloseFile() failed: %d", ret); - ret = MIME_DEC_ERR_DATA; - } - } else { - SCLogDebug("File already closed"); - } - uint32_t depth = smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp; - AppLayerParserTriggerRawStreamReassembly(flow, STREAM_TOSERVER); - SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", - depth); - StreamTcpReassemblySetMinInspectDepth(flow->protoctx, - STREAM_TOSERVER, depth); - } else { - /* Append data chunk to file */ - SCLogDebug("Appending file...%u bytes", len); - /* 0 is ok, -2 is not stored, -1 is error */ - ret = FileAppendData(files, &smtp_config.sbcfg, (uint8_t *)chunk, len); - if (ret == -2) { - ret = 0; - SCLogDebug("FileAppendData() - file no longer being extracted"); - } else if (ret < 0) { - SCLogDebug("FileAppendData() failed: %d", ret); - ret = MIME_DEC_ERR_DATA; - } - - if (files->tail && files->tail->content_inspected == 0 && - files->tail->size >= smtp_config.content_inspect_min_size) { - uint32_t depth = smtp_config.content_inspect_min_size + - (smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp); - AppLayerParserTriggerRawStreamReassembly(flow, STREAM_TOSERVER); - SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", - depth); - StreamTcpReassemblySetMinInspectDepth(flow->protoctx, - STREAM_TOSERVER, depth); - - /* after the start of the body inspection, disable the depth logic */ - } else if (files->tail && files->tail->content_inspected > 0) { - StreamTcpReassemblySetMinInspectDepth(flow->protoctx, - STREAM_TOSERVER, 0); - - /* expand the limit as long as we get file data, as the file data is bigger on the - * wire due to base64 */ - } else { - uint32_t depth = smtp_config.content_inspect_min_size + - (smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp); - SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %"PRIu32, - depth); - StreamTcpReassemblySetMinInspectDepth(flow->protoctx, - STREAM_TOSERVER, depth); - } - } - - if (ret == 0) { - SCLogDebug("Successfully processed file data!"); - } - } else { - SCLogDebug("Body not a Ctnt_attachment"); - } - SCReturnInt(ret); -} - /** * \internal * \brief Get the next line from input. It doesn't do any length validation. @@ -770,39 +592,34 @@ static int SMTPProcessCommandBDAT( SCReturnInt(0); } -static void SetMimeEvents(SMTPState *state) +static void SetMimeEvents(SMTPState *state, uint32_t events) { - if (state->curr_tx->mime_state->msg == NULL) { + if (events == 0) { return; } - /* Generate decoder events */ - MimeDecEntity *msg = state->curr_tx->mime_state->msg; - if (msg->anomaly_flags & ANOM_INVALID_BASE64) { + if (events & MIME_ANOM_INVALID_BASE64) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_INVALID_BASE64); } - if (msg->anomaly_flags & ANOM_INVALID_QP) { + if (events & MIME_ANOM_INVALID_QP) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_INVALID_QP); } - if (msg->anomaly_flags & ANOM_LONG_LINE) { + if (events & MIME_ANOM_LONG_LINE) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_LINE); } - if (msg->anomaly_flags & ANOM_LONG_ENC_LINE) { + if (events & MIME_ANOM_LONG_ENC_LINE) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_ENC_LINE); } - if (msg->anomaly_flags & ANOM_LONG_HEADER_NAME) { + if (events & MIME_ANOM_LONG_HEADER_NAME) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_HEADER_NAME); } - if (msg->anomaly_flags & ANOM_LONG_HEADER_VALUE) { + if (events & MIME_ANOM_LONG_HEADER_VALUE) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_HEADER_VALUE); } - if (msg->anomaly_flags & ANOM_MALFORMED_MSG) { - SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_MALFORMED_MSG); - } - if (msg->anomaly_flags & ANOM_LONG_BOUNDARY) { + if (events & MIME_ANOM_LONG_BOUNDARY) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG); } - if (msg->anomaly_flags & ANOM_LONG_FILENAME) { + if (events & MIME_ANOM_LONG_FILENAME) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME); } } @@ -841,14 +658,11 @@ static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0, 0); } else if (smtp_config.decode_mime && tx->mime_state != NULL) { /* Complete parsing task */ - int ret = MimeDecParseComplete(tx->mime_state); - if (ret != MIME_DEC_OK) { - SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED); - SCLogDebug("MimeDecParseComplete() function failed"); + SCSmtpMimeComplete(tx->mime_state); + if (tx->files_ts.tail && tx->files_ts.tail->state == FILE_STATE_OPENED) { + FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0, + FileFlowToFlags(f, STREAM_TOSERVER)); } - - /* Generate decoder events */ - SetMimeEvents(state); } SMTPTransactionComplete(state); SCLogDebug("marked tx as done"); @@ -863,17 +677,78 @@ static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { if (smtp_config.decode_mime && tx->mime_state != NULL) { - int ret = MimeDecParseLine(line->buf, line->len, line->delim_len, tx->mime_state); - if (ret != MIME_DEC_OK) { - if (ret != MIME_DEC_ERR_STATE) { - /* Generate decoder events */ - SetMimeEvents(state); - - SCLogDebug("MimeDecParseLine() function returned an error code: %d", ret); - SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED); - } - /* keep the parser in its error state so we can log that, - * the parser will reject new data */ + uint32_t events; + uint16_t flags = FileFlowToFlags(f, STREAM_TOSERVER); + const uint8_t *filename = NULL; + uint16_t filename_len = 0; + uint32_t depth; + + /* we depend on detection engine for file pruning */ + flags |= FILE_USE_DETECT; + MimeSmtpParserResult ret = SCSmtpMimeParseLine( + line->buf, line->len, line->delim_len, &events, tx->mime_state); + SetMimeEvents(state, events); + switch (ret) { + case MimeSmtpFileOpen: + // get filename owned by mime state + SCMimeSmtpGetFilename(state->curr_tx->mime_state, &filename, &filename_len); + + if (filename_len == 0) { + // not an attachment + break; + } + depth = smtp_config.content_inspect_min_size + + (state->toserver_data_count - state->toserver_last_data_stamp); + SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %" PRIu32, + depth); + StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth); + + if (filename_len > SC_FILENAME_MAX) { + filename_len = SC_FILENAME_MAX; + SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME); + } + if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg, + state->file_track_id++, filename, filename_len, NULL, 0, + flags) != 0) { + SCLogDebug("FileOpenFile() failed"); + } + SMTPNewFile(state->curr_tx, tx->files_ts.tail); + break; + case MimeSmtpFileChunk: + // rust already run FileAppendData + if (tx->files_ts.tail && tx->files_ts.tail->content_inspected == 0 && + tx->files_ts.tail->size >= smtp_config.content_inspect_min_size) { + depth = smtp_config.content_inspect_min_size + + (state->toserver_data_count - state->toserver_last_data_stamp); + AppLayerParserTriggerRawStreamReassembly(f, STREAM_TOSERVER); + SCLogDebug( + "StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", depth); + StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth); + /* after the start of the body inspection, disable the depth logic */ + } else if (tx->files_ts.tail && tx->files_ts.tail->content_inspected > 0) { + StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, 0); + /* expand the limit as long as we get file data, as the file data is bigger + * on the wire due to base64 */ + } else { + depth = smtp_config.content_inspect_min_size + + (state->toserver_data_count - state->toserver_last_data_stamp); + SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %" PRIu32, + depth); + StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth); + } + break; + case MimeSmtpFileClose: + if (tx->files_ts.tail && tx->files_ts.tail->state == FILE_STATE_OPENED) { + if (FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0, flags) != 0) { + SCLogDebug("FileCloseFile() failed: %d", ret); + } + } else { + SCLogDebug("File already closed"); + } + depth = state->toserver_data_count - state->toserver_last_data_stamp; + AppLayerParserTriggerRawStreamReassembly(f, STREAM_TOSERVER); + SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", depth); + StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth); } } } @@ -1169,8 +1044,8 @@ static int NoNewTx(SMTPState *state, const SMTPLine *line) * -1 for errors and inconsistent states * -2 if MIME state could not be allocated * */ -static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *pstate, - SMTPInput *input, const SMTPLine *line) +static int SMTPProcessRequest( + SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line) { SCEnter(); SMTPTransaction *tx = state->curr_tx; @@ -1224,19 +1099,11 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps } } else if (smtp_config.decode_mime) { DEBUG_VALIDATE_BUG_ON(tx->mime_state); - tx->mime_state = MimeDecInitParser(f, SMTPProcessDataChunk); + tx->mime_state = SCMimeSmtpStateInit(&tx->files_ts, &smtp_config.sbcfg); if (tx->mime_state == NULL) { - return MIME_DEC_ERR_MEM; - } - - /* Add new MIME message to end of list */ - if (tx->msg_head == NULL) { - tx->msg_head = tx->mime_state->msg; - tx->msg_tail = tx->mime_state->msg; - } - else { - tx->msg_tail->next = tx->mime_state->msg; - tx->msg_tail = tx->mime_state->msg; + SCLogDebug("MimeDecInitParser() failed to " + "allocate data"); + return -1; } } state->curr_tx->is_data = true; @@ -1343,7 +1210,7 @@ static int SMTPPreProcessCommands( /* fall back to strict line parsing for mime header parsing */ if (state->curr_tx && state->curr_tx->mime_state && - state->curr_tx->mime_state->state_flag < HEADER_DONE) + SCMimeSmtpGetState(state->curr_tx->mime_state) < MimeSmtpBody) return 1; bool line_complete = false; @@ -1387,7 +1254,7 @@ static int SMTPPreProcessCommands( input->consumed = total_consumed; input->len -= current_line_consumed; DEBUG_VALIDATE_BUG_ON(input->consumed + input->len != input->orig_len); - if (SMTPProcessRequest(state, f, pstate, input, line) == -1) { + if (SMTPProcessRequest(state, f, pstate, line) == -1) { return -1; } line_complete = false; @@ -1438,7 +1305,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, } AppLayerResult res = SMTPGetLine(state, &input, &line, direction); while (res.status == 0) { - int retval = SMTPProcessRequest(state, f, pstate, &input, &line); + int retval = SMTPProcessRequest(state, f, pstate, &line); if (retval != 0) SCReturnStruct(APP_LAYER_ERROR); if (line.delim_len == 0 && line.len == SMTP_LINE_BUFFER_LIMIT) { @@ -1596,10 +1463,8 @@ static void SMTPLocalStorageFree(void *ptr) static void SMTPTransactionFree(SMTPTransaction *tx, SMTPState *state) { if (tx->mime_state != NULL) { - MimeDecDeInitParser(tx->mime_state); + SCMimeSmtpStateFree(tx->mime_state); } - /* Free list of MIME message recursively */ - MimeDecFreeEntity(tx->msg_head); if (tx->tx_data.events != NULL) AppLayerDecoderEventsFreeEvents(&tx->tx_data.events); @@ -1915,8 +1780,6 @@ void SMTPParserCleanup(void) static void SMTPTestInitConfig(void) { - MimeDecSetConfig(&smtp_config.mime_config); - smtp_config.content_limit = FILEDATA_CONTENT_LIMIT; smtp_config.content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW; smtp_config.content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE; @@ -4078,9 +3941,8 @@ static int SMTPParserTest14(void) /* Enable mime decoding */ smtp_config.decode_mime = true; - smtp_config.mime_config.decode_base64 = true; - smtp_config.mime_config.decode_quoted_printable = true; - MimeDecSetConfig(&smtp_config.mime_config); + SCMimeSmtpConfigDecodeBase64(1); + SCMimeSmtpConfigDecodeQuoted(1); /* DATA request */ r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP, @@ -4121,7 +3983,6 @@ static int SMTPParserTest14(void) if (smtp_state->cmds_cnt != 0 || smtp_state->cmds_idx != 0 || smtp_state->curr_tx->mime_state == NULL || - smtp_state->curr_tx->msg_head == NULL || /* MIME data structures */ smtp_state->parser_state != (SMTP_PARSER_STATE_FIRST_REPLY_SEEN | SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { printf("smtp parser in inconsistent state l.%d\n", __LINE__); @@ -4139,7 +4000,6 @@ static int SMTPParserTest14(void) if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 || smtp_state->cmds[0] != SMTP_COMMAND_DATA_MODE || smtp_state->curr_tx->mime_state == NULL || - smtp_state->curr_tx->msg_head == NULL || /* MIME data structures */ smtp_state->parser_state != (SMTP_PARSER_STATE_FIRST_REPLY_SEEN)) { printf("smtp parser in inconsistent state l.%d\n", __LINE__); goto end; diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h index 37369af034f3..93c3bd812c93 100644 --- a/src/app-layer-smtp.h +++ b/src/app-layer-smtp.h @@ -24,7 +24,6 @@ #ifndef SURICATA_APP_LAYER_SMTP_H #define SURICATA_APP_LAYER_SMTP_H -#include "util-decode-mime.h" #include "util-streaming-buffer.h" #include "rust.h" @@ -81,12 +80,8 @@ typedef struct SMTPTransaction_ { // another DATA command within the same context // will trigger an app-layer event. bool is_data; - /** the first message contained in the session */ - MimeDecEntity *msg_head; - /** the last message contained in the session */ - MimeDecEntity *msg_tail; /** the mime decoding parser state */ - MimeDecParseState *mime_state; + MimeStateSMTP *mime_state; /* MAIL FROM parameters */ uint8_t *mail_from; @@ -99,10 +94,14 @@ typedef struct SMTPTransaction_ { TAILQ_ENTRY(SMTPTransaction_) next; } SMTPTransaction; +/** + * \brief Structure for containing configuration options + * + */ + typedef struct SMTPConfig { bool decode_mime; - MimeDecConfig mime_config; uint32_t content_limit; uint32_t content_inspect_min_size; uint32_t content_inspect_window; @@ -158,7 +157,6 @@ typedef struct SMTPState_ { /* Create SMTP config structure */ extern SMTPConfig smtp_config; -int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, MimeDecParseState *state); void *SMTPStateAlloc(void *orig_state, AppProto proto_orig); void RegisterSMTPParsers(void); void SMTPParserCleanup(void); diff --git a/src/output-json-email-common.c b/src/output-json-email-common.c index ab8718aeb413..817549731daa 100644 --- a/src/output-json-email-common.c +++ b/src/output-json-email-common.c @@ -83,82 +83,29 @@ struct { { NULL, NULL, LOG_EMAIL_DEFAULT}, }; -static inline char *SkipWhiteSpaceTill(char *p, char *savep) -{ - char *sp = p; - if (unlikely(p == NULL)) { - return NULL; - } - while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) { - sp++; - } - return sp; -} - -static bool EveEmailJsonArrayFromCommaList(JsonBuilder *js, const uint8_t *val, size_t len) -{ - char *savep = NULL; - char *p; - char *sp; - char *to_line = BytesToString((uint8_t *)val, len); - if (likely(to_line != NULL)) { - p = strtok_r(to_line, ",", &savep); - if (p == NULL) { - SCFree(to_line); - return false; - } - sp = SkipWhiteSpaceTill(p, savep); - jb_append_string(js, sp); - while ((p = strtok_r(NULL, ",", &savep)) != NULL) { - sp = SkipWhiteSpaceTill(p, savep); - jb_append_string(js, sp); - } - } else { - return false; - } - SCFree(to_line); - return true; -} - static void EveEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx) { if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) { - MimeDecEntity *entity = tx->msg_tail; + MimeStateSMTP *entity = tx->mime_state; if (entity == NULL) { return; } - MimeDecField *field = MimeDecFindField(entity, "subject"); - if (field != NULL) { - char smd5[SC_MD5_HEX_LEN + 1]; - SCMd5HashBufferToHex((uint8_t *)field->value, field->value_len, smd5, sizeof(smd5)); - jb_set_string(js, "subject_md5", smd5); - } + SCMimeSmtpLogSubjectMd5(js, entity); } if (email_ctx->flags & LOG_EMAIL_BODY_MD5) { - MimeDecParseState *mime_state = tx->mime_state; - if (mime_state && mime_state->has_md5 && (mime_state->state_flag == PARSE_DONE)) { - jb_set_hex(js, "body_md5", mime_state->md5, (uint32_t)sizeof(mime_state->md5)); + MimeStateSMTP *entity = tx->mime_state; + if (entity == NULL) { + return; } + SCMimeSmtpLogBodyMd5(js, entity); } } -static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data) -{ - JsonBuilder *ajs = data; - - if (ajs == NULL) - return 0; - jb_append_string_from_bytes(ajs, val, (uint32_t)len); - return 1; -} - static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx) { int f = 0; - JsonBuilderMark mark = { 0, 0, 0 }; - MimeDecField *field; - MimeDecEntity *entity = tx->msg_tail; + MimeStateSMTP *entity = tx->mime_state; if (entity == NULL) { return; } @@ -169,31 +116,14 @@ static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED)) ) { if (email_fields[f].flags & LOG_EMAIL_ARRAY) { - jb_get_mark(js, &mark); - jb_open_array(js, email_fields[f].config_field); - int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, js); - if (found > 0) { - jb_close(js); - } else { - jb_restore_mark(js, &mark); - } + SCMimeSmtpLogFieldArray( + js, entity, email_fields[f].email_field, email_fields[f].config_field); } else if (email_fields[f].flags & LOG_EMAIL_COMMA) { - field = MimeDecFindField(entity, email_fields[f].email_field); - if (field) { - jb_get_mark(js, &mark); - jb_open_array(js, email_fields[f].config_field); - if (EveEmailJsonArrayFromCommaList(js, field->value, field->value_len)) { - jb_close(js); - } else { - jb_restore_mark(js, &mark); - } - } + SCMimeSmtpLogFieldComma( + js, entity, email_fields[f].email_field, email_fields[f].config_field); } else { - field = MimeDecFindField(entity, email_fields[f].email_field); - if (field != NULL) { - jb_set_string_from_bytes( - js, email_fields[f].config_field, field->value, field->value_len); - } + SCMimeSmtpLogFieldString( + js, entity, email_fields[f].email_field, email_fields[f].config_field); } } @@ -205,9 +135,7 @@ static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id, JsonBuilder *sjs) { SMTPState *smtp_state; - MimeDecParseState *mime_state; - MimeDecEntity *entity; - JsonBuilderMark mark = { 0, 0, 0 }; + MimeStateSMTP *mime_state; /* check if we have SMTP state or not */ AppProto proto = FlowGetAppProtocol(f); @@ -221,110 +149,14 @@ static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t } SMTPTransaction *tx = vtx; mime_state = tx->mime_state; - entity = tx->msg_tail; - SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0); + SCLogDebug("lets go mime_state %p", mime_state); break; default: /* don't know how we got here */ SCReturnBool(false); } if ((mime_state != NULL)) { - if (entity == NULL) { - SCReturnBool(false); - } - - jb_set_string(sjs, "status", MimeDecParseStateGetStatus(mime_state)); - - MimeDecField *field; - - /* From: */ - field = MimeDecFindField(entity, "from"); - if (field != NULL) { - char *s = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(s != NULL)) { - //printf("From: \"%s\"\n", s); - char * sp = SkipWhiteSpaceTill(s, s + strlen(s)); - jb_set_string(sjs, "from", sp); - SCFree(s); - } - } - - /* To: */ - field = MimeDecFindField(entity, "to"); - if (field != NULL) { - jb_get_mark(sjs, &mark); - jb_open_array(sjs, "to"); - if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) { - jb_close(sjs); - } else { - jb_restore_mark(sjs, &mark); - } - } - - /* Cc: */ - field = MimeDecFindField(entity, "cc"); - if (field != NULL) { - jb_get_mark(sjs, &mark); - jb_open_array(sjs, "cc"); - if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) { - jb_close(sjs); - } else { - jb_restore_mark(sjs, &mark); - } - } - - if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) { - SCReturnBool(false); - } - - entity = (MimeDecEntity *)mime_state->stack->top->data; - int attach_cnt = 0; - int url_cnt = 0; - JsonBuilder *js_attach = jb_new_array(); - JsonBuilder *js_url = jb_new_array(); - if (entity->url_list != NULL) { - MimeDecUrl *url; - bool has_ipv6_url = false; - bool has_ipv4_url = false; - bool has_exe_url = false; - for (url = entity->url_list; url != NULL; url = url->next) { - jb_append_string_from_bytes(js_url, url->url, url->url_len); - if (url->url_flags & URL_IS_EXE) - has_exe_url = true; - if (url->url_flags & URL_IS_IP6) - has_ipv6_url = true; - if (url->url_flags & URL_IS_IP4) - has_ipv6_url = true; - url_cnt += 1; - } - jb_set_bool(sjs, "has_ipv6_url", has_ipv6_url); - jb_set_bool(sjs, "has_ipv4_url", has_ipv4_url); - jb_set_bool(sjs, "has_exe_url", has_exe_url); - } - for (entity = entity->child; entity != NULL; entity = entity->next) { - if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - jb_append_string_from_bytes(js_attach, entity->filename, entity->filename_len); - attach_cnt += 1; - } - if (entity->url_list != NULL) { - MimeDecUrl *url; - for (url = entity->url_list; url != NULL; url = url->next) { - jb_append_string_from_bytes(js_url, url->url, url->url_len); - url_cnt += 1; - } - } - } - if (attach_cnt > 0) { - jb_close(js_attach); - jb_set_object(sjs, "attachment", js_attach); - } - jb_free(js_attach); - if (url_cnt > 0) { - jb_close(js_url); - jb_set_object(sjs, "url", js_url); - } - jb_free(js_url); + SCMimeSmtpLogData(sjs, mime_state); SCReturnBool(true); } diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index ec33efe97572..7d335d18ca8e 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -79,6 +79,7 @@ #include "util-memcmp.h" #include "util-misc.h" #include "util-signal.h" +#include "util-base64.h" #include "reputation.h" #include "util-atomic.h" @@ -203,7 +204,6 @@ static void RegisterUnittests(void) SCAtomicRegisterTests(); MemrchrRegisterTests(); AppLayerUnittestsRegister(); - MimeDecRegisterTests(); StreamingBufferRegisterTests(); MacSetRegisterTests(); #ifdef OS_WIN32 diff --git a/src/tests/fuzz/fuzz_mimedecparseline.c b/src/tests/fuzz/fuzz_mimedecparseline.c index e067162d89a7..da0d350c497a 100644 --- a/src/tests/fuzz/fuzz_mimedecparseline.c +++ b/src/tests/fuzz/fuzz_mimedecparseline.c @@ -7,22 +7,11 @@ #include "suricata-common.h" #include "suricata.h" -#include "util-decode-mime.h" +#include "rust.h" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); static int initialized = 0; -static int dummy = 0; - -static int MimeParserDataFromFileCB(const uint8_t *chunk, uint32_t len, - MimeDecParseState *state) -{ - if (len > 0 && chunk[len-1] == 0) { - // do not get optimized away - dummy++; - } - return MIME_DEC_OK; -} int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { @@ -36,19 +25,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) initialized = 1; } - uint32_t line_count = 0; - - MimeDecParseState *state = MimeDecInitParser(&line_count, MimeParserDataFromFileCB); - MimeDecEntity *msg_head = state->msg; + uint32_t events; + FileContainer *files = FileContainerAlloc(); + StreamingBufferConfig sbcfg = STREAMING_BUFFER_CONFIG_INITIALIZER; + MimeStateSMTP *state = SCMimeSmtpStateInit(files, &sbcfg); const uint8_t * buffer = data; while (1) { uint8_t * next = memchr(buffer, '\n', size); if (next == NULL) { - if (state->state_flag >= BODY_STARTED) - (void)MimeDecParseLine(buffer, size, 0, state); + if (SCMimeSmtpGetState(state) >= MimeSmtpBody) + (void)SCSmtpMimeParseLine(buffer, size, 0, &events, state); break; } else { - (void) MimeDecParseLine(buffer, next - buffer, 1, state); + (void)SCSmtpMimeParseLine(buffer, next - buffer, 1, &events, state); if (buffer + size < next + 1) { break; } @@ -57,10 +46,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) } } /* Completed */ - (void)MimeDecParseComplete(state); + (void)SCSmtpMimeComplete(state); /* De Init parser */ - MimeDecDeInitParser(state); - MimeDecFreeEntity(msg_head); + SCMimeSmtpStateFree(state); + FileContainerFree(files, &sbcfg); return 0; } diff --git a/src/util-decode-mime.c b/src/util-decode-mime.c deleted file mode 100644 index 4d5b8c1da5eb..000000000000 --- a/src/util-decode-mime.c +++ /dev/null @@ -1,3590 +0,0 @@ -/* Copyright (C) 2012 BAE Systems - * Copyright (C) 2020-2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author David Abarbanel - * - */ - -#include "suricata-common.h" -#include "suricata.h" -#include "app-layer-smtp.h" -#include "util-decode-mime.h" -#include "util-ip.h" -#include "util-spm-bs.h" -#include "util-unittest.h" -#include "util-memcmp.h" -#include "util-print.h" -#include "util-validate.h" -#include "rust.h" - -/* Character constants */ -#ifndef CR -#define CR 13 -#define LF 10 -#endif - -#define CRLF "\r\n" -#define COLON 58 -#define DASH 45 -#define PRINTABLE_START 33 -#define PRINTABLE_END 126 -#define EOL_LEN 2 - -/* Base-64 constants */ -#define BASE64_STR "Base64" - -/* Mime Constants */ -#define MAX_LINE_LEN 998 /* Def in RFC 2045, excluding CRLF sequence */ -#define MAX_ENC_LINE_LEN 76 /* Def in RFC 2045, excluding CRLF sequence */ -#define MAX_HEADER_NAME 75 /* 75 + ":" = 76 */ -#define MAX_HEADER_VALUE 2000 /* Default - arbitrary limit */ -#define BOUNDARY_BUF 256 -#define CTNT_TYPE_STR "content-type" -#define CTNT_DISP_STR "content-disposition" -#define CTNT_TRAN_STR "content-transfer-encoding" -#define MSG_ID_STR "message-id" -#define MSG_STR "message/" -#define MULTIPART_STR "multipart/" -#define QP_STR "quoted-printable" -#define TXT_STR "text/plain" -#define HTML_STR "text/html" - -/* Memory Usage Constants */ -#define STACK_FREE_NODES 10 - -/* Other Constants */ -#define MAX_IP4_CHARS 15 -#define MAX_IP6_CHARS 39 - -/* Globally hold configuration data */ -static MimeDecConfig mime_dec_config = { true, true, true, NULL, false, false, MAX_HEADER_VALUE }; - -/* Mime Parser String translation */ -static const char *StateFlags[] = { "NONE", - "HEADER_READY", - "HEADER_STARTED", - "HEADER_DONE", - "BODY_STARTED", - "BODY_DONE", - "BODY_END_BOUND", - "PARSE_DONE", - "PARSE_ERROR", - NULL }; - -/* URL executable file extensions */ -static const char *UrlExeExts[] = { ".exe", ".vbs", ".bin", ".cmd", ".bat", ".jar", ".js", ".ps", - ".ps1", ".sh", ".run", ".hta", ".bin", ".elf", NULL }; - -/** - * \brief Function used to print character strings that are not null-terminated - * - * \param log_level The logging level in which to print - * \param label A label for the string to print - * \param src The source string - * \param len The length of the string - * - * \return none - */ -static void PrintChars(int log_level, const char *label, const uint8_t *src, uint32_t len) -{ -#ifdef DEBUG - if (log_level <= sc_log_global_log_level) { - printf("[%s]\n", label); - PrintRawDataFp(stdout, (uint8_t *)src, len); - } -#endif -} - -/** - * \brief Set global config policy - * - * \param config Config policy to set - * \return none - */ -void MimeDecSetConfig(MimeDecConfig *config) -{ - if (config != NULL) { - mime_dec_config = *config; - - /* Set to default */ - if (mime_dec_config.header_value_depth == 0) { - mime_dec_config.header_value_depth = MAX_HEADER_VALUE; - } - } else { - SCLogWarning("Invalid null configuration parameters"); - } -} - -/** - * \brief Get global config policy - * - * \return config data structure - */ -MimeDecConfig * MimeDecGetConfig(void) -{ - return &mime_dec_config; -} - -/** - * \brief Follow the 'next' pointers to the leaf - * - * \param node The root entity - * - * \return Pointer to leaf on 'next' side - * - */ -static MimeDecEntity *findLastSibling(MimeDecEntity *node) -{ - if (node == NULL) - return NULL; - while(node->next != NULL) - node = node->next; - return node; -} - -/** - * \brief Frees a mime entity tree - * - * \param entity The root entity - * - * \return none - * - */ -void MimeDecFreeEntity (MimeDecEntity *entity) -{ - if (entity == NULL) - return; - MimeDecEntity *lastSibling = findLastSibling(entity); - while (entity != NULL) - { - /* move child to next to transform the tree into a list */ - if (entity->child != NULL) { - lastSibling->next = entity->child; - entity->child = NULL; - lastSibling = findLastSibling(lastSibling); - } - - MimeDecEntity *next = entity->next; - DEBUG_VALIDATE_BUG_ON( - (next != NULL && entity == lastSibling) || (next == NULL && entity != lastSibling)); - MimeDecFreeField(entity->field_list); - MimeDecFreeUrl(entity->url_list); - SCFree(entity->filename); - SCFree(entity); - entity = next; - } -} - -/** - * \brief Iteratively frees a header field entry list - * - * \param field The header field - * - * \return none - * - */ -void MimeDecFreeField(MimeDecField *field) -{ - MimeDecField *temp, *curr; - - if (field != NULL) { - - curr = field; - while (curr != NULL) { - temp = curr; - curr = curr->next; - - /* Free contents of node */ - SCFree(temp->name); - SCFree(temp->value); - - /* Now free node data */ - SCFree(temp); - } - } -} - -/** - * \brief Iteratively frees a URL entry list - * - * \param url The url entry - * - * \return none - * - */ -void MimeDecFreeUrl(MimeDecUrl *url) -{ - MimeDecUrl *temp, *curr; - - if (url != NULL) { - - curr = url; - while (curr != NULL) { - temp = curr; - curr = curr->next; - - /* Now free node data */ - SCFree(temp->url); - SCFree(temp); - } - } -} - -/** - * \brief Creates and adds a header field entry to an entity - * - * The entity is optional. If NULL is specified, than a new stand-alone field - * is created. - * - * \param entity The parent entity - * - * \return The field object, or NULL if the operation fails - * - */ -MimeDecField * MimeDecAddField(MimeDecEntity *entity) -{ - MimeDecField *node = SCCalloc(1, sizeof(MimeDecField)); - if (unlikely(node == NULL)) { - return NULL; - } - - /* If list is empty, then set as head of list */ - if (entity->field_list == NULL) { - entity->field_list = node; - } else { - /* Otherwise add to beginning of list since these are out-of-order in - * the message */ - node->next = entity->field_list; - entity->field_list = node; - } - - return node; -} - - -/** - * \brief Searches for header fields with the specified name - * - * \param entity The entity to search - * \param name The header name (lowercase) - * - * \return number of items found - * - */ -int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data) -{ - MimeDecField *curr = entity->field_list; - int found = 0; - - while (curr != NULL) { - /* name is stored lowercase */ - if (strlen(name) == curr->name_len) { - if (SCMemcmp(curr->name, name, curr->name_len) == 0) { - if (DataCallback(curr->value, curr->value_len, data)) - found++; - } - } - curr = curr->next; - } - - return found; -} - -/** - * \brief Searches for a header field with the specified name - * - * \param entity The entity to search - * \param name The header name (lowercase) - * - * \return The field object, or NULL if not found - * - */ -MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name) { - MimeDecField *curr = entity->field_list; - - while (curr != NULL) { - /* name is stored lowercase */ - if (strlen(name) == curr->name_len) { - if (SCMemcmp(curr->name, name, curr->name_len) == 0) { - break; - } - } - curr = curr->next; - } - - return curr; -} - -/** - * \brief Creates and adds a URL entry to the specified entity - * - * The entity is optional and if NULL is specified, then a new list will be created. - * - * \param entity The entity - * - * \return URL entry or NULL if the operation fails - * - */ -static MimeDecUrl * MimeDecAddUrl(MimeDecEntity *entity, uint8_t *url, uint32_t url_len, uint8_t flags) -{ - MimeDecUrl *node = SCCalloc(1, sizeof(MimeDecUrl)); - if (unlikely(node == NULL)) { - return NULL; - } - - node->url = url; - node->url_len = url_len; - node->url_flags = flags; - - /* If list is empty, then set as head of list */ - if (entity->url_list == NULL) { - entity->url_list = node; - } else { - /* Otherwise add to beginning of list since these are out-of-order in - * the message */ - node->next = entity->url_list; - entity->url_list = node; - } - - return node; -} - -/** - * \brief Creates and adds a child entity to the specified parent entity - * - * \param parent The parent entity - * - * \return The child entity, or NULL if the operation fails - * - */ -MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent) -{ - MimeDecEntity *node = SCCalloc(1, sizeof(MimeDecEntity)); - if (unlikely(node == NULL)) { - return NULL; - } - - /* If parent is NULL then just return the new pointer */ - if (parent != NULL) { - if (parent->child == NULL) { - parent->child = node; - parent->last_child = node; - } else { - parent->last_child->next = node; - parent->last_child = node; - } - } - - return node; -} - -/** - * \brief Creates a mime header field and fills in its values and adds it to the - * specified entity - * - * \param entity Entity in which to add the field - * \param name String containing the name - * \param nlen Length of the name - * \param value String containing the value - * \param vlen Length of the value - * - * \return The field or NULL if the operation fails - * - * name and val are passed as ptr to ptr and each will be set to NULL - * only if the pointer is consumed. This gives the caller an easy way - * to free the memory if not consumed. - */ -static MimeDecField *MimeDecFillField( - MimeDecEntity *entity, uint8_t **name, uint32_t nlen, uint8_t **value, uint32_t vlen) -{ - if (nlen == 0 && vlen == 0) - return NULL; - - MimeDecField *field = MimeDecAddField(entity); - if (unlikely(field == NULL)) { - return NULL; - } - - if (nlen > 0) { - uint8_t *n = *name; - - /* convert to lowercase and store */ - for (uint32_t u = 0; u < nlen; u++) - n[u] = u8_tolower(n[u]); - - field->name = (uint8_t *)n; - field->name_len = nlen; - *name = NULL; - } - - if (vlen > 0) { - field->value = (uint8_t *)*value; - field->value_len = vlen; - *value = NULL; - } - - return field; -} - -/** - * \brief Pushes a node onto a stack and returns the new node. - * - * \param stack The top of the stack - * - * \return pointer to a new node, otherwise NULL if it fails - */ -static MimeDecStackNode * PushStack(MimeDecStack *stack) -{ - /* Attempt to pull from free nodes list */ - MimeDecStackNode *node = stack->free_nodes; - if (node == NULL) { - node = SCCalloc(1, sizeof(MimeDecStackNode)); - if (unlikely(node == NULL)) { - return NULL; - } - } else { - /* Move free nodes pointer over */ - stack->free_nodes = stack->free_nodes->next; - stack->free_nodes_cnt--; - memset(node, 0x00, sizeof(MimeDecStackNode)); - } - - /* Push to top of stack */ - node->next = stack->top; - stack->top = node; - - /* Return a pointer to the top of the stack */ - return node; -} - -/** - * \brief Pops the top node from the stack and returns the next node. - * - * \param stack The top of the stack - * - * \return pointer to the next node, otherwise NULL if no nodes remain - */ -static MimeDecStackNode * PopStack(MimeDecStack *stack) -{ - /* Move stack pointer to next item */ - MimeDecStackNode *curr = stack->top; - if (curr != NULL) { - curr = curr->next; - } - - /* Always free alloc'd memory */ - SCFree(stack->top->bdef); - - /* Now move head to free nodes list */ - if (stack->free_nodes_cnt < STACK_FREE_NODES) { - stack->top->next = stack->free_nodes; - stack->free_nodes = stack->top; - stack->free_nodes_cnt++; - } else { - SCFree(stack->top); - } - stack->top = curr; - - /* Return a pointer to the top of the stack */ - return curr; -} - -/** - * \brief Frees the stack along with the free-nodes list - * - * \param stack The stack pointer - * - * \return none - */ -static void FreeMimeDecStack(MimeDecStack *stack) -{ - MimeDecStackNode *temp, *curr; - - if (stack != NULL) { - /* Top of stack */ - curr = stack->top; - while (curr != NULL) { - temp = curr; - curr = curr->next; - - /* Now free node */ - SCFree(temp->bdef); - SCFree(temp); - } - - /* Free nodes */ - curr = stack->free_nodes; - while (curr != NULL) { - temp = curr; - curr = curr->next; - - /* Now free node */ - SCFree(temp); - } - - SCFree(stack); - } -} - -/** - * \brief Adds a data value to the data values linked list - * - * \param dv The head of the linked list (NULL if new list) - * - * \return pointer to a new node, otherwise NULL if it fails - */ -static DataValue * AddDataValue(DataValue *dv) -{ - DataValue *curr, *node = SCCalloc(1, sizeof(DataValue)); - if (unlikely(node == NULL)) { - return NULL; - } - - if (dv != NULL) { - curr = dv; - while (curr->next != NULL) { - curr = curr->next; - } - - curr->next = node; - } - - return node; -} - -/** - * \brief Frees a linked list of data values starting at the head - * - * \param dv The head of the linked list - * - * \return none - */ -static void FreeDataValue(DataValue *dv) -{ - DataValue *temp, *curr; - - if (dv != NULL) { - curr = dv; - while (curr != NULL) { - temp = curr; - curr = curr->next; - - /* Now free node */ - SCFree(temp->value); - SCFree(temp); - } - } -} - -/** - * \brief Converts a list of data values into a single value (returns dynamically - * allocated memory) - * - * \param dv The head of the linked list (NULL if new list) - * \param olen The output length of the single value - * - * \return pointer to a single value, otherwise NULL if it fails or is zero-length - */ -static uint8_t *GetFullValue(const DataValue *dv, uint32_t *olen) -{ - uint32_t offset = 0; - uint8_t *val = NULL; - uint32_t len = 0; - *olen = 0; - - /* First calculate total length */ - for (const DataValue *curr = dv; curr != NULL; curr = curr->next) { - if (unlikely(len > UINT32_MAX - curr->value_len)) { - // This should never happen as caller checks already against mdcfg->header_value_depth - DEBUG_VALIDATE_BUG_ON(1); - return NULL; - } - len += curr->value_len; - } - /* Must have at least one character in the value */ - if (len > 0) { - val = SCCalloc(1, len); - if (unlikely(val == NULL)) { - return NULL; - } - for (const DataValue *curr = dv; curr != NULL; curr = curr->next) { - memcpy(val + offset, curr->value, curr->value_len); - offset += curr->value_len; - } - } - *olen = len; - return val; -} - -/** - * \brief Find a string while searching up to N characters within a source - * buffer - * - * \param src The source string (not null-terminated) - * \param len The length of the source string - * \param find The string to find (null-terminated) - * \param find_len length of the 'find' string - * - * \return Pointer to the position it was found, otherwise NULL if not found - */ -static inline uint8_t *FindBuffer( - const uint8_t *src, uint32_t len, const uint8_t *find, uint16_t find_len) -{ - /* Use utility search function */ - return BasicSearchNocase(src, len, find, find_len); -} - -/** - * \brief Get a line (CRLF or just CR or LF) from a buffer (similar to GetToken) - * - * \param buf The input buffer (not null-terminated) - * \param blen The length of the input buffer - * \param remainPtr Pointer to remaining after tokenizing iteration - * \param tokLen Output token length (if non-null line) - * - * \return Pointer to line - */ -static uint8_t * GetLine(uint8_t *buf, uint32_t blen, uint8_t **remainPtr, - uint32_t *tokLen) -{ - uint32_t i; - uint8_t *tok; - - /* So that it can be used just like strtok_r */ - if (buf == NULL) { - buf = *remainPtr; - } else { - *remainPtr = buf; - } - if (buf == NULL) - return NULL; - - tok = buf; - - /* length must be specified */ - for (i = 0; i < blen && buf[i] != 0; i++) { - - /* Found delimiter */ - if (buf[i] == CR || buf[i] == LF) { - - /* Add another if we find either CRLF or LFCR */ - *remainPtr += (i + 1); - if ((i + 1 < blen) && buf[i] != buf[i + 1] && - (buf[i + 1] == CR || buf[i + 1] == LF)) { - (*remainPtr)++; - } - break; - } - } - - /* If no delimiter found, then point to end of buffer */ - if (buf == *remainPtr) { - (*remainPtr) += i; - } - - /* Calculate token length */ - *tokLen = (buf + i) - tok; - - return tok; -} - -/** - * \brief Get token from buffer and return pointer to it - * - * \param buf The input buffer (not null-terminated) - * \param blen The length of the input buffer - * \param delims Character delimiters (null-terminated) - * \param remainPtr Pointer to remaining after tokenizing iteration - * \param tokLen Output token length (if non-null line) - * - * \return Pointer to token, or NULL if not found - */ -static uint8_t * GetToken(uint8_t *buf, uint32_t blen, const char *delims, - uint8_t **remainPtr, uint32_t *tokenLen) -{ - uint32_t i, j, delimFound = 0; - uint8_t *tok = NULL; - - /* So that it can be used just like strtok_r */ - if (buf == NULL) { - buf = *remainPtr; - } else { - *remainPtr = buf; - } - if (buf == NULL) - return NULL; - - /* Must specify length */ - for (i = 0; i < blen && buf[i] != 0; i++) { - - /* Look for delimiters */ - for (j = 0; delims[j] != 0; j++) { - if (buf[i] == delims[j]) { - /* Data must be found before delimiter matters */ - if (tok != NULL) { - (*remainPtr) += (i + 1); - } - delimFound = 1; - break; - } - } - - /* If at least one non-delimiter found, then a token is found */ - if (tok == NULL && !delimFound) { - tok = buf + i; - } else { - /* Reset delimiter */ - delimFound = 0; - } - - /* If delimiter found, then break out of loop */ - if (buf != *remainPtr) { - break; - } - } - - /* Make sure remaining points to end of buffer if delimiters not found */ - if (tok != NULL) { - if (buf == *remainPtr) { - (*remainPtr) += i; - } - - /* Calculate token length */ - *tokenLen = (buf + i) - tok; - } - - return tok; -} - -/** - * \brief Stores the final MIME header value into the current entity on the - * stack. - * - * \param state The parser state - * - * \return MIME_DEC_OK if stored, otherwise a negative number indicating error - */ -static int StoreMimeHeader(MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - - /* Lets save the most recent header */ - if (state->hname != NULL || state->hvalue != NULL) { - SCLogDebug("Storing last header"); - uint32_t vlen; - uint8_t *val = GetFullValue(state->hvalue, &vlen); - if (val != NULL) { - if (state->hname == NULL) { - SCLogDebug("Error: Invalid parser state - header value without" - " name"); - ret = MIME_DEC_ERR_PARSE; - - } else if (state->stack->top != NULL) { - /* Store each header name and value */ - if (MimeDecFillField(state->stack->top->data, &state->hname, state->hlen, &val, - vlen) == NULL) { - ret = MIME_DEC_ERR_MEM; - } - } else { - SCLogDebug("Error: Stack pointer missing"); - ret = MIME_DEC_ERR_DATA; - } - } else { - if (state->hvalue != NULL) { - /* Memory allocation must have failed since val is NULL */ - ret = MIME_DEC_ERR_MEM; - } - } - - SCFree(val); - SCFree(state->hname); - state->hname = NULL; - FreeDataValue(state->hvalue); - state->hvalue = NULL; - state->hvlen = 0; - } - - return ret; -} - -/** - * \brief Function determines whether a url string points to an executable - * based on file extension only. - * - * \param url The url string - * \param len The url string length - * - * \retval 1 The url points to an EXE - * \retval 0 The url does NOT point to an EXE - */ -static int IsExeUrl(const uint8_t *url, uint32_t len) -{ - int isExeUrl = 0; - uint32_t i, extLen; - uint8_t *ext; - - /* Now check for executable extensions and if not found, cut off at first '/' */ - for (i = 0; UrlExeExts[i] != NULL; i++) { - extLen = strlen(UrlExeExts[i]); - ext = FindBuffer(url, len, (uint8_t *)UrlExeExts[i], (uint16_t)strlen(UrlExeExts[i])); - if (ext != NULL && (ext + extLen - url == (int)len || ext[extLen] == '?')) { - isExeUrl = 1; - break; - } - } - - return isExeUrl; -} - -/** - * \brief Function determines whether a host string is a numeric IP v4 address - * - * \param urlhost The host string - * \param len The host string length - * - * \retval 1 The host is a numeric IP - * \retval 0 The host is NOT a numeric IP - */ -static int IsIpv4Host(const uint8_t *urlhost, uint32_t len) -{ - struct sockaddr_in sa; - char tempIp[MAX_IP4_CHARS + 1]; - - /* Cut off at '/' */ - uint32_t i = 0; - for ( ; i < len && urlhost[i] != 0; i++) { - - if (urlhost[i] == '/') { - break; - } - } - - /* Too many chars */ - if (i > MAX_IP4_CHARS) { - return 0; - } - - /* Create null-terminated string */ - memcpy(tempIp, urlhost, i); - tempIp[i] = '\0'; - - if (!IPv4AddressStringIsValid(tempIp)) - return 0; - - return inet_pton(AF_INET, tempIp, &(sa.sin_addr)); -} - -/** - * \brief Function determines whether a host string is a numeric IP v6 address - * - * \param urlhost The host string - * \param len The host string length - * - * \retval 1 The host is a numeric IP - * \retval 0 The host is NOT a numeric IP - */ -static int IsIpv6Host(const uint8_t *urlhost, uint32_t len) -{ - struct in6_addr in6; - char tempIp[MAX_IP6_CHARS + 1]; - - /* Cut off at '/' */ - uint32_t i = 0; - for (i = 0; i < len && urlhost[i] != 0; i++) { - if (urlhost[i] == '/') { - break; - } - } - - /* Too many chars */ - if (i > MAX_IP6_CHARS) { - return 0; - } - - /* Create null-terminated string */ - memcpy(tempIp, urlhost, i); - tempIp[i] = '\0'; - - if (!IPv6AddressStringIsValid(tempIp)) - return 0; - - return inet_pton(AF_INET6, tempIp, &in6); -} - -/** - * \brief Traverses through the list of URLs for an exact match of the specified - * string - * - * \param entity The MIME entity - * \param url The matching URL string (lowercase) - * \param url_len The matching URL string length - * - * \return URL object or NULL if not found - */ -static MimeDecUrl *FindExistingUrl(MimeDecEntity *entity, uint8_t *url, uint32_t url_len) -{ - MimeDecUrl *curr = entity->url_list; - - while (curr != NULL) { - if (url_len == curr->url_len) { - /* search url and stored url are both in - * lowercase, so we can do an exact match */ - if (SCMemcmp(curr->url, url, url_len) == 0) { - break; - } - } - curr = curr->next; - } - - return curr; -} - -/** - * \brief This function searches a text or html line for a URL string - * - * The URL strings are searched for using the URL schemes defined in the global - * MIME config e.g. "http", "https". - * - * The found URL strings are stored in lowercase and with their schemes - * stripped unless the MIME config flag for log_url_scheme is set. - * - * Numeric IPs, malformed numeric IPs, and URLs pointing to executables are - * also flagged as URLs of interest. - * - * \param line the line - * \param len the line length - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int FindUrlStrings(const uint8_t *line, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; - MimeDecConfig *mdcfg = MimeDecGetConfig(); - uint8_t *fptr, *remptr, *tok = NULL, *tempUrl, *urlHost; - uint32_t tokLen = 0, i, tempUrlLen, urlHostLen; - uint16_t schemeStrLen = 0; - uint8_t flags = 0; - ConfNode *scheme = NULL; - char *schemeStr = NULL; - - if (mdcfg != NULL && mdcfg->extract_urls_schemes == NULL) { - SCLogDebug("Error: MIME config extract_urls_schemes was NULL."); - return MIME_DEC_ERR_DATA; - } - - TAILQ_FOREACH (scheme, &mdcfg->extract_urls_schemes->head, next) { - schemeStr = scheme->val; - // checked against UINT16_MAX when setting in SMTPConfigure - schemeStrLen = (uint16_t)strlen(schemeStr); - - remptr = (uint8_t *)line; - do { - SCLogDebug("Looking for URL String starting with: %s", schemeStr); - - /* Check for token definition */ - fptr = FindBuffer(remptr, len - (remptr - line), (uint8_t *)schemeStr, schemeStrLen); - if (fptr != NULL) { - if (!mdcfg->log_url_scheme) { - fptr += schemeStrLen; /* Strip scheme from stored URL */ - } - tok = GetToken(fptr, len - (fptr - line), " \"\'<>]\t", &remptr, &tokLen); - if (tok == fptr) { - SCLogDebug("Found url string"); - - /* First copy to temp URL string */ - tempUrl = SCMalloc(tokLen); - if (unlikely(tempUrl == NULL)) { - return MIME_DEC_ERR_MEM; - } - - PrintChars(SC_LOG_DEBUG, "RAW URL", tok, tokLen); - - /* Copy over to temp URL while decoding */ - tempUrlLen = 0; - for (i = 0; i < tokLen && tok[i] != 0; i++) { - /* url is all lowercase */ - tempUrl[tempUrlLen] = u8_tolower(tok[i]); - tempUrlLen++; - } - - urlHost = tempUrl; - urlHostLen = tempUrlLen; - if (mdcfg->log_url_scheme) { - /* tempUrl contains the scheme in the string but - * IsIpv4Host & IsPv6Host methods below require - * an input URL string with scheme stripped. Get a - * reference sub-string urlHost which starts with - * the host instead of the scheme. */ - urlHost += schemeStrLen; - urlHostLen -= schemeStrLen; - } - - /* Determine if URL points to an EXE */ - if (IsExeUrl(tempUrl, tempUrlLen)) { - flags |= URL_IS_EXE; - - PrintChars(SC_LOG_DEBUG, "EXE URL", tempUrl, tempUrlLen); - } - - /* Make sure remaining URL exists */ - if (tempUrlLen > 0) { - if (!(FindExistingUrl(entity, tempUrl, tempUrlLen))) { - /* Now look for numeric IP */ - if (IsIpv4Host(urlHost, urlHostLen)) { - flags |= URL_IS_IP4; - - PrintChars(SC_LOG_DEBUG, "IP URL4", tempUrl, tempUrlLen); - } else if (IsIpv6Host(urlHost, urlHostLen)) { - flags |= URL_IS_IP6; - - PrintChars(SC_LOG_DEBUG, "IP URL6", tempUrl, tempUrlLen); - } - - /* Add URL list item */ - MimeDecAddUrl(entity, tempUrl, tempUrlLen, flags); - } else { - SCFree(tempUrl); - } - } else { - SCFree(tempUrl); - } - - /* Reset flags for next URL */ - flags = 0; - } - } - } while (fptr != NULL); - } - - return ret; -} - -/** - * \brief This function is a pre-processor for handling decoded data chunks that - * then invokes the caller's callback function for further processing - * - * \param chunk The decoded chunk - * \param len The decoded chunk length (varies) - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessDecodedDataChunk(const uint8_t *chunk, uint32_t len, - MimeDecParseState *state) -{ - DEBUG_VALIDATE_BUG_ON(len > DATA_CHUNK_SIZE); - - int ret = MIME_DEC_OK; - uint8_t *remainPtr, *tok; - uint32_t tokLen; - - if ((state->stack != NULL) && (state->stack->top != NULL) && - (state->stack->top->data != NULL)) { - MimeDecConfig *mdcfg = MimeDecGetConfig(); - if (mdcfg != NULL && mdcfg->extract_urls) { - MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; - /* If plain text or html, then look for URLs */ - if (((entity->ctnt_flags & CTNT_IS_TEXT) || - (entity->ctnt_flags & CTNT_IS_MSG) || - (entity->ctnt_flags & CTNT_IS_HTML)) && - ((entity->ctnt_flags & CTNT_IS_ATTACHMENT) == 0)) { - - /* Parse each line one by one */ - remainPtr = (uint8_t *)chunk; - do { - tok = GetLine( - remainPtr, len - (remainPtr - (uint8_t *)chunk), &remainPtr, &tokLen); - if (tok != remainPtr) { - /* Search line for URL */ - ret = FindUrlStrings(tok, tokLen, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: FindUrlStrings() function" - " failed: %d", - ret); - break; - } - } - } while (tok != remainPtr && remainPtr - (uint8_t *)chunk < (int)len); - } - } - - /* Now invoke callback */ - if (state->DataChunkProcessorFunc != NULL) { - ret = state->DataChunkProcessorFunc(chunk, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: state->dataChunkProcessor() callback function" - " failed"); - } - } - } else { - SCLogDebug("Error: Stack pointer missing"); - ret = MIME_DEC_ERR_DATA; - } - - /* Reset data chunk buffer */ - state->data_chunk_len = 0; - - /* Mark body / file as no longer at beginning */ - state->body_begin = 0; - - return ret; -} - -/** - * \brief Processes a remainder (line % 4 = remainder) from the previous line - * such that all base64 decoding attempts are divisible by 4 - * - * \param buf The current line - * \param len The length of the line - * \param state The current parser state - * \param force Flag indicating whether decoding should always occur - * - * \return Number of bytes consumed from `buf` - */ -static uint32_t ProcessBase64Remainder( - const uint8_t *buf, const uint32_t len, MimeDecParseState *state, int force) -{ - uint32_t buf_consumed = 0; /* consumed bytes from 'buf' */ - uint8_t cnt = 0; - uint8_t block[B64_BLOCK]; - - SCLogDebug("len %u force %d", len, force); - - /* Strip spaces in remainder */ - for (uint8_t i = 0; i < state->bvr_len; i++) { - if (IsBase64Alphabet(state->bvremain[i])) { - block[cnt++] = state->bvremain[i]; - } else { - /* any invalid char is skipped over but it is consumed by the parser */ - buf_consumed++; - } - } - - /* should be impossible, but lets be defensive */ - DEBUG_VALIDATE_BUG_ON(cnt > B64_BLOCK); - if (cnt > B64_BLOCK) { - state->bvr_len = 0; - return 0; - } - - /* if we don't have 4 bytes see if we can fill it from `buf` */ - if (buf && len > 0 && cnt != B64_BLOCK) { - for (uint32_t i = 0; i < len && cnt < B64_BLOCK; i++) { - if (IsBase64Alphabet(buf[i])) { - block[cnt++] = buf[i]; - } - buf_consumed++; - } - DEBUG_VALIDATE_BUG_ON(cnt > B64_BLOCK); - for (uint32_t i = 0; i < cnt; i++) { - state->bvremain[i] = block[i]; - } - state->bvr_len = cnt; - } else if (!force && cnt != B64_BLOCK) { - SCLogDebug("incomplete data and no buffer to backfill"); - return 0; - } - - /* in force mode pad the block */ - if (force && cnt != B64_BLOCK) { - SCLogDebug("force and cnt %u != %u", cnt, B64_BLOCK); - for (uint8_t i = state->bvr_len; i < B64_BLOCK; i++) { - state->bvremain[state->bvr_len++] = '='; - } - } - - /* If data chunk buffer will be full, then clear it now */ - if (DATA_CHUNK_SIZE - state->data_chunk_len < ASCII_BLOCK) { - - /* Invoke pre-processor and callback */ - uint32_t ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function failed"); - } - } - - if (state->bvr_len == B64_BLOCK || force) { - uint32_t consumed_bytes = 0; - uint32_t remdec = 0; - const uint32_t avail_space = DATA_CHUNK_SIZE - state->data_chunk_len; - PrintChars(SC_LOG_DEBUG, "BASE64 INPUT (bvremain)", state->bvremain, state->bvr_len); - Base64Ecode code = DecodeBase64(state->data_chunk + state->data_chunk_len, avail_space, - state->bvremain, state->bvr_len, &consumed_bytes, &remdec, BASE64_MODE_RFC2045); - SCLogDebug("DecodeBase64 result %u", code); - if (remdec > 0 && (code == BASE64_ECODE_OK || code == BASE64_ECODE_BUF)) { - PrintChars(SC_LOG_DEBUG, "BASE64 DECODED (bvremain)", - state->data_chunk + state->data_chunk_len, remdec); - - state->data_chunk_len += remdec; - - /* If data chunk buffer is now full, then clear */ - if (DATA_CHUNK_SIZE - state->data_chunk_len < ASCII_BLOCK) { - - /* Invoke pre-processor and callback */ - uint32_t ret = - ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function " - "failed"); - } - } - } else if (code == BASE64_ECODE_ERR) { - /* Track failed base64 */ - state->stack->top->data->anomaly_flags |= ANOM_INVALID_BASE64; - state->msg->anomaly_flags |= ANOM_INVALID_BASE64; - SCLogDebug("Error: DecodeBase64() function failed"); - PrintChars(SC_LOG_DEBUG, "Base64 failed string", state->bvremain, state->bvr_len); - } - - /* Reset remaining */ - state->bvr_len = 0; - } - - DEBUG_VALIDATE_BUG_ON(buf_consumed > len); - return buf_consumed; -} - -static inline MimeDecRetCode ProcessBase64BodyLineCopyRemainder( - const uint8_t *buf, const uint32_t buf_len, const uint32_t offset, MimeDecParseState *state) -{ - DEBUG_VALIDATE_BUG_ON(offset > buf_len); - if (offset > buf_len) - return MIME_DEC_ERR_DATA; - - for (uint32_t i = offset; i < buf_len; i++) { - // Skip any characters outside of the base64 alphabet as per RFC 2045 - if (IsBase64Alphabet(buf[i])) { - DEBUG_VALIDATE_BUG_ON(state->bvr_len >= B64_BLOCK); - if (state->bvr_len >= B64_BLOCK) - return MIME_DEC_ERR_DATA; - state->bvremain[state->bvr_len++] = buf[i]; - } - } - return MIME_DEC_OK; -} - -/** - * \brief Processes a body line by base64-decoding and passing to the data chunk - * processing callback function when the buffer is read - * - * \param buf The current line - * \param len The length of the line - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessBase64BodyLine(const uint8_t *buf, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - uint32_t numDecoded, remaining = len, offset = 0; - - /* Track long line TODO should we count space padding too? */ - if (len > MAX_ENC_LINE_LEN) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_ENC_LINE; - state->msg->anomaly_flags |= ANOM_LONG_ENC_LINE; - SCLogDebug("max encoded input line length exceeded %u > %u", len, MAX_ENC_LINE_LEN); - } - - if (state->bvr_len + len < B64_BLOCK) { - return ProcessBase64BodyLineCopyRemainder(buf, len, 0, state); - } - - /* First process remaining from previous line. We will consume - * state->bvremain, filling it from 'buf' until we have a properly - * sized block. Spaces are skipped (rfc2045). If state->bvr_len - * is not 0 after processing we have no data left at 'buf'. */ - if (state->bvr_len > 0) { - uint32_t consumed = ProcessBase64Remainder(buf, len, state, 0); - DEBUG_VALIDATE_BUG_ON(consumed > len); - if (consumed > len) - return MIME_DEC_ERR_PARSE; - - uint32_t left = len - consumed; - if (left < B64_BLOCK) { - DEBUG_VALIDATE_BUG_ON(left + state->bvr_len > B64_BLOCK); - return ProcessBase64BodyLineCopyRemainder(buf, len, consumed, state); - } - - remaining -= consumed; - offset = consumed; - } - - while (remaining > 0 && remaining >= B64_BLOCK) { - uint32_t consumed_bytes = 0; - uint32_t avail_space = DATA_CHUNK_SIZE - state->data_chunk_len; - PrintChars(SC_LOG_DEBUG, "BASE64 INPUT (line)", buf + offset, remaining); - Base64Ecode code = DecodeBase64(state->data_chunk + state->data_chunk_len, avail_space, - buf + offset, remaining, &consumed_bytes, &numDecoded, BASE64_MODE_RFC2045); - SCLogDebug("DecodeBase64 result %u", code); - DEBUG_VALIDATE_BUG_ON(consumed_bytes > remaining); - if (consumed_bytes > remaining) - return MIME_DEC_ERR_PARSE; - - uint32_t leftover_bytes = remaining - consumed_bytes; - if (numDecoded > 0 && (code == BASE64_ECODE_OK || code == BASE64_ECODE_BUF)) { - PrintChars(SC_LOG_DEBUG, "BASE64 DECODED (line)", - state->data_chunk + state->data_chunk_len, numDecoded); - - state->data_chunk_len += numDecoded; - - if ((int)(DATA_CHUNK_SIZE - state->data_chunk_len) < 0) { - SCLogDebug("Error: Invalid Chunk length: %u", state->data_chunk_len); - return MIME_DEC_ERR_PARSE; - } - /* If buffer full, then invoke callback */ - if (DATA_CHUNK_SIZE - state->data_chunk_len < ASCII_BLOCK) { - /* Invoke pre-processor and callback */ - ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function failed"); - break; - } - } - } else if (code == BASE64_ECODE_ERR) { - /* Track failed base64 */ - state->stack->top->data->anomaly_flags |= ANOM_INVALID_BASE64; - state->msg->anomaly_flags |= ANOM_INVALID_BASE64; - SCLogDebug("Error: DecodeBase64() function failed"); - return MIME_DEC_ERR_DATA; - } - - /* corner case: multiples spaces in the last data, leading it to exceed the block - * size. We strip of spaces this while storing it in bvremain */ - if (consumed_bytes == 0 && leftover_bytes > B64_BLOCK) { - DEBUG_VALIDATE_BUG_ON(state->bvr_len != 0); - ret = ProcessBase64BodyLineCopyRemainder(buf, len, offset, state); - break; - } else if (leftover_bytes > 0 && leftover_bytes <= B64_BLOCK) { - /* If remaining is 4 by this time, we encountered spaces during processing */ - DEBUG_VALIDATE_BUG_ON(state->bvr_len != 0); - ret = ProcessBase64BodyLineCopyRemainder(buf, len, offset + consumed_bytes, state); - break; - } - - /* Update counts */ - remaining = leftover_bytes; - offset += consumed_bytes; - } - if (ret == MIME_DEC_OK && state->data_chunk_len > 0) { - ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - } - return ret; -} - -/** - * \brief Decoded a hex character into its equivalent byte value for - * quoted-printable decoding - * - * \param h The hex char - * - * \return byte value on success, -1 if failed - **/ -static int8_t DecodeQPChar(char h) -{ - int8_t res = 0; - - /* 0-9 */ - if (h >= 48 && h <= 57) { - res = h - 48; - } else if (h >= 65 && h <= 70) { - /* A-F */ - res = h - 55; - } else { - /* Invalid */ - res = -1; - } - - return res; - -} - -/** - * \brief Processes a quoted-printable encoded body line by decoding and passing - * to the data chunk processing callback function when the buffer is read - * - * \param buf The current line - * \param len The length of the line - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessQuotedPrintableBodyLine(const uint8_t *buf, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - uint32_t remaining, offset; - MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; - uint8_t c, h1, h2, val; - int16_t res; - - /* Track long line */ - if (len > MAX_ENC_LINE_LEN) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_ENC_LINE; - state->msg->anomaly_flags |= ANOM_LONG_ENC_LINE; - SCLogDebug("Error: Max encoded input line length exceeded %u > %u", - len, MAX_ENC_LINE_LEN); - } - if (len == 0) { - memcpy(state->data_chunk + state->data_chunk_len, buf + len, - state->current_line_delimiter_len); - state->data_chunk_len += state->current_line_delimiter_len; - return ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - } - - remaining = len; - offset = 0; - while (remaining > 0) { - - c = *(buf + offset); - - /* Copy over normal character */ - if (c != '=') { - state->data_chunk[state->data_chunk_len] = c; - state->data_chunk_len++; - - /* Add CRLF sequence if end of line, unless its a partial line */ - if (remaining == 1 && state->current_line_delimiter_len > 0) { - memcpy(state->data_chunk + state->data_chunk_len, CRLF, EOL_LEN); - state->data_chunk_len += EOL_LEN; - } - } else if (remaining > 1) { - /* If last character handle as soft line break by ignoring, - otherwise process as escaped '=' character */ - - /* Not enough characters */ - if (remaining < 3) { - entity->anomaly_flags |= ANOM_INVALID_QP; - state->msg->anomaly_flags |= ANOM_INVALID_QP; - SCLogDebug("Error: Quoted-printable decoding failed"); - } else { - h1 = *(buf + offset + 1); - res = DecodeQPChar(h1); - if (res < 0) { - entity->anomaly_flags |= ANOM_INVALID_QP; - state->msg->anomaly_flags |= ANOM_INVALID_QP; - SCLogDebug("Error: Quoted-printable decoding failed"); - } else { - val = (uint8_t)(res << 4); /* Shift result left */ - h2 = *(buf + offset + 2); - res = DecodeQPChar(h2); - if (res < 0) { - entity->anomaly_flags |= ANOM_INVALID_QP; - state->msg->anomaly_flags |= ANOM_INVALID_QP; - SCLogDebug("Error: Quoted-printable decoding failed"); - } else { - /* Decoding sequence succeeded */ - val += res; - - state->data_chunk[state->data_chunk_len] = val; - state->data_chunk_len++; - - /* Add CRLF sequence if end of line, unless for partial lines */ - if (remaining == 3 && state->current_line_delimiter_len > 0) { - memcpy(state->data_chunk + state->data_chunk_len, - CRLF, EOL_LEN); - state->data_chunk_len += EOL_LEN; - } - - /* Account for extra 2 characters in 3-character QP - * sequence */ - remaining -= 2; - offset += 2; - } - } - } - } - - /* Change by 1 */ - remaining--; - offset++; - - /* If buffer full, then invoke callback */ - if (DATA_CHUNK_SIZE - state->data_chunk_len < EOL_LEN + 1) { - - /* Invoke pre-processor and callback */ - ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, - state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function " - "failed"); - } - } - } - - return ret; -} - -/** - * \brief Processes a body line by base64-decoding (if applicable) and passing to - * the data chunk processing callback function - * - * \param buf The current line - * \param len The length of the line - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessBodyLine(const uint8_t *buf, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - uint32_t remaining, offset, avail, tobuf; - MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; - - SCLogDebug("Processing body line"); - - /* Process base-64 content if enabled */ - MimeDecConfig *mdcfg = MimeDecGetConfig(); - if (mdcfg != NULL && mdcfg->decode_base64 && - (entity->ctnt_flags & CTNT_IS_BASE64)) { - - ret = ProcessBase64BodyLine(buf, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessBase64BodyLine() function failed"); - } - } else if (mdcfg != NULL && mdcfg->decode_quoted_printable && - (entity->ctnt_flags & CTNT_IS_QP)) { - /* Process quoted-printable content if enabled */ - ret = ProcessQuotedPrintableBodyLine(buf, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessQuotedPrintableBodyLine() function " - "failed"); - } - } else { - /* Process non-decoded content */ - remaining = len; - offset = 0; - while (remaining > 0) { - /* Plan to add CRLF to the end of each line */ - avail = DATA_CHUNK_SIZE - state->data_chunk_len; - tobuf = avail > remaining ? remaining : avail; - - /* Copy over to buffer */ - memcpy(state->data_chunk + state->data_chunk_len, buf + offset, tobuf); - state->data_chunk_len += tobuf; - - if ((int) (DATA_CHUNK_SIZE - state->data_chunk_len) < 0) { - SCLogDebug("Error: Invalid Chunk length: %u", - state->data_chunk_len); - ret = MIME_DEC_ERR_PARSE; - break; - } - - /* If buffer full, then invoke callback */ - if (DATA_CHUNK_SIZE - state->data_chunk_len == 0) { - /* Invoke pre-processor and callback */ - ret = ProcessDecodedDataChunk(state->data_chunk, - state->data_chunk_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function " - "failed"); - } - } - - remaining -= tobuf; - offset += tobuf; - } - if (ret == MIME_DEC_OK) { - ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function " - "failed"); - } - } - // keep end of line for next call (and skip it on completion) - memcpy(state->data_chunk, buf + offset, state->current_line_delimiter_len); - state->data_chunk_len = state->current_line_delimiter_len; - } - - return ret; -} - -/** - * \brief Find the start of a header name on the current line - * - * \param buf The input line (not null-terminated) - * \param blen The length of the input line - * \param glen The output length of the header name - * - * \return Pointer to header name, or NULL if not found - */ -static uint8_t * FindMimeHeaderStart(const uint8_t *buf, uint32_t blen, uint32_t *hlen) -{ - uint32_t i, valid = 0; - uint8_t *hname = NULL; - - /* Init */ - *hlen = 0; - - /* Look for sequence of printable characters followed by ':', or - CRLF then printable characters followed by ':' */ - for (i = 0; i < blen && buf[i] != 0; i++) { - - /* If ready for printable characters and found one, then increment */ - if (buf[i] != COLON && buf[i] >= PRINTABLE_START && - buf[i] <= PRINTABLE_END) { - valid++; - } else if (valid > 0 && buf[i] == COLON) { - /* If ready for printable characters, found some, and found colon - * delimiter, then a match is found */ - hname = (uint8_t *) buf + i - valid; - *hlen = valid; - break; - } else { - /* Otherwise reset and quit */ - break; - } - } - - return hname; -} - -/** - * \brief Find full header name and value on the current line based on the - * current state - * - * \param buf The current line (no CRLF) - * \param blen The length of the current line - * \param state The current state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int FindMimeHeader(const uint8_t *buf, uint32_t blen, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - uint8_t *hname, *hval = NULL; - DataValue *dv; - uint32_t hlen, vlen; - int finish_header = 0, new_header = 0; - MimeDecConfig *mdcfg = MimeDecGetConfig(); - - DEBUG_VALIDATE_BUG_ON(state->current_line_delimiter_len == 0 && blen < SMTP_LINE_BUFFER_LIMIT); - - /* Find first header */ - hname = FindMimeHeaderStart(buf, blen, &hlen); - if (hname != NULL) { - - /* Warn and track but don't do anything yet */ - if (hlen > MAX_HEADER_NAME) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_HEADER_NAME; - state->msg->anomaly_flags |= ANOM_LONG_HEADER_NAME; - SCLogDebug("Error: Header name exceeds limit (%u > %u)", - hlen, MAX_HEADER_NAME); - } - - /* Value starts after 'header:' (normalize spaces) */ - hval = hname + hlen + 1; - if (hval - buf >= (int)blen) { - SCLogDebug("No Header value found"); - hval = NULL; - } else { - while (hval[0] == ' ') { - - /* If last character before end of bounds, set to NULL */ - if (hval - buf >= (int)blen - 1) { - SCLogDebug("No Header value found"); - hval = NULL; - break; - } - - hval++; - } - } - - /* If new header found, then previous header is finished */ - if (state->state_flag == HEADER_STARTED) { - finish_header = 1; - } - - /* Now process new header */ - new_header = 1; - - /* Must wait for next line to determine if finished */ - state->state_flag = HEADER_STARTED; - } else if (blen == 0) { - /* Found body */ - /* No more headers */ - state->state_flag = HEADER_DONE; - - finish_header = 1; - - SCLogDebug("All Header processing finished"); - } else if (state->state_flag == HEADER_STARTED) { - /* Found multi-line value (ie. Received header) */ - /* If max header value exceeded, flag it */ - vlen = blen; - if ((mdcfg != NULL) && (state->hvlen + vlen > mdcfg->header_value_depth)) { - SCLogDebug("Error: Header value of length (%u) is too long", - state->hvlen + vlen); - vlen = mdcfg->header_value_depth - state->hvlen; - state->stack->top->data->anomaly_flags |= ANOM_LONG_HEADER_VALUE; - state->msg->anomaly_flags |= ANOM_LONG_HEADER_VALUE; - } - if (vlen > 0) { - dv = AddDataValue(state->hvalue); - if (dv == NULL) { - return MIME_DEC_ERR_MEM; - } - if (state->hvalue == NULL) { - state->hvalue = dv; - } - - dv->value = SCMalloc(vlen); - if (unlikely(dv->value == NULL)) { - return MIME_DEC_ERR_MEM; - } - memcpy(dv->value, buf, vlen); - dv->value_len = vlen; - state->hvlen += vlen; - } - } else { - /* Likely a body without headers */ - SCLogDebug("No headers found"); - - state->state_flag = BODY_STARTED; - - /* Flag beginning of body */ - state->body_begin = 1; - state->body_end = 0; - - // Begin the body md5 computation if config asks so - if (MimeDecGetConfig()->body_md5 && state->md5_ctx == NULL) { - state->md5_ctx = SCMd5New(); - SCMd5Update(state->md5_ctx, buf, blen + state->current_line_delimiter_len); - } - - ret = ProcessBodyLine(buf, blen, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessBodyLine() function failed"); - return ret; - } - } - - /* If we need to finish a header, then do so below and then cleanup */ - if (finish_header) { - /* Store the header value */ - ret = StoreMimeHeader(state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: StoreMimeHeader() function failed"); - return ret; - } - } - - /* When next header is found, we always create a new one */ - if (new_header) { - /* Copy name and value to state */ - state->hname = SCMalloc(hlen); - if (unlikely(state->hname == NULL)) { - return MIME_DEC_ERR_MEM; - } - memcpy(state->hname, hname, hlen); - state->hlen = hlen; - - if (state->hvalue != NULL) { - SCLogDebug("Error: Parser failed due to unexpected header " - "value"); - return MIME_DEC_ERR_DATA; - } - - if (hval != NULL) { - /* If max header value exceeded, flag it */ - vlen = blen - (hval - buf); - if ((mdcfg != NULL) && (state->hvlen + vlen > mdcfg->header_value_depth)) { - SCLogDebug("Error: Header value of length (%u) is too long", - state->hvlen + vlen); - vlen = mdcfg->header_value_depth - state->hvlen; - state->stack->top->data->anomaly_flags |= ANOM_LONG_HEADER_VALUE; - state->msg->anomaly_flags |= ANOM_LONG_HEADER_VALUE; - } - - if (vlen > 0) { - state->hvalue = AddDataValue(NULL); - if (state->hvalue == NULL) { - return MIME_DEC_ERR_MEM; - } - state->hvalue->value = SCMalloc(vlen); - if (unlikely(state->hvalue->value == NULL)) { - return MIME_DEC_ERR_MEM; - } - memcpy(state->hvalue->value, hval, vlen); - state->hvalue->value_len = vlen; - state->hvlen += vlen; - } - } - } - - return ret; -} - -/** - * \brief Processes the current line for mime headers and also does post-processing - * when all headers found - * - * \param buf The current line - * \param len The length of the line - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - MimeDecField *field; - uint8_t *rptr = NULL; - uint32_t blen = 0; - MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; - uint8_t bptr[RS_MIME_MAX_TOKEN_LEN]; - - /* Look for mime header in current line */ - ret = FindMimeHeader(buf, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: FindMimeHeader() function failed: %d", ret); - return ret; - } - - /* Post-processing after all headers done */ - if (state->state_flag == HEADER_DONE) { - /* First determine encoding by looking at Content-Transfer-Encoding */ - field = MimeDecFindField(entity, CTNT_TRAN_STR); - if (field != NULL) { - /* Look for base64 */ - if (FindBuffer(field->value, field->value_len, (const uint8_t *)BASE64_STR, - (uint16_t)strlen(BASE64_STR))) { - SCLogDebug("Base64 encoding found"); - entity->ctnt_flags |= CTNT_IS_BASE64; - } else if (FindBuffer(field->value, field->value_len, (const uint8_t *)QP_STR, - (uint16_t)strlen(QP_STR))) { - /* Look for quoted-printable */ - SCLogDebug("quoted-printable encoding found"); - entity->ctnt_flags |= CTNT_IS_QP; - } - } - - /* Check for file attachment in content disposition */ - field = MimeDecFindField(entity, CTNT_DISP_STR); - if (field != NULL) { - bool truncated_name = false; - if (rs_mime_find_header_token(field->value, field->value_len, - (const uint8_t *)"filename", strlen("filename"), &bptr, &blen)) { - SCLogDebug("File attachment found in disposition"); - entity->ctnt_flags |= CTNT_IS_ATTACHMENT; - - if (blen > RS_MIME_MAX_TOKEN_LEN) { - blen = RS_MIME_MAX_TOKEN_LEN; - truncated_name = true; - } - - /* Copy over using dynamic memory */ - entity->filename = SCMalloc(blen); - if (unlikely(entity->filename == NULL)) { - return MIME_DEC_ERR_MEM; - } - memcpy(entity->filename, bptr, blen); - entity->filename_len = blen; - - if (truncated_name) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_FILENAME; - state->msg->anomaly_flags |= ANOM_LONG_FILENAME; - } - } - } - - /* Check for boundary, encapsulated message, and file name in Content-Type */ - field = MimeDecFindField(entity, CTNT_TYPE_STR); - if (field != NULL) { - /* Check if child entity boundary definition found */ - // RS_MIME_MAX_TOKEN_LEN is RS_MIME_MAX_TOKEN_LEN on the rust side - if (rs_mime_find_header_token(field->value, field->value_len, - (const uint8_t *)"boundary", strlen("boundary"), &bptr, &blen)) { - state->found_child = 1; - entity->ctnt_flags |= CTNT_IS_MULTIPART; - - if (blen > (BOUNDARY_BUF - 2)) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_BOUNDARY; - return MIME_DEC_ERR_PARSE; - } - - /* Store boundary in parent node */ - state->stack->top->bdef = SCMalloc(blen); - if (unlikely(state->stack->top->bdef == NULL)) { - return MIME_DEC_ERR_MEM; - } - memcpy(state->stack->top->bdef, bptr, blen); - state->stack->top->bdef_len = (uint16_t)blen; - } - - /* Look for file name (if not already found) */ - if (!(entity->ctnt_flags & CTNT_IS_ATTACHMENT)) { - bool truncated_name = false; - if (rs_mime_find_header_token(field->value, field->value_len, - (const uint8_t *)"name", strlen("name"), &bptr, &blen)) { - SCLogDebug("File attachment found"); - entity->ctnt_flags |= CTNT_IS_ATTACHMENT; - - if (blen > RS_MIME_MAX_TOKEN_LEN) { - blen = RS_MIME_MAX_TOKEN_LEN; - truncated_name = true; - } - - /* Copy over using dynamic memory */ - entity->filename = SCMalloc(blen); - if (unlikely(entity->filename == NULL)) { - return MIME_DEC_ERR_MEM; - } - memcpy(entity->filename, bptr, blen); - entity->filename_len = blen; - - if (truncated_name) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_FILENAME; - state->msg->anomaly_flags |= ANOM_LONG_FILENAME; - } - } - } - - /* Pull out short-hand content type */ - entity->ctnt_type = GetToken(field->value, field->value_len, " \r\n;", - &rptr, &entity->ctnt_type_len); - if (entity->ctnt_type != NULL) { - /* Check for encapsulated message */ - if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len, (const uint8_t *)MSG_STR, - (uint16_t)strlen(MSG_STR))) { - SCLogDebug("Found encapsulated message entity"); - - entity->ctnt_flags |= CTNT_IS_ENV; - - /* Create and push child to stack */ - MimeDecEntity *child = MimeDecAddEntity(entity); - if (child == NULL) - return MIME_DEC_ERR_MEM; - child->ctnt_flags |= (CTNT_IS_ENCAP | CTNT_IS_MSG); - PushStack(state->stack); - state->stack->top->data = child; - - /* Mark as encapsulated child */ - state->stack->top->is_encap = 1; - - /* Ready to parse headers */ - state->state_flag = HEADER_READY; - } else if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len, - (const uint8_t *)MULTIPART_STR, - (uint16_t)strlen(MULTIPART_STR))) { - /* Check for multipart */ - SCLogDebug("Found multipart entity"); - entity->ctnt_flags |= CTNT_IS_MULTIPART; - } else if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len, - (const uint8_t *)TXT_STR, (uint16_t)strlen(TXT_STR))) { - /* Check for plain text */ - SCLogDebug("Found plain text entity"); - entity->ctnt_flags |= CTNT_IS_TEXT; - } else if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len, - (const uint8_t *)HTML_STR, (uint16_t)strlen(HTML_STR))) { - /* Check for html */ - SCLogDebug("Found html entity"); - entity->ctnt_flags |= CTNT_IS_HTML; - } - } - } - - /* Store pointer to Message-ID */ - field = MimeDecFindField(entity, MSG_ID_STR); - if (field != NULL) { - entity->msg_id = field->value; - entity->msg_id_len = field->value_len; - } - - /* Flag beginning of body */ - state->body_begin = 1; - state->body_end = 0; - } - - return ret; -} - -/** - * \brief Indicates to the parser that the body of an entity has completed - * processing on the previous line - * - * \param state The current parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ - -static int ProcessBodyComplete(MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - - SCLogDebug("Process body complete called"); - - /* Mark the file as hitting the end */ - state->body_end = 1; - - if (state->bvr_len > 0) { - SCLogDebug("Found (%u) remaining base64 bytes not processed", - state->bvr_len); - - /* Process the remainder */ - ret = ProcessBase64Remainder(NULL, 0, state, 1); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessBase64BodyLine() function failed"); - } - } - - MimeDecEntity *entity = (MimeDecEntity *)state->stack->top->data; - if ((entity->ctnt_flags & (CTNT_IS_BASE64 | CTNT_IS_QP)) == 0) { - // last eol of plaintext is the beginning of the boundary - state->data_chunk_len = 0; - } - /* Invoke pre-processor and callback with remaining data */ - ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessDecodedDataChunk() function failed"); - } - - /* Now reset */ - state->body_begin = 0; - state->body_end = 0; - - return ret; -} - -/** - * \brief When a mime boundary is found, look for end boundary and also do stack - * management - * - * \param buf The current line - * \param len The length of the line - * \param bdef_len The length of the current boundary - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessMimeBoundary( - const uint8_t *buf, uint32_t len, uint16_t bdef_len, MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - uint8_t *rptr; - MimeDecEntity *child; - - SCLogDebug("PROCESSING BOUNDARY - START: %d", - state->state_flag); - - /* If previous line was not an end boundary, then we process the body as - * completed */ - if (state->state_flag != BODY_END_BOUND) { - - /* First lets complete the body */ - ret = ProcessBodyComplete(state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessBodyComplete() function failed"); - return ret; - } - } else { - /* If last line was an end boundary, then now we are ready to parse - * headers again */ - state->state_flag = HEADER_READY; - } - - /* Update remaining buffer */ - rptr = (uint8_t *) buf + bdef_len + 2; - - /* If entity is encapsulated and current and parent didn't define the boundary, - * then pop out */ - if (state->stack->top->is_encap && state->stack->top->bdef_len == 0) { - - if (state->stack->top->next == NULL) { - SCLogDebug("Error: Missing parent entity from stack"); - return MIME_DEC_ERR_DATA; - } - - if (state->stack->top->next->bdef_len == 0) { - - SCLogDebug("POPPED ENCAPSULATED CHILD FROM STACK: %p=%p", - state->stack->top, state->stack->top->data); - - /* If end of boundary found, pop the child off the stack */ - PopStack(state->stack); - if (state->stack->top == NULL) { - SCLogDebug("Error: Message is malformed"); - return MIME_DEC_ERR_DATA; - } - } - } - - /* Now check for end of nested boundary */ - if (len - (rptr - buf) > 1 && rptr[0] == DASH && rptr[1] == DASH) { - SCLogDebug("FOUND END BOUNDARY, POPPING: %p=%p", - state->stack->top, state->stack->top->data); - - /* If end of boundary found, pop the child off the stack */ - PopStack(state->stack); - if (state->stack->top == NULL) { - SCLogDebug("Error: Message is malformed"); - return MIME_DEC_ERR_DATA; - } - - /* If current is an encapsulated message with a boundary definition, - * then pop him as well */ - if (state->stack->top->is_encap && state->stack->top->bdef_len != 0) { - SCLogDebug("FOUND END BOUNDARY AND ENCAP, POPPING: %p=%p", - state->stack->top, state->stack->top->data); - - PopStack(state->stack); - if (state->stack->top == NULL) { - SCLogDebug("Error: Message is malformed"); - return MIME_DEC_ERR_DATA; - } - } - - state->state_flag = BODY_END_BOUND; - } else if (state->found_child) { - /* Otherwise process new child */ - SCLogDebug("Child entity created"); - - /* Create and push child to stack */ - child = MimeDecAddEntity(state->stack->top->data); - if (child == NULL) - return MIME_DEC_ERR_MEM; - child->ctnt_flags |= CTNT_IS_BODYPART; - PushStack(state->stack); - state->stack->top->data = child; - - /* Reset flag */ - state->found_child = 0; - } else { - /* Otherwise process sibling */ - if (state->stack->top->next == NULL) { - SCLogDebug("Error: Missing parent entity from stack"); - return MIME_DEC_ERR_DATA; - } - - SCLogDebug("SIBLING CREATED, POPPING PARENT: %p=%p", - state->stack->top, state->stack->top->data); - - /* First pop current to get access to parent */ - PopStack(state->stack); - if (state->stack->top == NULL) { - SCLogDebug("Error: Message is malformed"); - return MIME_DEC_ERR_DATA; - } - - /* Create and push child to stack */ - child = MimeDecAddEntity(state->stack->top->data); - if (child == NULL) - return MIME_DEC_ERR_MEM; - child->ctnt_flags |= CTNT_IS_BODYPART; - PushStack(state->stack); - state->stack->top->data = child; - } - - /* After boundary look for headers */ - if (state->state_flag != BODY_END_BOUND) { - state->state_flag = HEADER_READY; - } - - SCLogDebug("PROCESSING BOUNDARY - END: %d", state->state_flag); - return ret; -} - -/** - * \brief Processes the MIME Entity body based on the input line and current - * state of the parser - * - * \param buf The current line - * \param len The length of the line - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessMimeBody(const uint8_t *buf, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - uint8_t temp[BOUNDARY_BUF]; - uint8_t *bstart; - int body_found = 0; - uint16_t tlen; - - /* pass empty lines on if we're parsing the body, otherwise we have no use - * for them, and in fact they would disrupt the state tracking */ - if (len == 0) { - /* don't start a new body after an end bound based on an empty line */ - if (state->state_flag == BODY_END_BOUND) { - SCLogDebug("skip empty line"); - return MIME_DEC_OK; - } else if (state->state_flag == HEADER_DONE) { - SCLogDebug("empty line, lets see if we skip it. We're in state %s", - MimeDecParseStateGetStatus(state)); - MimeDecEntity *entity = (MimeDecEntity *)state->stack->top->data; - MimeDecConfig *mdcfg = MimeDecGetConfig(); - if (entity != NULL && mdcfg != NULL) { - if (mdcfg->decode_base64 && (entity->ctnt_flags & CTNT_IS_BASE64)) { - SCLogDebug("skip empty line"); - return MIME_DEC_OK; - } - SCLogDebug("not skipping empty line"); - } - } else { - SCLogDebug("not skipping line at state %s", MimeDecParseStateGetStatus(state)); - } - } - - /* First look for boundary */ - MimeDecStackNode *node = state->stack->top; - if (node == NULL) { - SCLogDebug("Error: Invalid stack state"); - return MIME_DEC_ERR_PARSE; - } - - /* Traverse through stack to find a boundary definition */ - if (state->state_flag == BODY_END_BOUND || node->bdef == NULL) { - - /* If not found, then use parent's boundary */ - node = node->next; - while (node != NULL && node->bdef == NULL) { - SCLogDebug("Traversing through stack for node with boundary"); - node = node->next; - } - } - - /* This means no boundary / parent w/boundary was found so we are in the body */ - if (node == NULL) { - body_found = 1; - } else { - - /* Now look for start of boundary */ - if (len > 1 && buf[0] == '-' && buf[1] == '-') { - - tlen = node->bdef_len + 2; - if (tlen > BOUNDARY_BUF) { - if (state->stack->top->data) - state->stack->top->data->anomaly_flags |= ANOM_LONG_BOUNDARY; - SCLogDebug("Error: Long boundary: tlen %u > %d. Set ANOM_LONG_BOUNDARY", tlen, - BOUNDARY_BUF); - return MIME_DEC_ERR_PARSE; - } - - memcpy(temp, "--", 2); - memcpy(temp + 2, node->bdef, node->bdef_len); - - /* Find either next boundary or end boundary */ - bstart = FindBuffer(buf, len, temp, tlen); - if (bstart != NULL) { - ret = ProcessMimeBoundary(buf, len, node->bdef_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessMimeBoundary() function " - "failed"); - return ret; - } - } else { - /* Otherwise add value to body */ - body_found = 1; - } - } else { - /* Otherwise add value to body */ - body_found = 1; - } - } - - /* Process body line */ - if (body_found) { - state->state_flag = BODY_STARTED; - - ret = ProcessBodyLine(buf, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessBodyLine() function failed"); - return ret; - } - } - - return ret; -} - -const char *MimeDecParseStateGetStatus(MimeDecParseState *state) -{ - return StateFlags[state->state_flag]; -} - -/** - * \brief Processes the MIME Entity based on the input line and current state of - * the parser - * - * \param buf The current line - * \param len The length of the line - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -static int ProcessMimeEntity(const uint8_t *buf, uint32_t len, - MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - - SCLogDebug("START FLAG: %s", StateFlags[state->state_flag]); - - if (state->state_flag == PARSE_ERROR) { - SCLogDebug("START FLAG: PARSE_ERROR, bail"); - return MIME_DEC_ERR_STATE; - } - - /* Track long line */ - if (len > MAX_LINE_LEN) { - state->stack->top->data->anomaly_flags |= ANOM_LONG_LINE; - state->msg->anomaly_flags |= ANOM_LONG_LINE; - SCLogDebug("Error: Max input line length exceeded %u > %u", len, - MAX_LINE_LEN); - } - - if (!g_disable_hashing) { - if ((state->state_flag != HEADER_READY && state->state_flag != HEADER_STARTED) || - (state->stack->top->data->ctnt_flags & CTNT_IS_BODYPART)) { - if (MimeDecGetConfig()->body_md5) { - if (state->body_begin == 1 && state->md5_ctx == NULL) { - state->md5_ctx = SCMd5New(); - } - SCMd5Update(state->md5_ctx, buf, len + state->current_line_delimiter_len); - } - } - } - - /* Looking for headers */ - if (state->state_flag == HEADER_READY || - state->state_flag == HEADER_STARTED) { - - SCLogDebug("Processing Headers"); - - /* Process message headers */ - ret = ProcessMimeHeaders(buf, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessMimeHeaders() function failed: %d", - ret); - return ret; - } - } else { - /* Processing body */ - SCLogDebug("Processing Body of: %p", state->stack->top); - - ret = ProcessMimeBody(buf, len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessMimeBody() function failed: %d", - ret); - return ret; - } - } - - SCLogDebug("END FLAG: %s", StateFlags[state->state_flag]); - - return ret; -} - -/** - * \brief Init the parser by allocating memory for the state and top-level entity - * - * \param data A caller-specified pointer to data for access within the data chunk - * processor callback function - * \param dcpfunc The data chunk processor callback function - * - * \return A pointer to the state object, or NULL if the operation fails - */ -MimeDecParseState * MimeDecInitParser(void *data, - int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, - MimeDecParseState *state)) -{ - MimeDecParseState *state; - MimeDecEntity *mimeMsg; - - state = SCCalloc(1, sizeof(MimeDecParseState)); - if (unlikely(state == NULL)) { - return NULL; - } - - state->stack = SCCalloc(1, sizeof(MimeDecStack)); - if (unlikely(state->stack == NULL)) { - SCFree(state); - return NULL; - } - - mimeMsg = SCCalloc(1, sizeof(MimeDecEntity)); - if (unlikely(mimeMsg == NULL)) { - SCFree(state->stack); - SCFree(state); - return NULL; - } - mimeMsg->ctnt_flags |= CTNT_IS_MSG; - - /* Init state */ - state->msg = mimeMsg; - PushStack(state->stack); - if (state->stack->top == NULL) { - SCFree(state->stack); - SCFree(state->msg); - SCFree(state); - return NULL; - } - state->stack->top->data = mimeMsg; - state->state_flag = HEADER_READY; - state->data = data; - state->DataChunkProcessorFunc = DataChunkProcessorFunc; - - return state; -} - -/** - * \brief De-Init parser by freeing up any residual memory - * - * \param state The parser state - * - * \return none - */ -void MimeDecDeInitParser(MimeDecParseState *state) -{ - uint32_t cnt = 0; - - while (state->stack->top != NULL) { - SCLogDebug("Remaining on stack: [%p]=>[%p]", - state->stack->top, state->stack->top->data); - - PopStack(state->stack); - cnt++; - } - - if (cnt > 1) { - state->msg->anomaly_flags |= ANOM_MALFORMED_MSG; - SCLogDebug("Warning: Stack is not empty upon completion of " - "processing (%u items remaining)", cnt); - } - - SCFree(state->hname); - FreeDataValue(state->hvalue); - FreeMimeDecStack(state->stack); - if (state->md5_ctx) - SCMd5Free(state->md5_ctx); - SCFree(state); -} - -/** - * \brief Called to indicate that the last message line has been processed and - * the parsing operation is complete - * - * This function should be called directly by the caller. - * - * \param state The parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -int MimeDecParseComplete(MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - - SCLogDebug("Parsing flagged as completed"); - - if (state->state_flag == PARSE_ERROR) { - SCLogDebug("parser in error state: PARSE_ERROR"); - return MIME_DEC_ERR_STATE; - } - - /* Store the header value */ - ret = StoreMimeHeader(state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: StoreMimeHeader() function failed"); - return ret; - } - - /* Lets complete the body */ - ret = ProcessBodyComplete(state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: ProcessBodyComplete() function failed"); - return ret; - } - - if (state->md5_ctx) { - SCMd5Finalize(state->md5_ctx, state->md5, sizeof(state->md5)); - state->md5_ctx = NULL; - state->has_md5 = true; - } - - if (state->stack->top == NULL) { - state->msg->anomaly_flags |= ANOM_MALFORMED_MSG; - SCLogDebug("Error: Message is malformed"); - return MIME_DEC_ERR_DATA; - } - - /* If encapsulated, pop off the stack */ - if (state->stack->top->is_encap) { - PopStack(state->stack); - if (state->stack->top == NULL) { - state->msg->anomaly_flags |= ANOM_MALFORMED_MSG; - SCLogDebug("Error: Message is malformed"); - return MIME_DEC_ERR_DATA; - } - } - - /* Look extra stack items remaining */ - if (state->stack->top->next != NULL) { - state->msg->anomaly_flags |= ANOM_MALFORMED_MSG; - SCLogDebug("Warning: Message has unclosed message part boundary"); - } - - state->state_flag = PARSE_DONE; - - return ret; -} - -/** - * \brief Parse a line of a MIME message and update the parser state - * - * \param line A string representing the line (w/out CRLF) - * \param len The length of the line - * \param delim_len The length of the line end delimiter - * \param state The parser state - * - * \return MIME_DEC_OK on success, otherwise < 0 on failure - */ -int MimeDecParseLine(const uint8_t *line, const uint32_t len, - const uint8_t delim_len, MimeDecParseState *state) -{ - int ret = MIME_DEC_OK; - - /* For debugging purposes */ - if (len > 0) { - PrintChars(SC_LOG_DEBUG, "SMTP LINE", line, len); - } else { - SCLogDebug("SMTP LINE - EMPTY"); - } - - state->current_line_delimiter_len = delim_len; - /* Process the entity */ - ret = ProcessMimeEntity(line, len, state); - if (ret != MIME_DEC_OK) { - state->state_flag = PARSE_ERROR; - SCLogDebug("Error: ProcessMimeEntity() function failed: %d", ret); - } - - return ret; -} - -/** - * \brief Parses an entire message when available in its entirety (wraps the - * line-based parsing functions) - * - * \param buf Buffer pointing to the full message - * \param blen Length of the buffer - * \param data Caller data to be available in callback - * \param dcpfunc Callback for processing each decoded body data chunk - * - * \return A pointer to the decoded MIME message, or NULL if the operation fails - */ -MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, - int (*dcpfunc)(const uint8_t *chunk, uint32_t len, - MimeDecParseState *state)) -{ - int ret = MIME_DEC_OK; - uint8_t *remainPtr, *tok; - uint32_t tokLen; - - MimeDecParseState *state = MimeDecInitParser(data, dcpfunc); - if (state == NULL) { - SCLogDebug("Error: MimeDecInitParser() function failed to create " - "state"); - return NULL; - } - - MimeDecEntity *msg = state->msg; - - /* Parse each line one by one */ - remainPtr = (uint8_t *) buf; - uint8_t *line = NULL; - do { - tok = GetLine(remainPtr, blen - (remainPtr - buf), &remainPtr, &tokLen); - if (tok != remainPtr) { - - line = tok; - - if ((remainPtr - tok) - tokLen > UINT8_MAX) { - SCLogDebug("Error: MimeDecParseLine() overflow: %ld", (remainPtr - tok) - tokLen); - ret = MIME_DEC_ERR_OVERFLOW; - break; - } - state->current_line_delimiter_len = (uint8_t)((remainPtr - tok) - tokLen); - /* Parse the line */ - ret = MimeDecParseLine(line, tokLen, state->current_line_delimiter_len, state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: MimeDecParseLine() function failed: %d", - ret); - break; - } - } - - } while (tok != remainPtr && remainPtr - buf < (int)blen); - - if (ret == MIME_DEC_OK) { - SCLogDebug("Message parser was successful"); - - /* Now complete message */ - ret = MimeDecParseComplete(state); - if (ret != MIME_DEC_OK) { - SCLogDebug("Error: MimeDecParseComplete() function failed"); - } - } - - /* De-allocate memory for parser */ - MimeDecDeInitParser(state); - - if (ret != MIME_DEC_OK) { - MimeDecFreeEntity(msg); - msg = NULL; - } - - return msg; -} - -#ifdef UNITTESTS - -/* Helper body chunk callback function */ -static int TestDataChunkCallback(const uint8_t *chunk, uint32_t len, - MimeDecParseState *state) -{ - uint32_t *line_count = (uint32_t *) state->data; - - if (state->body_begin) { - SCLogDebug("Body begin (len=%u)", len); - } - - /* Add up the line counts */ - if (len > 0) { - if ((*line_count) == 0) { - (*line_count)++; - } - PrintChars(SC_LOG_DEBUG, "CHUNK", chunk, len); - for (uint32_t i = 0; i < len; i++) { - if (chunk[i] == CR || chunk[i] == LF) { - if (i + 1 < len && chunk[i] != chunk[i + 1] && - (chunk[i + 1] == CR || chunk[i + 1] == LF)) { - i++; - } - (*line_count)++; - } - } - - SCLogDebug("line count (len=%u): %u", len, *line_count); - } - - if (state->body_end) { - SCLogDebug("Body end (len=%u)", len); - } - - return MIME_DEC_OK; -} - -/* Test simple case of line counts */ -static int MimeDecParseLineTest01(void) -{ - uint32_t line_count = 0; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, - TestDataChunkCallback); - - const char *str = "From: Sender1\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - str = "To: Recipient1\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - str = "Content-Type: text/plain\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - str = "\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - str = "A simple message line 1\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - str = "A simple message line 2\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - str = "A simple message line 3\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 1, 1, state) == MIME_DEC_OK); - - /* Completed */ - FAIL_IF_NOT(MimeDecParseComplete(state) == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT_NULL(msg->next); - FAIL_IF_NOT_NULL(msg->child); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - FAIL_IF_NOT(line_count == 3); - PASS; -} - -/* Test simple case of EXE URL extraction */ -static int MimeDecParseLineTest02(void) -{ - uint32_t line_count = 0; - - ConfNode *url_schemes = ConfNodeNew(); - ConfNode *scheme = ConfNodeNew(); - FAIL_IF_NULL(url_schemes); - FAIL_IF_NULL(scheme); - - url_schemes->is_seq = 1; - scheme->val = SCStrdup("http://"); - FAIL_IF_NULL(scheme->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme, next); - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - MimeDecGetConfig()->extract_urls_schemes = url_schemes; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, - TestDataChunkCallback); - - const char *str = "From: Sender1\r\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 2, 2, state) == MIME_DEC_OK); - - str = "To: Recipient1\r\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 2, 2, state) == MIME_DEC_OK); - - str = "Content-Type: text/plain\r\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 2, 2, state) == MIME_DEC_OK); - - str = "\r\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 2, 2, state) == MIME_DEC_OK); - - str = "A simple message line 1\r\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 2, 2, state) == MIME_DEC_OK); - - str = "A simple message line 2 click on http://www.test.com/malware.exe?" - "hahah hopefully you click this link\r\n"; - FAIL_IF_NOT(MimeDecParseLine((uint8_t *)str, strlen(str) - 2, 2, state) == MIME_DEC_OK); - - /* Completed */ - FAIL_IF_NOT(MimeDecParseComplete(state) == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NULL(msg); - FAIL_IF_NULL(msg->url_list); - FAIL_IF_NOT((msg->url_list->url_flags & URL_IS_EXE)); - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - ConfNodeFree(url_schemes); - MimeDecGetConfig()->extract_urls_schemes = NULL; - - FAIL_IF_NOT(line_count == 2); - PASS; -} - -/* Test error case where no url schemes set in config */ -static int MimeFindUrlStringsTest01(void) -{ - int ret = MIME_DEC_OK; - uint32_t line_count = 0; - - MimeDecGetConfig()->extract_urls = true; - MimeDecGetConfig()->extract_urls_schemes = NULL; - MimeDecGetConfig()->log_url_scheme = false; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - - const char *str = "test"; - ret = FindUrlStrings((uint8_t *)str, strlen(str), state); - /* Expected error since extract_url_schemes is NULL */ - FAIL_IF_NOT(ret == MIME_DEC_ERR_DATA); - - /* Completed */ - ret = MimeDecParseComplete(state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -/* Test simple case of URL extraction */ -static int MimeFindUrlStringsTest02(void) -{ - int ret = MIME_DEC_OK; - uint32_t line_count = 0; - ConfNode *url_schemes = ConfNodeNew(); - ConfNode *scheme = ConfNodeNew(); - FAIL_IF_NULL(url_schemes); - FAIL_IF_NULL(scheme); - - url_schemes->is_seq = 1; - scheme->val = SCStrdup("http://"); - FAIL_IF_NULL(scheme->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme, next); - - MimeDecGetConfig()->extract_urls = true; - MimeDecGetConfig()->extract_urls_schemes = url_schemes; - MimeDecGetConfig()->log_url_scheme = false; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - - const char *str = "A simple message click on " - "http://www.test.com/malware.exe? " - "hahah hopefully you click this link"; - ret = FindUrlStrings((uint8_t *)str, strlen(str), state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - /* Completed */ - ret = MimeDecParseComplete(state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - - FAIL_IF(msg->url_list == NULL); - - FAIL_IF_NOT(msg->url_list->url_flags & URL_IS_EXE); - FAIL_IF_NOT( - memcmp("www.test.com/malware.exe?", msg->url_list->url, msg->url_list->url_len) == 0); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - ConfNodeFree(url_schemes); - MimeDecGetConfig()->extract_urls_schemes = NULL; - - PASS; -} - -/* Test URL extraction with multiple schemes and URLs */ -static int MimeFindUrlStringsTest03(void) -{ - int ret = MIME_DEC_OK; - uint32_t line_count = 0; - ConfNode *url_schemes = ConfNodeNew(); - ConfNode *scheme1 = ConfNodeNew(); - ConfNode *scheme2 = ConfNodeNew(); - FAIL_IF_NULL(url_schemes); - FAIL_IF_NULL(scheme1); - FAIL_IF_NULL(scheme2); - - url_schemes->is_seq = 1; - scheme1->val = SCStrdup("http://"); - FAIL_IF_NULL(scheme1->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme1, next); - scheme2->val = SCStrdup("https://"); - FAIL_IF_NULL(scheme2->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme2, next); - - MimeDecGetConfig()->extract_urls = true; - MimeDecGetConfig()->extract_urls_schemes = url_schemes; - MimeDecGetConfig()->log_url_scheme = false; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - - const char *str = "A simple message click on " - "http://www.test.com/malware.exe? " - "hahah hopefully you click this link, or " - "you can go to http://www.test.com/test/01.html and " - "https://www.test.com/test/02.php"; - ret = FindUrlStrings((uint8_t *)str, strlen(str), state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - /* Completed */ - ret = MimeDecParseComplete(state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - - FAIL_IF(msg->url_list == NULL); - - MimeDecUrl *url = msg->url_list; - FAIL_IF_NOT(memcmp("www.test.com/test/02.php", url->url, url->url_len) == 0); - - url = url->next; - FAIL_IF_NOT(memcmp("www.test.com/test/01.html", url->url, url->url_len) == 0); - - url = url->next; - FAIL_IF_NOT(memcmp("www.test.com/malware.exe?", url->url, url->url_len) == 0); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - ConfNodeFree(url_schemes); - MimeDecGetConfig()->extract_urls_schemes = NULL; - - PASS; -} - -/* Test URL extraction with multiple schemes and URLs with - * log_url_scheme enabled in the MIME config */ -static int MimeFindUrlStringsTest04(void) -{ - int ret = MIME_DEC_OK; - uint32_t line_count = 0; - ConfNode *url_schemes = ConfNodeNew(); - ConfNode *scheme1 = ConfNodeNew(); - ConfNode *scheme2 = ConfNodeNew(); - FAIL_IF_NULL(url_schemes); - FAIL_IF_NULL(scheme1); - FAIL_IF_NULL(scheme2); - - url_schemes->is_seq = 1; - scheme1->val = SCStrdup("http://"); - FAIL_IF_NULL(scheme1->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme1, next); - scheme2->val = SCStrdup("https://"); - FAIL_IF_NULL(scheme2->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme2, next); - - MimeDecGetConfig()->extract_urls = true; - MimeDecGetConfig()->extract_urls_schemes = url_schemes; - MimeDecGetConfig()->log_url_scheme = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - - const char *str = "A simple message click on " - "http://www.test.com/malware.exe? " - "hahah hopefully you click this link, or " - "you can go to http://www.test.com/test/01.html and " - "https://www.test.com/test/02.php"; - ret = FindUrlStrings((uint8_t *)str, strlen(str), state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - /* Completed */ - ret = MimeDecParseComplete(state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - - FAIL_IF(msg->url_list == NULL); - - MimeDecUrl *url = msg->url_list; - FAIL_IF_NOT(memcmp("https://www.test.com/test/02.php", url->url, url->url_len) == 0); - - url = url->next; - FAIL_IF_NOT(memcmp("http://www.test.com/test/01.html", url->url, url->url_len) == 0); - - url = url->next; - FAIL_IF_NOT(memcmp("http://www.test.com/malware.exe?", url->url, url->url_len) == 0); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - ConfNodeFree(url_schemes); - MimeDecGetConfig()->extract_urls_schemes = NULL; - - PASS; -} - -/* Test URL extraction of IPV4 and IPV6 URLs with log_url_scheme - * enabled in the MIME config */ -static int MimeFindUrlStringsTest05(void) -{ - int ret = MIME_DEC_OK; - uint32_t line_count = 0; - ConfNode *url_schemes = ConfNodeNew(); - ConfNode *scheme = ConfNodeNew(); - FAIL_IF_NULL(url_schemes); - FAIL_IF_NULL(scheme); - - url_schemes->is_seq = 1; - scheme->val = SCStrdup("http://"); - FAIL_IF_NULL(scheme->val); - TAILQ_INSERT_TAIL(&url_schemes->head, scheme, next); - - MimeDecGetConfig()->extract_urls = true; - MimeDecGetConfig()->extract_urls_schemes = url_schemes; - MimeDecGetConfig()->log_url_scheme = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - - const char *str = "A simple message click on " - "http://192.168.1.1/test/01.html " - "hahah hopefully you click this link or this one " - "http://0:0:0:0:0:0:0:0/test/02.php"; - ret = FindUrlStrings((uint8_t *)str, strlen(str), state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - /* Completed */ - ret = MimeDecParseComplete(state); - FAIL_IF_NOT(ret == MIME_DEC_OK); - - MimeDecEntity *msg = state->msg; - - FAIL_IF(msg->url_list == NULL); - - MimeDecUrl *url = msg->url_list; - FAIL_IF_NOT(url->url_flags & URL_IS_IP6); - FAIL_IF_NOT(memcmp("http://0:0:0:0:0:0:0:0/test/02.php", url->url, url->url_len) == 0); - - url = url->next; - FAIL_IF_NOT(url->url_flags & URL_IS_IP4); - FAIL_IF_NOT(memcmp("http://192.168.1.1/test/01.html", url->url, url->url_len) == 0); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - ConfNodeFree(url_schemes); - MimeDecGetConfig()->extract_urls_schemes = NULL; - - PASS; -} - -/* Test full message with linebreaks */ -static int MimeDecParseFullMsgTest01(void) -{ - uint32_t expected_count = 3; - uint32_t line_count = 0; - - char msg[] = "From: Sender1\r\n" - "To: Recipient1\r\n" - "Content-Type: text/plain\r\n" - "\r\n" - "Line 1\r\n" - "Line 2\r\n" - "Line 3\r\n"; - - MimeDecEntity *entity = MimeDecParseFullMsg((uint8_t *)msg, strlen(msg), &line_count, - TestDataChunkCallback); - if (entity == NULL) { - SCLogInfo("Warning: Message failed to parse"); - return 0; - } - - MimeDecFreeEntity(entity); - - if (expected_count != line_count) { - SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d", - expected_count, line_count); - return 0; - } - - return 1; -} - -/* Test full message with linebreaks */ -static int MimeDecParseFullMsgTest02(void) -{ - uint32_t expected_count = 3; - uint32_t line_count = 0; - - char msg[] = "From: Sender2\r\n" - "To: Recipient2\r\n" - "Subject: subject2\r\n" - "Content-Type: text/plain\r\n" - "\r\n" - "Line 1\r\n" - "Line 2\r\n" - "Line 3\r\n"; - - MimeDecEntity *entity = MimeDecParseFullMsg((uint8_t *)msg, strlen(msg), &line_count, - TestDataChunkCallback); - - if (entity == NULL) { - SCLogInfo("Warning: Message failed to parse"); - return 0; - } - - MimeDecField *field = MimeDecFindField(entity, "subject"); - if (field == NULL) { - SCLogInfo("Warning: Message failed to parse"); - return 0; - } - - if (field->value_len != sizeof("subject2") - 1) { - SCLogInfo("Warning: failed to get subject"); - return 0; - } - - if (memcmp(field->value, "subject2", field->value_len) != 0) { - SCLogInfo("Warning: failed to get subject"); - return 0; - } - - - MimeDecFreeEntity(entity); - - if (expected_count != line_count) { - SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d", expected_count, - line_count); - return 0; - } - - return 1; -} - -static int MimeBase64DecodeTest01(void) -{ - int ret = 0; - uint32_t consumed_bytes = 0, num_decoded = 0; - - const char *msg = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890@" - "#$%^&*()-=_+,./;'[]<>?:"; - const char *base64msg = "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QU" - "VJTVFVWV1hZWjEyMzQ1Njc4OTBAIyQlXiYqKCktPV8rLC4vOydbXTw+Pzo="; - - uint8_t *dst = SCMalloc(strlen(msg) + 1); - if (dst == NULL) - return 0; - - ret = DecodeBase64(dst, strlen(msg) + 1, (const uint8_t *)base64msg, strlen(base64msg), - &consumed_bytes, &num_decoded, BASE64_MODE_RFC2045); - - if (memcmp(dst, msg, strlen(msg)) == 0) { - ret = 1; - } - - SCFree(dst); - - return ret; -} - -static int MimeIsExeURLTest01(void) -{ - int ret = 0; - const char *url1 = "http://www.google.com/"; - const char *url2 = "http://www.google.com/test.exe"; - - if(IsExeUrl((const uint8_t *)url1, strlen(url1)) != 0){ - SCLogDebug("Debug: URL1 error"); - goto end; - } - if(IsExeUrl((const uint8_t *)url2, strlen(url2)) != 1){ - SCLogDebug("Debug: URL2 error"); - goto end; - } - ret = 1; - - end: - - return ret; -} - -#define TEST(str, len, expect) { \ - SCLogDebug("str %s", (str)); \ - int r = IsIpv4Host((const uint8_t *)(str),(len)); \ - FAIL_IF_NOT(r == (expect)); \ -} -static int MimeIsIpv4HostTest01(void) -{ - TEST("192.168.1.1", 11, 1); - TEST("192.168.1.1.4", 13, 0); - TEST("999.168.1.1", 11, 0); - TEST("1111.168.1.1", 12, 0); - TEST("999.oogle.com", 14, 0); - TEST("0:0:0:0:0:0:0:0", 15, 0); - TEST("192.168.255.255", 15, 1); - TEST("192.168.255.255/testurl.html", 28, 1); - TEST("www.google.com", 14, 0); - PASS; -} -#undef TEST - -#define TEST(str, len, expect) { \ - SCLogDebug("str %s", (str)); \ - int r = IsIpv6Host((const uint8_t *)(str),(len)); \ - FAIL_IF_NOT(r == (expect)); \ -} -static int MimeIsIpv6HostTest01(void) -{ - TEST("0:0:0:0:0:0:0:0", 19, 1); - TEST("0000:0000:0000:0000:0000:0000:0000:0000", 39, 1); - TEST("XXXX:0000:0000:0000:0000:0000:0000:0000", 39, 0); - TEST("00001:0000:0000:0000:0000:0000:0000:0000", 40, 0); - TEST("0:0:0:0:0:0:0:0", 19, 1); - TEST("0:0:0:0:0:0:0:0:0", 20, 0); - TEST("192:168:1:1:0:0:0:0", 19, 1); - TEST("999.oogle.com", 14, 0); - TEST("192.168.255.255", 15, 0); - TEST("192.168.255.255/testurl.html", 28, 0); - TEST("www.google.com", 14, 0); - PASS; -} -#undef TEST - -static int MimeDecParseLongFilename01(void) -{ - /* contains 276 character filename -- length restricted to 255 chars */ - char mimemsg[] = "Content-Disposition: attachment; filename=\"" - "12characters12characters12characters12characters" - "12characters12characters12characters12characters" - "12characters12characters12characters12characters" - "12characters12characters12characters12characters" - "12characters12characters12characters12characters" - "12characters12characters12characters.exe\""; - - uint32_t line_count = 0; - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, - TestDataChunkCallback); - - const char *str = "From: Sender1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "To: Recipient1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Type: text/plain"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - /* Contains 276 character filename */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); - - str = ""; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "A simple message line 1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - /* Completed */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT(msg); - - FAIL_IF_NOT(msg->anomaly_flags & ANOM_LONG_FILENAME); - FAIL_IF_NOT(msg->filename_len == RS_MIME_MAX_TOKEN_LEN); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -static int MimeDecParseSmallRemInp(void) -{ - // Remainder dA - // New input: AAAA - char mimemsg[] = "TWltZSBkZWNvZGluZyB pcyBzbyBO T1QgZnV uISBJIGNhbm5vdA"; - - uint32_t line_count = 0; - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - state->stack->top->data->ctnt_flags |= CTNT_IS_ATTACHMENT; - - const char *str = "From: Sender1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "To: Recipient1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Type: text/plain"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Transfer-Encoding: base64"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = ""; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); - - str = "AAAA"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - /* Completed */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT(msg); - - /* filename is not too long */ - FAIL_IF(msg->anomaly_flags & ANOM_LONG_FILENAME); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -static int MimeDecParseRemSp(void) -{ - // Should have remainder vd A - char mimemsg[] = "TWltZSBkZWNvZGluZyBpc yBzbyBOT1QgZnVuISBJIGNhbm5vd A"; - - uint32_t line_count = 0; - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - state->stack->top->data->ctnt_flags |= CTNT_IS_ATTACHMENT; - - const char *str = "From: Sender1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "To: Recipient1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Type: text/plain"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Transfer-Encoding: base64"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = ""; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); - /* Completed */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT(msg); - - /* filename is not too long */ - FAIL_IF(msg->anomaly_flags & ANOM_LONG_FILENAME); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -static int MimeDecVerySmallInp(void) -{ - // Remainder: A - // New input: aA - char mimemsg[] = "TWltZSBkZWNvZGluZyB pcyBzbyBO T1QgZnV uISBJIGNhbm5vA"; - - uint32_t line_count = 0; - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - state->stack->top->data->ctnt_flags |= CTNT_IS_ATTACHMENT; - - const char *str = "From: Sender1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "To: Recipient1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Type: text/plain"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Transfer-Encoding: base64"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = ""; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); - - str = "aA"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - /* Completed */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT(msg); - - /* filename is not too long */ - FAIL_IF(msg->anomaly_flags & ANOM_LONG_FILENAME); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -static int MimeDecParseOddLen(void) -{ - char mimemsg[] = "TWltZSBkZWNvZGluZyB pcyBzbyBO T1QgZnV uISBJIGNhbm5vdA"; - - uint32_t line_count = 0; - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, TestDataChunkCallback); - state->stack->top->data->ctnt_flags |= CTNT_IS_ATTACHMENT; - - const char *str = "From: Sender1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "To: Recipient1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Type: text/plain"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Transfer-Encoding: base64"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = ""; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); - /* Completed */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT(msg); - - /* filename is not too long */ - FAIL_IF(msg->anomaly_flags & ANOM_LONG_FILENAME); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -static int MimeDecParseLongFilename02(void) -{ - /* contains 40 character filename and 500+ characters following filename */ - char mimemsg[] = "Content-Disposition: attachment; filename=\"" - "12characters12characters12characters.exe\"; " - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd" - "somejunkasfdasfsafasafdsasdasassdssdsd"; - - uint32_t line_count = 0; - - MimeDecGetConfig()->decode_base64 = true; - MimeDecGetConfig()->decode_quoted_printable = true; - MimeDecGetConfig()->extract_urls = true; - - /* Init parser */ - MimeDecParseState *state = MimeDecInitParser(&line_count, - TestDataChunkCallback); - - const char *str = "From: Sender1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "To: Recipient1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "Content-Type: text/plain"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - /* Contains 40 character filename */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)mimemsg, strlen(mimemsg), 1, state)); - - str = ""; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - str = "A simple message line 1"; - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseLine((uint8_t *)str, strlen(str), 1, state)); - - /* Completed */ - FAIL_IF_NOT(MIME_DEC_OK == MimeDecParseComplete(state)); - - MimeDecEntity *msg = state->msg; - FAIL_IF_NOT(msg); - - /* filename is not too long */ - FAIL_IF(msg->anomaly_flags & ANOM_LONG_FILENAME); - - MimeDecFreeEntity(msg); - - /* De Init parser */ - MimeDecDeInitParser(state); - - PASS; -} - -#endif /* UNITTESTS */ - -void MimeDecRegisterTests(void) -{ -#ifdef UNITTESTS - UtRegisterTest("MimeDecParseLineTest01", MimeDecParseLineTest01); - UtRegisterTest("MimeDecParseLineTest02", MimeDecParseLineTest02); - UtRegisterTest("MimeFindUrlStringsTest01", MimeFindUrlStringsTest01); - UtRegisterTest("MimeFindUrlStringsTest02", MimeFindUrlStringsTest02); - UtRegisterTest("MimeFindUrlStringsTest03", MimeFindUrlStringsTest03); - UtRegisterTest("MimeFindUrlStringsTest04", MimeFindUrlStringsTest04); - UtRegisterTest("MimeFindUrlStringsTest05", MimeFindUrlStringsTest05); - UtRegisterTest("MimeDecParseFullMsgTest01", MimeDecParseFullMsgTest01); - UtRegisterTest("MimeDecParseFullMsgTest02", MimeDecParseFullMsgTest02); - UtRegisterTest("MimeBase64DecodeTest01", MimeBase64DecodeTest01); - UtRegisterTest("MimeIsExeURLTest01", MimeIsExeURLTest01); - UtRegisterTest("MimeIsIpv4HostTest01", MimeIsIpv4HostTest01); - UtRegisterTest("MimeIsIpv6HostTest01", MimeIsIpv6HostTest01); - UtRegisterTest("MimeDecParseLongFilename01", MimeDecParseLongFilename01); - UtRegisterTest("MimeDecParseLongFilename02", MimeDecParseLongFilename02); - UtRegisterTest("MimeDecParseSmallRemInp", MimeDecParseSmallRemInp); - UtRegisterTest("MimeDecParseRemSp", MimeDecParseRemSp); - UtRegisterTest("MimeDecVerySmallInp", MimeDecVerySmallInp); - UtRegisterTest("MimeDecParseOddLen", MimeDecParseOddLen); -#endif /* UNITTESTS */ -} diff --git a/src/util-decode-mime.h b/src/util-decode-mime.h deleted file mode 100644 index cc79d98a6eb0..000000000000 --- a/src/util-decode-mime.h +++ /dev/null @@ -1,243 +0,0 @@ -/* Copyright (C) 2012 BAE Systems - * Copyright (C) 2021 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author David Abarbanel - * - */ - -#ifndef MIME_DECODE_H_ -#define MIME_DECODE_H_ - -#include "conf.h" -#include "util-base64.h" -#include "util-file.h" - -/* Content Flags */ -#define CTNT_IS_MSG 1 -#define CTNT_IS_ENV 2 -#define CTNT_IS_ENCAP 4 -#define CTNT_IS_BODYPART 8 -#define CTNT_IS_MULTIPART 16 -#define CTNT_IS_ATTACHMENT 32 -#define CTNT_IS_BASE64 64 -#define CTNT_IS_QP 128 -#define CTNT_IS_TEXT 256 -#define CTNT_IS_HTML 512 - -/* URL Flags */ -#define URL_IS_IP4 1 -#define URL_IS_IP6 2 -#define URL_IS_EXE 4 - -/* Anomaly Flags */ -#define ANOM_INVALID_BASE64 1 /* invalid base64 chars */ -#define ANOM_INVALID_QP 2 /* invalid quoted-printable chars */ -#define ANOM_LONG_HEADER_NAME 4 /* header is abnormally long */ -#define ANOM_LONG_HEADER_VALUE 8 /* header value is abnormally long - * (includes multi-line) */ -#define ANOM_LONG_LINE 16 /* Lines that exceed 998 octets */ -#define ANOM_LONG_ENC_LINE 32 /* Lines that exceed 76 octets */ -#define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */ -#define ANOM_LONG_BOUNDARY 128 /* Boundary too long */ -#define ANOM_LONG_FILENAME 256 /* filename truncated */ - -/* Publicly exposed size constants */ -#define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */ - -/* Mime Parser Constants */ -#define HEADER_READY 0x01 -#define HEADER_STARTED 0x02 -#define HEADER_DONE 0x03 -#define BODY_STARTED 0x04 -#define BODY_DONE 0x05 -#define BODY_END_BOUND 0x06 -#define PARSE_DONE 0x07 -#define PARSE_ERROR 0x08 - -/** - * \brief Mime Decoder Error Codes - */ -typedef enum MimeDecRetCode { - MIME_DEC_OK = 0, - MIME_DEC_MORE = 1, - MIME_DEC_ERR_DATA = -1, - MIME_DEC_ERR_MEM = -2, - MIME_DEC_ERR_PARSE = -3, - MIME_DEC_ERR_STATE = -4, /**< parser in error state */ - MIME_DEC_ERR_OVERFLOW = -5, -} MimeDecRetCode; - -/** - * \brief Structure for containing configuration options - * - */ -typedef struct MimeDecConfig { - bool decode_base64; /**< Decode base64 bodies */ - bool decode_quoted_printable; /**< Decode quoted-printable bodies */ - bool extract_urls; /**< Extract and store URLs in data structure */ - ConfNode *extract_urls_schemes; /**< List of schemes of which to - extract urls */ - bool log_url_scheme; /**< Log the scheme of extracted URLs */ - bool body_md5; /**< Compute md5 sum of body */ - uint32_t header_value_depth; /**< Depth of which to store header values - (Default is 2000) */ -} MimeDecConfig; - -/** - * \brief This represents a header field name and associated value - */ -typedef struct MimeDecField { - uint8_t *name; /**< Name of the header field */ - uint32_t name_len; /**< Length of the name */ - uint32_t value_len; /**< Length of the value */ - uint8_t *value; /**< Value of the header field */ - struct MimeDecField *next; /**< Pointer to next field */ -} MimeDecField; - -/** - * \brief This represents a URL value node in a linked list - * - * Since HTML can sometimes contain a high number of URLs, this - * structure only features the URL host name/IP or those that are - * pointing to an executable file (see url_flags to determine which). - */ -typedef struct MimeDecUrl { - uint8_t *url; /**< String representation of full or partial URL (lowercase) */ - uint32_t url_len; /**< Length of the URL string */ - uint32_t url_flags; /**< Flags indicating type of URL */ - struct MimeDecUrl *next; /**< Pointer to next URL */ -} MimeDecUrl; - -/** - * \brief This represents the MIME Entity (or also top level message) in a - * child-sibling tree - */ -typedef struct MimeDecEntity { - MimeDecField *field_list; /**< Pointer to list of header fields */ - MimeDecUrl *url_list; /**< Pointer to list of URLs */ - uint32_t header_flags; /**< Flags indicating header characteristics */ - uint32_t ctnt_flags; /**< Flags indicating type of content */ - uint32_t anomaly_flags; /**< Flags indicating an anomaly in the message */ - uint32_t filename_len; /**< Length of file attachment name */ - uint8_t *filename; /**< Name of file attachment */ - uint8_t *ctnt_type; /**< Quick access pointer to short-hand content type field */ - uint32_t ctnt_type_len; /**< Length of content type field value */ - uint32_t msg_id_len; /**< Quick access pointer to message Id */ - uint8_t *msg_id; /**< Quick access pointer to message Id */ - struct MimeDecEntity *next; /**< Pointer to list of sibling entities */ - struct MimeDecEntity *child; /**< Pointer to list of child entities */ - struct MimeDecEntity *last_child; /**< Pointer to tail of the list of child entities */ -} MimeDecEntity; - -/** - * \brief Structure contains boundary and entity for the current node (entity) - * in the stack - * - */ -typedef struct MimeDecStackNode { - MimeDecEntity *data; /**< Pointer to the entity data structure */ - uint8_t *bdef; /**< Copy of boundary definition for child entity */ - uint16_t bdef_len; /**< Boundary length for child entity */ - bool is_encap; /**< Flag indicating entity is encapsulated in message */ - struct MimeDecStackNode *next; /**< Pointer to next item on the stack */ -} MimeDecStackNode; - -/** - * \brief Structure holds the top of the stack along with some free reusable nodes - * - */ -typedef struct MimeDecStack { - MimeDecStackNode *top; /**< Pointer to the top of the stack */ - MimeDecStackNode *free_nodes; /**< Pointer to the list of free nodes */ - uint32_t free_nodes_cnt; /**< Count of free nodes in the list */ -} MimeDecStack; - -/** - * \brief Structure contains a list of value and lengths for robust data processing - * - */ -typedef struct DataValue { - uint8_t *value; /**< Copy of data value */ - uint32_t value_len; /**< Length of data value */ - struct DataValue *next; /**< Pointer to next value in the list */ -} DataValue; - -/** - * \brief Structure contains the current state of the MIME parser - * - */ -typedef struct MimeDecParseState { - MimeDecEntity *msg; /**< Pointer to the top-level message entity */ - MimeDecStack *stack; /**< Pointer to the top of the entity stack */ - uint8_t *hname; /**< Copy of the last known header name */ - uint32_t hlen; /**< Length of the last known header name */ - uint32_t hvlen; /**< Total length of value list */ - DataValue *hvalue; /**< Pointer to the incomplete header value list */ - uint8_t bvremain[B64_BLOCK]; /**< Remainder from base64-decoded line */ - uint8_t bvr_len; /**< Length of remainder from base64-decoded line */ - uint8_t data_chunk[DATA_CHUNK_SIZE]; /**< Buffer holding data chunk */ - SCMd5 *md5_ctx; - uint8_t md5[SC_MD5_LEN]; - bool has_md5; - uint8_t state_flag; /**< Flag representing current state of parser */ - uint32_t data_chunk_len; /**< Length of data chunk */ - int found_child; /**< Flag indicating a child entity was found */ - int body_begin; /**< Currently at beginning of body */ - int body_end; /**< Currently at end of body */ - uint8_t current_line_delimiter_len; /**< Length of line delimiter */ - void *data; /**< Pointer to data specific to the caller */ - int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len, - struct MimeDecParseState *state); /**< Data chunk processing function callback */ -} MimeDecParseState; - -/* Config functions */ -void MimeDecSetConfig(MimeDecConfig *config); -MimeDecConfig * MimeDecGetConfig(void); - -/* Memory functions */ -void MimeDecFreeEntity(MimeDecEntity *entity); -void MimeDecFreeField(MimeDecField *field); -void MimeDecFreeUrl(MimeDecUrl *url); - -/* List functions */ -MimeDecField * MimeDecAddField(MimeDecEntity *entity); -MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name); -int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data); -MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent); - -/* Helper functions */ -//MimeDecField * MimeDecFillField(MimeDecEntity *entity, const char *name, -// uint32_t nlen, const char *value, uint32_t vlen, int copy_name_value); - -/* Parser functions */ -MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t *chunk, - uint32_t len, MimeDecParseState *state)); -void MimeDecDeInitParser(MimeDecParseState *state); -int MimeDecParseComplete(MimeDecParseState *state); -int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state); -MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, - int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state)); -const char *MimeDecParseStateGetStatus(MimeDecParseState *state); - -/* Test functions */ -void MimeDecRegisterTests(void); - -#endif diff --git a/src/util-lua-smtp.c b/src/util-lua-smtp.c index e239080502a2..f64bb88c5afe 100644 --- a/src/util-lua-smtp.c +++ b/src/util-lua-smtp.c @@ -68,24 +68,23 @@ static int GetMimeDecField(lua_State *luastate, Flow *flow, const char *name) if(smtp_tx == NULL) { return LuaCallbackError(luastate, "Transaction ending or not found"); } - /* pointer to tail of msg list of MimeDecEntities in current transaction. */ - MimeDecEntity *mime = smtp_tx->msg_tail; + /* pointer to tail of msg list of MimeStateSMTP in current transaction. */ + MimeStateSMTP *mime = smtp_tx->mime_state; /* check if msg_tail was hit */ if(mime == NULL){ return LuaCallbackError(luastate, "Internal error: no fields in transaction"); } /* extract MIME field based on specific field name. */ - MimeDecField *field = MimeDecFindField(mime, name); + const uint8_t *field_value; + uint32_t field_len; /* check MIME field */ - if(field == NULL) { + if (!SCMimeSmtpGetHeader(mime, name, &field_value, &field_len)) { return LuaCallbackError(luastate, "Error: mimefield not found"); } - /* return extracted field. */ - if(field->value == NULL || field->value_len == 0){ + if (field_len == 0) { return LuaCallbackError(luastate, "Error, pointer error"); } - - return LuaPushStringBuffer(luastate, field->value, field->value_len); + return LuaPushStringBuffer(luastate, field_value, field_len); } /** @@ -139,29 +138,23 @@ static int GetMimeList(lua_State *luastate, Flow *flow) if(smtp_tx == NULL) { return LuaCallbackError(luastate, "Error: no SMTP transaction found"); } - /* Create a pointer to the tail of MimeDecEntity list */ - MimeDecEntity *mime = smtp_tx->msg_tail; + /* Create a pointer to the tail of MimeStateSMTP list */ + MimeStateSMTP *mime = smtp_tx->mime_state; if(mime == NULL) { return LuaCallbackError(luastate, "Error: no mime entity found"); } - MimeDecField *field = mime->field_list; - if(field == NULL) { - return LuaCallbackError(luastate, "Error: no field_list found"); - } - if(field->name == NULL || field->name_len == 0) { - return LuaCallbackError(luastate, "Error: field has no name"); - } + const uint8_t *field_name; + uint32_t field_len; /* Counter of MIME fields found */ int num = 1; /* loop trough the list of mimeFields, printing each name found */ lua_newtable(luastate); - while (field != NULL) { - if(field->name != NULL && field->name_len != 0) { + while (SCMimeSmtpGetHeaderName(mime, &field_name, &field_len, (uint32_t)num)) { + if (field_len != 0) { lua_pushinteger(luastate,num++); - LuaPushStringBuffer(luastate, field->name, field->name_len); + LuaPushStringBuffer(luastate, field_name, field_len); lua_settable(luastate,-3); } - field = field->next; } return 1; } diff --git a/src/util-spm-bs.c b/src/util-spm-bs.c index 08eacca70cf5..d78d5c7a1438 100644 --- a/src/util-spm-bs.c +++ b/src/util-spm-bs.c @@ -130,3 +130,13 @@ uint8_t *BasicSearchNocase(const uint8_t *haystack, uint32_t haystack_len, const return NULL; } + +uint32_t BasicSearchNocaseIndex( + const uint8_t *haystack, uint32_t haystack_len, const uint8_t *needle, uint16_t needle_len) +{ + uint8_t *r = BasicSearchNocase(haystack, haystack_len, needle, needle_len); + if (r == NULL) { + return haystack_len; + } + return (uint32_t)(r - haystack); +} diff --git a/src/util-spm-bs.h b/src/util-spm-bs.h index 81151c45abb4..a194013292eb 100644 --- a/src/util-spm-bs.h +++ b/src/util-spm-bs.h @@ -29,5 +29,6 @@ uint8_t *BasicSearch(const uint8_t *, uint32_t, const uint8_t *, uint16_t); uint8_t *BasicSearchNocase(const uint8_t *, uint32_t, const uint8_t *, uint16_t); +uint32_t BasicSearchNocaseIndex(const uint8_t *, uint32_t, const uint8_t *, uint16_t); #endif /* SURICATA_UTIL_SPM_BS */