Skip to content

Commit

Permalink
Fix lexer span panic with carriage return (#799)
Browse files Browse the repository at this point in the history
Switch to treating bare carriage return as an OS9-style
newline, like v8.
  • Loading branch information
vgel authored Oct 6, 2020
1 parent fb1b8d5 commit 7a52e26
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 8 deletions.
16 changes: 8 additions & 8 deletions boa/src/syntax/lexer/cursor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,6 @@ impl<R> Cursor<R> {
self.pos = Position::new(next_line, 1);
}

/// Performs a carriage return to modify the position in the source.
#[inline]
fn carriage_return(&mut self) {
let current_line = self.pos.line_number();
self.pos = Position::new(current_line, 1);
}

#[inline]
pub(super) fn strict_mode(&self) -> bool {
self.strict_mode
Expand Down Expand Up @@ -177,7 +170,14 @@ where
};

match chr {
Some('\r') => self.carriage_return(),
Some('\r') => {
// Try to take a newline if it's next, for windows "\r\n" newlines
// Otherwise, treat as a Mac OS9 bare '\r' newline
if self.peek()? == Some('\n') {
self.peeked.take();
}
self.next_line();
}
Some('\n') | Some('\u{2028}') | Some('\u{2029}') => self.next_line(),
Some(_) => self.next_column(),
None => {}
Expand Down
45 changes: 45 additions & 0 deletions boa/src/syntax/lexer/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -657,3 +657,48 @@ fn non_english_str() {

expect_tokens(&mut lexer, &expected);
}

mod carriage_return {
use super::*;

fn expect_tokens_with_lines(lines: usize, src: &str) {
let mut lexer = Lexer::new(src.as_bytes());

let mut expected = Vec::with_capacity(lines + 2);
expected.push(TokenKind::Punctuator(Punctuator::Sub));
for _ in 0..lines {
expected.push(TokenKind::LineTerminator);
}
expected.push(TokenKind::NumericLiteral(Numeric::Integer(3)));

expect_tokens(&mut lexer, &expected);
}

#[test]
fn regular_line() {
expect_tokens_with_lines(1, "-\n3");
expect_tokens_with_lines(2, "-\n\n3");
expect_tokens_with_lines(3, "-\n\n\n3");
}

#[test]
fn carriage_return() {
expect_tokens_with_lines(1, "-\r3");
expect_tokens_with_lines(2, "-\r\r3");
expect_tokens_with_lines(3, "-\r\r\r3");
}

#[test]
fn windows_line() {
expect_tokens_with_lines(1, "-\r\n3");
expect_tokens_with_lines(2, "-\r\n\r\n3");
expect_tokens_with_lines(3, "-\r\n\r\n\r\n3");
}

#[test]
fn mixed_line() {
expect_tokens_with_lines(2, "-\r\n\n3");
expect_tokens_with_lines(2, "-\n\r3");
expect_tokens_with_lines(3, "-\r\n\n\r3");
}
}

0 comments on commit 7a52e26

Please sign in to comment.