Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed comments lexing #256

Merged
merged 1 commit into from
Feb 24, 2020
Merged

Fixed comments lexing #256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions boa/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,13 @@ pub fn forward(engine: &mut Interpreter, src: &str) -> String {
/// If the interpreter fails parsing an error value is returned instead (error object)
pub fn forward_val(engine: &mut Interpreter, src: &str) -> ResultValue {
// Setup executor
let expr = parser_expr(src).unwrap();
engine.run(&expr)
match parser_expr(src) {
Ok(expr) => engine.run(&expr),
Err(e) => {
eprintln!("{}", e);
std::process::exit(1);
}
}
}

/// Create a clean Interpreter and execute the code
Expand Down
80 changes: 62 additions & 18 deletions boa/src/syntax/lexer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,28 +536,38 @@ impl<'a> Lexer<'a> {
match ch {
// line comment
'/' => {
let comment = self.read_line()?;
let comment = "/".to_owned() + &self.read_line()?;
self.push_token(TokenData::Comment(comment));
self.line_number += 1;
self.column_number = 0;
}
// block comment
'*' => {
let mut buf = String::new();
let mut buf = "/".to_owned();
let mut lines = 0;
loop {
if self.preview_next().is_none() {
return Err(LexerError::new("Unterminated Multiline Comment"));
}
match self.next() {
'*' => {
buf.push('*');
if self.next_is('/') {
buf.push('/');
break;
} else {
buf.push('*')
}
}
next_ch => buf.push(next_ch),
next_ch => {
if next_ch == '\n' {
lines += 1;
}
buf.push(next_ch)
},
}
}
self.push_token(TokenData::Comment(buf));
self.line_number += lines;
self.column_number = 0;
}
// division, assigndiv or regex literal
_ => {
Expand Down Expand Up @@ -669,7 +679,7 @@ impl<'a> Lexer<'a> {
// of characters as whitespaces:
// * Rust uses \p{White_Space},
// * ecma standard uses \{Space_Separator} + \u{0009}, \u{000B}, \u{000C}, \u{FEFF}
//
//
// Explicit whitespace: see https://tc39.es/ecma262/#table-32
'\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' |
// Unicode Space_Seperator category (minus \u{0020} and \u{00A0} which are allready stated above)
Expand All @@ -695,7 +705,7 @@ mod tests {
let mut lexer = Lexer::new(s1);
lexer.lex().expect("failed to lex");
assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Var));

assert_eq!(lexer.tokens[1].data, TokenData::Comment("//=".to_owned()));
assert_eq!(lexer.tokens[2].data, TokenData::Identifier("x".to_string()));
}

Expand All @@ -705,7 +715,10 @@ mod tests {
let mut lexer = Lexer::new(s);
lexer.lex().expect("failed to lex");
assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Var));

assert_eq!(
lexer.tokens[1].data,
TokenData::Comment("/* await \n break \n*/".to_owned())
);
assert_eq!(lexer.tokens[2].data, TokenData::Identifier("x".to_string()));
}

Expand Down Expand Up @@ -972,7 +985,7 @@ mod tests {

#[test]
fn check_variable_definition_tokens() {
let s = &String::from("let a = 'hello';");
let s = "let a = 'hello';";
let mut lexer = Lexer::new(s);
lexer.lex().expect("failed to lex");
assert_eq!(lexer.tokens[0].data, TokenData::Keyword(Keyword::Let));
Expand All @@ -989,38 +1002,69 @@ mod tests {

#[test]
fn check_positions() {
let s = &String::from("console.log(\"hello world\");");
// -------------------123456789
let s = "console.log(\"hello world\"); // Test";
// ------123456789
let mut lexer = Lexer::new(s);
lexer.lex().expect("failed to lex");
// The first column is 1 (not zero indexed)
assert_eq!(lexer.tokens[0].pos.column_number, 1);
assert_eq!(lexer.tokens[0].pos.line_number, 1);
// Dot Token starts on line 7
// Dot Token starts on column 8
assert_eq!(lexer.tokens[1].pos.column_number, 8);
assert_eq!(lexer.tokens[1].pos.line_number, 1);
// Log Token starts on line 7
// Log Token starts on column 9
assert_eq!(lexer.tokens[2].pos.column_number, 9);
assert_eq!(lexer.tokens[2].pos.line_number, 1);
// Open parenthesis token starts on line 12
// Open parenthesis token starts on column 12
assert_eq!(lexer.tokens[3].pos.column_number, 12);
assert_eq!(lexer.tokens[3].pos.line_number, 1);
// String token starts on line 13
// String token starts on column 13
assert_eq!(lexer.tokens[4].pos.column_number, 13);
assert_eq!(lexer.tokens[4].pos.line_number, 1);
// Close parenthesis token starts on line 26
// Close parenthesis token starts on column 26
assert_eq!(lexer.tokens[5].pos.column_number, 26);
assert_eq!(lexer.tokens[5].pos.line_number, 1);
// Semi Colon token starts on line 27
// Semi Colon token starts on column 27
assert_eq!(lexer.tokens[6].pos.column_number, 27);
assert_eq!(lexer.tokens[6].pos.line_number, 1);
// Comment start on column 29
// Semi Colon token starts on column 27
assert_eq!(lexer.tokens[7].pos.column_number, 29);
assert_eq!(lexer.tokens[7].pos.line_number, 1);
}

#[test]
fn check_line_numbers() {
let s = "// Copyright (C) 2017 Ecma International. All rights reserved.\n\
// This code is governed by the BSD license found in the LICENSE file.\n\
/*---\n\
description: |\n \
Collection of assertion functions used throughout test262\n\
defines: [assert]\n\
---*/\n\n\n\
function assert(mustBeTrue, message) {";

let mut lexer = Lexer::new(s);
lexer.lex().expect("failed to lex");
// The first column is 1 (not zero indexed), first line is also 1
assert_eq!(lexer.tokens[0].pos.column_number, 1);
assert_eq!(lexer.tokens[0].pos.line_number, 1);
// Second comment starts on line 2
assert_eq!(lexer.tokens[1].pos.column_number, 1);
assert_eq!(lexer.tokens[1].pos.line_number, 2);
// Multiline comment starts on line 3
assert_eq!(lexer.tokens[2].pos.column_number, 1);
assert_eq!(lexer.tokens[2].pos.line_number, 3);
// Function Token is on line 10
assert_eq!(lexer.tokens[3].pos.column_number, 1);
assert_eq!(lexer.tokens[3].pos.line_number, 10);
}

// Increment/Decrement
#[test]
fn check_decrement_advances_lexer_2_places() {
// Here we want an example of decrementing an integer
let s = &String::from("let a = b--;");
let s = "let a = b--;";
let mut lexer = Lexer::new(s);
lexer.lex().expect("failed to lex");
assert_eq!(lexer.tokens[4].data, TokenData::Punctuator(Punctuator::Dec));
Expand Down
6 changes: 4 additions & 2 deletions boa/src/syntax/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,15 @@ impl fmt::Display for ParseError {
match self {
ParseError::Expected(expected, actual, routine) => write!(
f,
"Expected token '{}', got '{}' in routine '{}'",
"Expected token '{}', got '{}' in routine '{}' at line {}, col {}",
expected
.first()
.map(|t| t.to_string())
.unwrap_or_else(String::new),
actual,
routine
routine,
actual.pos.line_number,
actual.pos.column_number
),
ParseError::ExpectedExpr(expected, actual) => {
write!(f, "Expected expression '{}', got '{}'", expected, actual)
Expand Down
14 changes: 8 additions & 6 deletions boa_cli/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use structopt::StructOpt;
struct Opt {
/// The javascript file to be evaluated.
#[structopt(name = "FILE", parse(from_os_str), default_value = "tests/js/test.js")]
file: PathBuf,
files: Vec<PathBuf>,
/// Open a boa shell (WIP).
#[structopt(short, long)]
shell: bool,
Expand All @@ -21,14 +21,16 @@ struct Opt {
pub fn main() -> Result<(), std::io::Error> {
let args = Opt::from_args();

let buffer = read_to_string(args.file)?;

let realm = Realm::create();
let mut engine = Executor::new(realm);

match forward_val(&mut engine, &buffer) {
Ok(v) => print!("{}", v.to_string()),
Err(v) => eprint!("{}", v.to_string()),
for file in args.files {
let buffer = read_to_string(file)?;

match forward_val(&mut engine, &buffer) {
Ok(v) => print!("{}", v.to_string()),
Err(v) => eprint!("{}", v.to_string()),
}
}

Ok(())
Expand Down