Allow unicode escaped characters in identifiers that are keywords (#2021

) This Pull Request changes the following: - Remove syntax error for unicode escaped characters in keywords from the lexer. - Adjust the lexer tokens for keywords to indicate if they contain unicode escaped characters. - Throw syntax errors in parser, when keywords cannot contain unicode escaped characters.
boa-dev · Apr 9, 2022 · 2a63de3 · 2a63de3
1 parent 2e0254b
commit 2a63de3
Show file tree

Hide file tree

Showing 41 changed files with 403 additions and 267 deletions.
diff --git a/boa_engine/src/syntax/lexer/identifier.rs b/boa_engine/src/syntax/lexer/identifier.rs
@@ -90,13 +90,6 @@ impl<R> Tokenizer<R> for Identifier {
             Self::take_identifier_name(cursor, start_pos, self.init)?;
 
         let token_kind = if let Ok(keyword) = identifier_name.parse() {
-            if contains_escaped_chars {
-                return Err(Error::Syntax(
-                    "unicode escaped characters are not allowed in keyword".into(),
-                    start_pos,
-                ));
-            }
-
             if cursor.strict_mode() && keyword == Keyword::With {
                 return Err(Error::Syntax(
                     "using 'with' statement not allowed in strict mode".into(),
@@ -108,7 +101,7 @@ impl<R> Tokenizer<R> for Identifier {
                 Keyword::True => TokenKind::BooleanLiteral(true),
                 Keyword::False => TokenKind::BooleanLiteral(false),
                 Keyword::Null => TokenKind::NullLiteral,
-                _ => TokenKind::Keyword(keyword),
+                _ => TokenKind::Keyword((keyword, contains_escaped_chars)),
             }
         } else {
             if cursor.strict_mode()

diff --git a/boa_engine/src/syntax/lexer/tests.rs b/boa_engine/src/syntax/lexer/tests.rs
@@ -36,7 +36,7 @@ fn check_single_line_comment() {
     let mut interner = Interner::default();
 
     let expected = [
-        TokenKind::Keyword(Keyword::Var),
+        TokenKind::Keyword((Keyword::Var, false)),
         TokenKind::LineTerminator,
         TokenKind::LineTerminator,
         TokenKind::BooleanLiteral(true),
@@ -52,7 +52,7 @@ fn check_single_line_comment_with_crlf_ending() {
     let mut interner = Interner::default();
 
     let expected = [
-        TokenKind::Keyword(Keyword::Var),
+        TokenKind::Keyword((Keyword::Var, false)),
         TokenKind::LineTerminator,
         TokenKind::LineTerminator,
         TokenKind::BooleanLiteral(true),
@@ -69,7 +69,7 @@ fn check_multi_line_comment() {
 
     let sym = interner.get_or_intern_static("x");
     let expected = [
-        TokenKind::Keyword(Keyword::Var),
+        TokenKind::Keyword((Keyword::Var, false)),
         TokenKind::LineTerminator,
         TokenKind::identifier(sym),
     ];
@@ -251,40 +251,40 @@ fn check_keywords() {
     let mut interner = Interner::default();
 
     let expected = [
-        TokenKind::Keyword(Keyword::Await),
-        TokenKind::Keyword(Keyword::Break),
-        TokenKind::Keyword(Keyword::Case),
-        TokenKind::Keyword(Keyword::Catch),
-        TokenKind::Keyword(Keyword::Class),
-        TokenKind::Keyword(Keyword::Const),
-        TokenKind::Keyword(Keyword::Continue),
-        TokenKind::Keyword(Keyword::Debugger),
-        TokenKind::Keyword(Keyword::Default),
-        TokenKind::Keyword(Keyword::Delete),
-        TokenKind::Keyword(Keyword::Do),
-        TokenKind::Keyword(Keyword::Else),
-        TokenKind::Keyword(Keyword::Export),
-        TokenKind::Keyword(Keyword::Extends),
-        TokenKind::Keyword(Keyword::Finally),
-        TokenKind::Keyword(Keyword::For),
-        TokenKind::Keyword(Keyword::Function),
-        TokenKind::Keyword(Keyword::If),
-        TokenKind::Keyword(Keyword::Import),
-        TokenKind::Keyword(Keyword::In),
-        TokenKind::Keyword(Keyword::InstanceOf),
-        TokenKind::Keyword(Keyword::New),
-        TokenKind::Keyword(Keyword::Return),
-        TokenKind::Keyword(Keyword::Super),
-        TokenKind::Keyword(Keyword::Switch),
-        TokenKind::Keyword(Keyword::This),
-        TokenKind::Keyword(Keyword::Throw),
-        TokenKind::Keyword(Keyword::Try),
-        TokenKind::Keyword(Keyword::TypeOf),
-        TokenKind::Keyword(Keyword::Var),
-        TokenKind::Keyword(Keyword::Void),
-        TokenKind::Keyword(Keyword::While),
-        TokenKind::Keyword(Keyword::With),
-        TokenKind::Keyword(Keyword::Yield),
+        TokenKind::Keyword((Keyword::Await, false)),
+        TokenKind::Keyword((Keyword::Break, false)),
+        TokenKind::Keyword((Keyword::Case, false)),
+        TokenKind::Keyword((Keyword::Catch, false)),
+        TokenKind::Keyword((Keyword::Class, false)),
+        TokenKind::Keyword((Keyword::Const, false)),
+        TokenKind::Keyword((Keyword::Continue, false)),
+        TokenKind::Keyword((Keyword::Debugger, false)),
+        TokenKind::Keyword((Keyword::Default, false)),
+        TokenKind::Keyword((Keyword::Delete, false)),
+        TokenKind::Keyword((Keyword::Do, false)),
+        TokenKind::Keyword((Keyword::Else, false)),
+        TokenKind::Keyword((Keyword::Export, false)),
+        TokenKind::Keyword((Keyword::Extends, false)),
+        TokenKind::Keyword((Keyword::Finally, false)),
+        TokenKind::Keyword((Keyword::For, false)),
+        TokenKind::Keyword((Keyword::Function, false)),
+        TokenKind::Keyword((Keyword::If, false)),
+        TokenKind::Keyword((Keyword::Import, false)),
+        TokenKind::Keyword((Keyword::In, false)),
+        TokenKind::Keyword((Keyword::InstanceOf, false)),
+        TokenKind::Keyword((Keyword::New, false)),
+        TokenKind::Keyword((Keyword::Return, false)),
+        TokenKind::Keyword((Keyword::Super, false)),
+        TokenKind::Keyword((Keyword::Switch, false)),
+        TokenKind::Keyword((Keyword::This, false)),
+        TokenKind::Keyword((Keyword::Throw, false)),
+        TokenKind::Keyword((Keyword::Try, false)),
+        TokenKind::Keyword((Keyword::TypeOf, false)),
+        TokenKind::Keyword((Keyword::Var, false)),
+        TokenKind::Keyword((Keyword::Void, false)),
+        TokenKind::Keyword((Keyword::While, false)),
+        TokenKind::Keyword((Keyword::With, false)),
+        TokenKind::Keyword((Keyword::Yield, false)),
     ];
 
     expect_tokens(&mut lexer, &expected, &mut interner);
@@ -299,7 +299,7 @@ fn check_variable_definition_tokens() {
     let a_sym = interner.get_or_intern_static("a");
     let hello_sym = interner.get_or_intern_static("hello");
     let expected = [
-        TokenKind::Keyword(Keyword::Let),
+        TokenKind::Keyword((Keyword::Let, false)),
         TokenKind::identifier(a_sym),
         TokenKind::Punctuator(Punctuator::Assign),
         TokenKind::string_literal(hello_sym),

diff --git a/boa_engine/src/syntax/lexer/token.rs b/boa_engine/src/syntax/lexer/token.rs
@@ -105,8 +105,8 @@ pub enum TokenKind {
     /// A private identifier.
     PrivateIdentifier(Sym),
 
-    /// A keyword.
-    Keyword(Keyword),
+    /// A keyword and a flag if the keyword contains unicode escaped chars.
+    Keyword((Keyword, bool)),
 
     /// A `null` literal.
     NullLiteral,
@@ -142,8 +142,8 @@ impl From<bool> for TokenKind {
     }
 }
 
-impl From<Keyword> for TokenKind {
-    fn from(kw: Keyword) -> Self {
+impl From<(Keyword, bool)> for TokenKind {
+    fn from(kw: (Keyword, bool)) -> Self {
         Self::Keyword(kw)
     }
 }
@@ -176,11 +176,6 @@ impl TokenKind {
         Self::Identifier(ident)
     }
 
-    /// Creates a `Keyword` token kind.
-    pub fn keyword(keyword: Keyword) -> Self {
-        Self::Keyword(keyword)
-    }
-
     /// Creates a `NumericLiteral` token kind.
     pub fn numeric_literal<L>(lit: L) -> Self
     where
@@ -229,7 +224,7 @@ impl TokenKind {
             Self::EOF => "end of file".to_owned(),
             Self::Identifier(ident) => interner.resolve_expect(ident).to_owned(),
             Self::PrivateIdentifier(ident) => format!("#{}", interner.resolve_expect(ident)),
-            Self::Keyword(word) => word.to_string(),
+            Self::Keyword((word, _)) => word.to_string(),
             Self::NullLiteral => "null".to_owned(),
             Self::NumericLiteral(Numeric::Rational(num)) => num.to_string(),
             Self::NumericLiteral(Numeric::Integer(num)) => num.to_string(),

diff --git a/boa_engine/src/syntax/parser/expression/assignment/exponentiation.rs b/boa_engine/src/syntax/parser/expression/assignment/exponentiation.rs
@@ -70,7 +70,7 @@ where
     Ok(if let Some(tok) = cursor.peek(0, interner)? {
         matches!(
             tok.kind(),
-            TokenKind::Keyword(Keyword::Delete | Keyword::Void | Keyword::TypeOf)
+            TokenKind::Keyword((Keyword::Delete | Keyword::Void | Keyword::TypeOf, _))
                 | TokenKind::Punctuator(
                     Punctuator::Add | Punctuator::Sub | Punctuator::Not | Punctuator::Neg
                 )

diff --git a/boa_engine/src/syntax/parser/expression/assignment/mod.rs b/boa_engine/src/syntax/parser/expression/assignment/mod.rs
@@ -97,12 +97,12 @@ where
             .kind()
         {
             // [+Yield]YieldExpression[?In, ?Await]
-            TokenKind::Keyword(Keyword::Yield) if self.allow_yield.0 => {
+            TokenKind::Keyword((Keyword::Yield, _)) if self.allow_yield.0 => {
                 return YieldExpression::new(self.allow_in, self.allow_await)
                     .parse(cursor, interner)
             }
             // ArrowFunction[?In, ?Yield, ?Await] -> ArrowParameters[?Yield, ?Await] -> BindingIdentifier[?Yield, ?Await]
-            TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => {
+            TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => {
                 if let Ok(tok) =
                     cursor.peek_expect_no_lineterminator(1, "assignment expression", interner)
                 {

diff --git a/boa_engine/src/syntax/parser/expression/assignment/yield.rs b/boa_engine/src/syntax/parser/expression/assignment/yield.rs
@@ -58,7 +58,7 @@ where
         let _timer = Profiler::global().start_event("YieldExpression", "Parsing");
 
         cursor.expect(
-            TokenKind::Keyword(Keyword::Yield),
+            TokenKind::Keyword((Keyword::Yield, false)),
             "yield expression",
             interner,
         )?;
@@ -87,7 +87,7 @@ where
                 | Punctuator::OpenBlock
                 | Punctuator::Div,
             )
-            | TokenKind::Keyword(
+            | TokenKind::Keyword((
                 Keyword::Yield
                 | Keyword::Await
                 | Keyword::Delete
@@ -98,7 +98,8 @@ where
                 | Keyword::Function
                 | Keyword::Class
                 | Keyword::Async,
-            )
+                _,
+            ))
             | TokenKind::BooleanLiteral(_)
             | TokenKind::NullLiteral
             | TokenKind::StringLiteral(_)

diff --git a/boa_engine/src/syntax/parser/expression/await_expr.rs b/boa_engine/src/syntax/parser/expression/await_expr.rs
@@ -53,7 +53,7 @@ where
         interner: &mut Interner,
     ) -> Result<Self::Output, ParseError> {
         cursor.expect(
-            TokenKind::Keyword(Keyword::Await),
+            TokenKind::Keyword((Keyword::Await, false)),
             "Await expression parsing",
             interner,
         )?;

diff --git a/boa_engine/src/syntax/parser/expression/left_hand_side/call.rs b/boa_engine/src/syntax/parser/expression/left_hand_side/call.rs
@@ -94,7 +94,7 @@ where
                         TokenKind::Identifier(name) => {
                             lhs = GetConstField::new(lhs, *name).into();
                         }
-                        TokenKind::Keyword(kw) => {
+                        TokenKind::Keyword((kw, _)) => {
                             lhs = GetConstField::new(lhs, kw.to_sym(interner)).into();
                         }
                         _ => {

diff --git a/boa_engine/src/syntax/parser/expression/left_hand_side/member.rs b/boa_engine/src/syntax/parser/expression/left_hand_side/member.rs
@@ -64,27 +64,32 @@ where
     fn parse(self, cursor: &mut Cursor<R>, interner: &mut Interner) -> ParseResult {
         let _timer = Profiler::global().start_event("MemberExpression", "Parsing");
 
-        let mut lhs = if cursor
-            .peek(0, interner)?
-            .ok_or(ParseError::AbruptEnd)?
-            .kind()
-            == &TokenKind::Keyword(Keyword::New)
-        {
-            let _next = cursor.next(interner).expect("new keyword disappeared");
-            let lhs = self.parse(cursor, interner)?;
-            let args = match cursor.peek(0, interner)? {
-                Some(next) if next.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => {
-                    Arguments::new(self.allow_yield, self.allow_await).parse(cursor, interner)?
-                }
-                _ => Box::new([]),
-            };
-            let call_node = Call::new(lhs, args);
+        let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?;
+        let mut lhs = match token.kind() {
+            TokenKind::Keyword((Keyword::New, true)) => {
+                return Err(ParseError::general(
+                    "keyword must not contain escaped characters",
+                    token.span().start(),
+                ));
+            }
+            TokenKind::Keyword((Keyword::New, false)) => {
+                let _next = cursor.next(interner).expect("new keyword disappeared");
+                let lhs = self.parse(cursor, interner)?;
+                let args = match cursor.peek(0, interner)? {
+                    Some(next) if next.kind() == &TokenKind::Punctuator(Punctuator::OpenParen) => {
+                        Arguments::new(self.allow_yield, self.allow_await)
+                            .parse(cursor, interner)?
+                    }
+                    _ => Box::new([]),
+                };
+                let call_node = Call::new(lhs, args);
 
-            Node::from(New::from(call_node))
-        } else {
-            PrimaryExpression::new(self.name, self.allow_yield, self.allow_await)
-                .parse(cursor, interner)?
+                Node::from(New::from(call_node))
+            }
+            _ => PrimaryExpression::new(self.name, self.allow_yield, self.allow_await)
+                .parse(cursor, interner)?,
         };
+
         while let Some(tok) = cursor.peek(0, interner)? {
             match tok.kind() {
                 TokenKind::Punctuator(Punctuator::Dot) => {
@@ -96,7 +101,7 @@ where
 
                     match token.kind() {
                         TokenKind::Identifier(name) => lhs = GetConstField::new(lhs, *name).into(),
-                        TokenKind::Keyword(kw) => {
+                        TokenKind::Keyword((kw, _)) => {
                             lhs = GetConstField::new(lhs, kw.to_sym(interner)).into();
                         }
                         TokenKind::BooleanLiteral(bool) => {

diff --git a/boa_engine/src/syntax/parser/expression/mod.rs b/boa_engine/src/syntax/parser/expression/mod.rs
@@ -98,7 +98,7 @@ macro_rules! expression { ($name:ident, $lower:ident, [$( $op:path ),*], [$( $lo
                             $lower::new($( self.$low_param ),*).parse(cursor, interner)?
                         ).into();
                     }
-                    TokenKind::Keyword(op) if $( op == $op )||* => {
+                    TokenKind::Keyword((op, false)) if $( op == $op )||* => {
                         let _next = cursor.next(interner).expect("token disappeared");
                         lhs = BinOp::new(
                             op.as_binop().expect("Could not get binary operation."),
@@ -540,7 +540,13 @@ where
                     )
                     .into();
                 }
-                TokenKind::Keyword(op)
+                TokenKind::Keyword((Keyword::InstanceOf | Keyword::In, true)) => {
+                    return Err(ParseError::general(
+                        "Keyword must not contain escaped characters",
+                        tok.span().start(),
+                    ));
+                }
+                TokenKind::Keyword((op, false))
                     if op == Keyword::InstanceOf
                         || (op == Keyword::In && self.allow_in == AllowIn(true)) =>
                 {

diff --git a/boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/async_function_expression/mod.rs
@@ -55,7 +55,11 @@ where
     ) -> Result<Self::Output, ParseError> {
         let _timer = Profiler::global().start_event("AsyncFunctionExpression", "Parsing");
         cursor.peek_expect_no_lineterminator(0, "async function expression", interner)?;
-        cursor.expect(Keyword::Function, "async function expression", interner)?;
+        cursor.expect(
+            (Keyword::Function, false),
+            "async function expression",
+            interner,
+        )?;
 
         let name = match cursor
             .peek(0, interner)?

diff --git a/boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/async_generator_expression/mod.rs
@@ -59,7 +59,11 @@ where
         let _timer = Profiler::global().start_event("AsyncGeneratorExpression", "Parsing");
 
         cursor.peek_expect_no_lineterminator(0, "async generator expression", interner)?;
-        cursor.expect(Keyword::Function, "async generator expression", interner)?;
+        cursor.expect(
+            (Keyword::Function, false),
+            "async generator expression",
+            interner,
+        )?;
         cursor.expect(
             TokenKind::Punctuator(Punctuator::Mul),
             "async generator expression",

diff --git a/boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/class_expression/mod.rs
@@ -56,7 +56,7 @@ where
 
         let token = cursor.peek(0, interner)?.ok_or(ParseError::AbruptEnd)?;
         let name = match token.kind() {
-            TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => {
+            TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => {
                 BindingIdentifier::new(self.allow_yield, self.allow_await)
                     .parse(cursor, interner)?
             }

diff --git a/boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs b/boa_engine/src/syntax/parser/expression/primary/function_expression/mod.rs
@@ -64,7 +64,7 @@ where
             .ok_or(ParseError::AbruptEnd)?
             .kind()
         {
-            TokenKind::Identifier(_) | TokenKind::Keyword(Keyword::Yield | Keyword::Await) => {
+            TokenKind::Identifier(_) | TokenKind::Keyword((Keyword::Yield | Keyword::Await, _)) => {
                 Some(BindingIdentifier::new(false, false).parse(cursor, interner)?)
             }
             _ => self.name,