diff --git a/src/parser/expression_parser.ml b/src/parser/expression_parser.ml index 28d98de7f4b..a6f3e2d4404 100644 --- a/src/parser/expression_parser.ml +++ b/src/parser/expression_parser.ml @@ -547,7 +547,7 @@ module Expression Expect.token env T_PERIOD; let meta = start_loc, "new" in match Peek.token env with - | T_IDENTIFIER "target" -> + | T_IDENTIFIER { raw = "target"; _ } -> let property = Parse.identifier env in let end_loc = fst property in Loc.btwn start_loc end_loc, Expression.(MetaProperty MetaProperty.({ diff --git a/src/parser/lexer.ml b/src/parser/lexer.ml index 55fa10d5072..194685282f8 100644 --- a/src/parser/lexer.ml +++ b/src/parser/lexer.ml @@ -88,10 +88,12 @@ let neg = [%sedlex.regexp? '-', Star whitespace] let line_terminator_sequence = [%sedlex.regexp? '\n' | '\r' | "\r\n"] -(* TODO: allow \u{} *) -let js_id_start = [%sedlex.regexp? '$' | '_' | id_start] +let hex_quad = [%sedlex.regexp? hex_digit, hex_digit, hex_digit, hex_digit] +let unicode_escape = [%sedlex.regexp? "\\u", hex_quad] +let codepoint_escape = [%sedlex.regexp? "\\u{", Plus hex_digit, '}'] +let js_id_start = [%sedlex.regexp? '$' | '_' | id_start | unicode_escape | codepoint_escape] let js_id_continue = [%sedlex.regexp? - '$' | '_' | 0x200C | 0x200D | id_continue + '$' | '_' | 0x200C | 0x200D | id_continue | unicode_escape | codepoint_escape ] let loc_of_offsets env start_offset end_offset = @@ -291,6 +293,56 @@ let mk_num_singleton number_type raw = let value = if neg then ~-.value else value in T_NUMBER_SINGLETON_TYPE { kind = number_type; value; raw } +let decode_identifier = + let assert_valid_unicode_in_identifier env loc code = + let lexbuf = Sedlexing.from_int_array [|code|] in + match%sedlex lexbuf with + | js_id_start -> env + | js_id_continue -> env + | any + | eof -> lex_error env loc Parse_error.IllegalUnicodeEscape + | _ -> failwith "unreachable" + in + let loc_and_lexeme env offset lexbuf = + let start_offset = offset + Sedlexing.lexeme_start lexbuf in + let end_offset = offset + Sedlexing.lexeme_end lexbuf in + let loc = loc_of_offsets env start_offset end_offset in + loc, lexeme lexbuf + in + let rec id_char env offset buf lexbuf = + match%sedlex lexbuf with + | unicode_escape -> + let loc, str = loc_and_lexeme env offset lexbuf in + let hex = String.sub str 2 (String.length str - 2) in + let code = int_of_string ("0x"^hex) in + let env = assert_valid_unicode_in_identifier env loc code in + Wtf8.add_wtf_8 buf code; + id_char env offset buf lexbuf + + | codepoint_escape -> + let loc, str = loc_and_lexeme env offset lexbuf in + let hex = String.sub str 3 (String.length str - 4) in + let code = int_of_string ("0x"^hex) in + let env = assert_valid_unicode_in_identifier env loc code in + Wtf8.add_wtf_8 buf code; + id_char env offset buf lexbuf + + | eof -> + env, Buffer.contents buf + + | any -> + let x = lexeme lexbuf in + Buffer.add_string buf x; + id_char env offset buf lexbuf + + | _ -> failwith "unreachable" + in + fun env raw -> + let offset = Sedlexing.lexeme_start env.lex_lb in + let lexbuf = Sedlexing.Utf8.from_string raw in + let buf = Buffer.create (String.length raw) in + id_char env offset buf lexbuf + let recover env lexbuf ~f = let env = illegal env (loc_of_lexbuf env lexbuf) in Sedlexing.rollback lexbuf; @@ -404,7 +456,7 @@ let string_escape env lexbuf = let code = int_of_string ("0o"^str) in (* 0o1 *) env, str, [|code|], true - | 'u', hex_digit, hex_digit, hex_digit, hex_digit -> + | 'u', hex_quad -> let str = lexeme lexbuf in let hex = String.sub str 1 (String.length str - 1) in let code = int_of_string ("0x"^hex) in @@ -747,11 +799,18 @@ let token (env: Lex_env.t) lexbuf : result = | "yield" -> Token (env, T_YIELD) (* Identifiers *) - | js_id_start, Star js_id_continue -> Token (env, T_IDENTIFIER (lexeme lexbuf)) + | js_id_start, Star js_id_continue -> + let loc = loc_of_lexbuf env lexbuf in + let raw = lexeme lexbuf in + let env, value = decode_identifier env raw in + Token (env, T_IDENTIFIER { loc; value; raw }) (* TODO: Use [Symbol.iterator] instead of @@iterator. *) - | "@@iterator" -> Token (env, T_IDENTIFIER "@@iterator") - | "@@asyncIterator" -> Token (env, T_IDENTIFIER "@@asyncIterator") + | "@@iterator" + | "@@asyncIterator" -> + let loc = loc_of_lexbuf env lexbuf in + let raw = lexeme lexbuf in + Token (env, T_IDENTIFIER { loc; value = raw; raw }) (* Syntax *) | "{" -> Token (env, T_LCURLY) @@ -1571,7 +1630,11 @@ let type_token env lexbuf = | "void" -> Token (env, T_VOID_TYPE) (* Identifiers *) - | js_id_start, Star js_id_continue -> Token (env, T_IDENTIFIER (lexeme lexbuf)) + | js_id_start, Star js_id_continue -> + let loc = loc_of_lexbuf env lexbuf in + let raw = lexeme lexbuf in + let env, value = decode_identifier env raw in + Token (env, T_IDENTIFIER { loc; value; raw }) | "%checks" -> Token (env, T_CHECKS) (* Syntax *) diff --git a/src/parser/object_parser.ml b/src/parser/object_parser.ml index 647a1742936..3a88c00e3a3 100644 --- a/src/parser/object_parser.ml +++ b/src/parser/object_parser.ml @@ -152,8 +152,9 @@ module Object -> Declaration.async env in let generator = Declaration.generator env in - match async, generator, key env with - | false, false, (_, (Property.Identifier (_, "get") as key)) -> + match async, generator, Peek.token env with + | false, false, T_IDENTIFIER { raw = "get"; _ } -> + let _, key = key env in begin match Peek.token env with | T_ASSIGN | T_COLON @@ -163,7 +164,8 @@ module Object | T_RCURLY -> init env start_loc key false false | _ -> get env start_loc, [] end - | false, false, (_, (Property.Identifier (_, "set") as key)) -> + | false, false, T_IDENTIFIER { raw = "set"; _ } -> + let _, key = key env in begin match Peek.token env with | T_ASSIGN | T_COLON @@ -173,7 +175,8 @@ module Object | T_RCURLY -> init env start_loc key false false | _ -> set env start_loc, [] end - | async, generator, (_, key) -> + | async, generator, _ -> + let _, key = key env in init env start_loc key async generator end @@ -563,7 +566,7 @@ module Object decorators; }))) - in fun env -> Ast.Expression.Object.Property.( + in fun env -> let start_loc = Peek.loc env in let decorators = decorator_list env in let static = @@ -580,9 +583,9 @@ module Object | false, Some _ -> Declaration.generator env | _ -> generator in - match (async, generator, key ~class_body:true env) with - | false, false, - (_, (Identifier (_, "get") as key)) -> + match async, generator, Peek.token env with + | false, false, T_IDENTIFIER { raw = "get"; _ } -> + let _, key = key ~class_body:true env in (match Peek.token env with | T_LESS_THAN | T_COLON @@ -593,8 +596,8 @@ module Object | _ -> error_unsupported_variance env variance; get env start_loc decorators static) - | false, false, - (_, (Identifier (_, "set") as key)) -> + | false, false, T_IDENTIFIER { raw = "set"; _ } -> + let _, key = key ~class_body:true env in (match Peek.token env with | T_LESS_THAN | T_COLON @@ -605,9 +608,9 @@ module Object | _ -> error_unsupported_variance env variance; set env start_loc decorators static) - | _, _, (_, key) -> + | _, _, _ -> + let _, key = key ~class_body:true env in init env start_loc decorators key async generator static variance - ) let class_declaration env decorators = (* 10.2.1 says all parts of a class definition are strict *) diff --git a/src/parser/parse_error.ml b/src/parser/parse_error.ml index 69ccd4a14f3..f5a6076ffb4 100644 --- a/src/parser/parse_error.ml +++ b/src/parser/parse_error.ml @@ -44,6 +44,7 @@ type t = | IllegalContinue | IllegalBreak | IllegalReturn + | IllegalUnicodeEscape | StrictModeWith | StrictCatchVariable | StrictVarName @@ -154,6 +155,7 @@ module PP = | IllegalContinue -> "Illegal continue statement" | IllegalBreak -> "Illegal break statement" | IllegalReturn -> "Illegal return statement" + | IllegalUnicodeEscape -> "Illegal Unicode escape" | StrictModeWith -> "Strict mode code may not include a with statement" | StrictCatchVariable -> "Catch variable may not be eval or arguments in strict mode" | StrictVarName -> "Variable name may not be eval or arguments in strict mode" diff --git a/src/parser/parser_common.ml b/src/parser/parser_common.ml index 7b1784d1866..e60c688915e 100644 --- a/src/parser/parser_common.ml +++ b/src/parser/parser_common.ml @@ -50,7 +50,7 @@ let identifier_name env = let loc = Peek.loc env in let name = match Peek.token env with (* obviously, Identifier is a valid IdentifierName *) - | T_IDENTIFIER id -> id + | T_IDENTIFIER { value; _ } -> value (* keywords are also IdentifierNames *) | T_AWAIT -> "await" | T_BREAK -> "break" diff --git a/src/parser/parser_env.ml b/src/parser/parser_env.ml index 7836eb348ca..0b974baa900 100644 --- a/src/parser/parser_env.ml +++ b/src/parser/parser_env.ml @@ -402,7 +402,7 @@ let is_future_reserved = function | _ -> false let token_is_future_reserved = Token.(function - | T_IDENTIFIER name when is_future_reserved name -> true + | T_IDENTIFIER { raw; _ } when is_future_reserved raw -> true | T_ENUM -> true | _ -> false ) @@ -420,7 +420,7 @@ let is_strict_reserved = function | _ -> false let token_is_strict_reserved = Token.(function - | T_IDENTIFIER name when is_strict_reserved name -> true + | T_IDENTIFIER { raw; _ } when is_strict_reserved raw -> true | T_INTERFACE | T_IMPLEMENTS | T_PACKAGE @@ -441,7 +441,7 @@ let is_restricted = function | _ -> false let token_is_restricted = Token.(function - | T_IDENTIFIER name when is_restricted name -> true + | T_IDENTIFIER { raw; _ } when is_restricted raw -> true | _ -> false ) @@ -629,6 +629,13 @@ module Expect = struct if Peek.token env <> t then error_unexpected env; Eat.token env + let identifier env name = + begin match Peek.token env with + | Token.T_IDENTIFIER { raw; _ } when raw = name -> () + | _ -> error_unexpected env + end; + Eat.token env + (* If the next token is t, then eat it and return true * else return false *) let maybe env t = diff --git a/src/parser/parser_env.mli b/src/parser/parser_env.mli index 30bf2d37cc6..89b7e4b8b38 100644 --- a/src/parser/parser_env.mli +++ b/src/parser/parser_env.mli @@ -151,6 +151,7 @@ end module Expect : sig val token : env -> Token.t -> unit + val identifier : env -> string -> unit val maybe : env -> Token.t -> bool end diff --git a/src/parser/statement_parser.ml b/src/parser/statement_parser.ml index b62d1222cc7..3cec5b146e0 100644 --- a/src/parser/statement_parser.ml +++ b/src/parser/statement_parser.ml @@ -664,11 +664,10 @@ module Statement Expect.token env T_EXTENDS; supers env [] end else [] in - let mixins = if Peek.token env = T_IDENTIFIER "mixins" - then begin - Eat.token env; - supers env [] - end else [] in + let mixins = match Peek.token env with + | T_IDENTIFIER { raw = "mixins"; _ } -> Eat.token env; supers env [] + | _ -> [] + in let body = Type._object ~allow_static:true env in Statement.Interface.({ id; @@ -827,7 +826,7 @@ module Statement fun ?(in_module=false) env -> let start_loc = Peek.loc env in Expect.token env T_DECLARE; - Expect.token env (T_IDENTIFIER "module"); + Expect.identifier env "module"; if in_module || Peek.token env = T_PERIOD then let loc, exports = with_loc declare_module_exports env in @@ -837,7 +836,7 @@ module Statement and declare_module_exports env = Expect.token env T_PERIOD; - Expect.token env (T_IDENTIFIER "exports"); + Expect.identifier env "exports"; let type_annot = Type.annotation env in Eat.semicolon env; Statement.DeclareModuleExports type_annot @@ -869,7 +868,7 @@ module Statement declare_var_statement env | T_EXPORT when in_module -> declare_export_declaration ~allow_export_type:in_module env - | T_IDENTIFIER "module" -> + | T_IDENTIFIER { raw = "module"; _ } -> declare_module ~in_module env | _ when in_module -> ( match Peek.token env with @@ -886,7 +885,7 @@ module Statement ) and export_source env = - Expect.token env (T_IDENTIFIER "from"); + Expect.identifier env "from"; match Peek.token env with | T_STRING (loc, value, raw, octal) -> if octal then strict_error env Error.StrictOctalLiteral; @@ -936,14 +935,15 @@ module Statement let specifier = with_loc (fun env -> let local = identifier_name env in let exported = - if Expect.maybe env (T_IDENTIFIER "as") then begin + match Peek.token env with + | T_IDENTIFIER { raw = "as"; _ } -> + Eat.token env; let exported = identifier_name env in record_export env exported; Some exported - end else begin + | _ -> record_export env local; None - end in { Statement.ExportNamedDeclaration.ExportSpecifier.local; exported; } ) env in @@ -1097,13 +1097,14 @@ module Statement let parse_export_star_as = (parse_options env).esproposal_export_star_as in - if Peek.token env = T_IDENTIFIER "as" - then ( + match Peek.token env with + | T_IDENTIFIER { raw = "as"; _ } -> Eat.token env; if parse_export_star_as then Some (Parse.identifier env) else (error env Error.UnexpectedTypeDeclaration; None) - ) else None + | _ -> + None in let specifiers = Some (ExportBatchSpecifier (loc, local_name)) @@ -1128,12 +1129,12 @@ module Statement let specifiers = export_specifiers env [] in Expect.token env T_RCURLY; let source = - if Peek.token env = T_IDENTIFIER "from" then + match Peek.token env with + | T_IDENTIFIER { raw = "from"; _ } -> Some (export_source env) - else begin + | _ -> assert_export_specifier_identifiers env specifiers; None - end in Eat.semicolon env; Statement.ExportNamedDeclaration { @@ -1215,13 +1216,14 @@ module Statement (parse_options env).esproposal_export_star_as in let local_name = - if Peek.token env = T_IDENTIFIER "as" - then ( + match Peek.token env with + | T_IDENTIFIER { raw = "as"; _ } -> Eat.token env; if parse_export_star_as then Some (Parse.identifier env) else (error env Error.UnexpectedTypeDeclaration; None) - ) else None + | _ -> + None in let specifiers = Statement.ExportNamedDeclaration.( Some (ExportBatchSpecifier (loc, local_name)) @@ -1271,12 +1273,12 @@ module Statement let specifiers = export_specifiers env [] in Expect.token env T_RCURLY; let source = - if Peek.token env = T_IDENTIFIER "from" then + match Peek.token env with + | T_IDENTIFIER { raw = "from"; _ } -> Some (export_source env) - else begin + | _ -> assert_export_specifier_identifiers env specifiers; None - end in Eat.semicolon env; Statement.DeclareExportDeclaration { @@ -1292,7 +1294,7 @@ module Statement let open Statement.ImportDeclaration in let source env = - Expect.token env (T_IDENTIFIER "from"); + Expect.identifier env "from"; match Peek.token env with | T_STRING (loc, value, raw, octal) -> if octal then strict_error env Error.StrictOctalLiteral; @@ -1320,7 +1322,11 @@ module Statement | _ -> false, None in let specifier = - if Peek.token env = T_IDENTIFIER "as" then begin + let is_as = match Peek.token env with + | T_IDENTIFIER { raw = "as"; _ } -> true + | _ -> false + in + if is_as then begin let as_ident = Parse.identifier env in if starts_w_type && not (Peek.is_identifier env) then begin (* `import {type as ,` or `import {type as }` *) @@ -1338,9 +1344,12 @@ module Statement error_at env (fst remote, Error.ImportTypeShorthandOnlyInPureImport); let remote = identifier_name env in let local = - if Expect.maybe env (T_IDENTIFIER "as") - then Some (Type.type_identifier env) - else None + match Peek.token env with + | T_IDENTIFIER { raw = "as"; _ } -> + Eat.token env; + Some (Type.type_identifier env) + | _ -> + None in { remote; local; kind } end else begin @@ -1356,7 +1365,7 @@ module Statement match Peek.token env with | T_MULT -> Expect.token env T_MULT; - Expect.token env (T_IDENTIFIER "as"); + Expect.identifier env "as"; let id = Parse.identifier env in [ImportNamespaceSpecifier (Loc.btwn start_loc (fst id), id)] | _ -> @@ -1405,7 +1414,7 @@ module Statement let importKind, default_specifier = ( match type_ident, Peek.token env with | Some type_ident, T_COMMA (* `import type,` *) - | Some type_ident, T_IDENTIFIER "from" -> (* `import type from` *) + | Some type_ident, T_IDENTIFIER { raw = "from"; _ } -> (* `import type from` *) ImportValue, ImportDefaultSpecifier type_ident | _ -> (* Either `import type Foo` or `import Foo` *) importKind, ImportDefaultSpecifier (Parse.identifier env) diff --git a/src/parser/test/esprima/ES6/identifier/escaped_all.skip b/src/parser/test/esprima/ES6/identifier/escaped_all.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_all.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/escaped_math_alef.skip b/src/parser/test/esprima/ES6/identifier/escaped_math_alef.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_math_alef.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/escaped_math_dal_part.skip b/src/parser/test/esprima/ES6/identifier/escaped_math_dal_part.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_math_dal_part.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/escaped_math_kaf_lam.skip b/src/parser/test/esprima/ES6/identifier/escaped_math_kaf_lam.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_math_kaf_lam.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/escaped_math_zain_start.skip b/src/parser/test/esprima/ES6/identifier/escaped_math_zain_start.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_math_zain_start.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/escaped_part.skip b/src/parser/test/esprima/ES6/identifier/escaped_part.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_part.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/escaped_start.skip b/src/parser/test/esprima/ES6/identifier/escaped_start.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/escaped_start.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/esprima/ES6/identifier/weierstrass_weierstrass.skip b/src/parser/test/esprima/ES6/identifier/weierstrass_weierstrass.skip deleted file mode 100644 index ed7bd50f29a..00000000000 --- a/src/parser/test/esprima/ES6/identifier/weierstrass_weierstrass.skip +++ /dev/null @@ -1 +0,0 @@ -TODO: parse errors diff --git a/src/parser/test/flow/invalid_syntax/migrated_0010.js b/src/parser/test/flow/invalid_syntax/migrated_0010.js index 7405552f34e..b07bc96688e 100644 --- a/src/parser/test/flow/invalid_syntax/migrated_0010.js +++ b/src/parser/test/flow/invalid_syntax/migrated_0010.js @@ -1,2 +1,2 @@ // 11.8.5.1 no unicode as flags -var x = /[P QR]/\\u0067 +var x = /[P QR]/\u0067 diff --git a/src/parser/test/flow/invalid_syntax/migrated_0010.tree.json b/src/parser/test/flow/invalid_syntax/migrated_0010.tree.json index 020f8804bd5..c842dbaf72e 100644 --- a/src/parser/test/flow/invalid_syntax/migrated_0010.tree.json +++ b/src/parser/test/flow/invalid_syntax/migrated_0010.tree.json @@ -1,21 +1,13 @@ { "errors":[ { - "loc":{"source":null,"start":{"line":2,"column":16},"end":{"line":2,"column":17}}, - "message":"Unexpected token ILLEGAL" - }, - { - "loc":{"source":null,"start":{"line":2,"column":17},"end":{"line":2,"column":18}}, - "message":"Unexpected token ILLEGAL" - }, - { - "loc":{"source":null,"start":{"line":2,"column":18},"end":{"line":2,"column":23}}, + "loc":{"source":null,"start":{"line":2,"column":16},"end":{"line":2,"column":22}}, "message":"Unexpected identifier" } ], "type":"Program", - "loc":{"source":null,"start":{"line":2,"column":0},"end":{"line":2,"column":23}}, - "range":[32,55], + "loc":{"source":null,"start":{"line":2,"column":0},"end":{"line":2,"column":22}}, + "range":[32,54], "body":[ { "type":"VariableDeclaration", @@ -48,13 +40,13 @@ }, { "type":"ExpressionStatement", - "loc":{"source":null,"start":{"line":2,"column":18},"end":{"line":2,"column":23}}, - "range":[50,55], + "loc":{"source":null,"start":{"line":2,"column":16},"end":{"line":2,"column":22}}, + "range":[48,54], "expression":{ "type":"Identifier", - "loc":{"source":null,"start":{"line":2,"column":18},"end":{"line":2,"column":23}}, - "range":[50,55], - "name":"u0067", + "loc":{"source":null,"start":{"line":2,"column":16},"end":{"line":2,"column":22}}, + "range":[48,54], + "name":"g", "typeAnnotation":null, "optional":false }, diff --git a/src/parser/test/test262_baseline.txt b/src/parser/test/test262_baseline.txt index 36cfc0a3395..c38435d4f5d 100644 --- a/src/parser/test/test262_baseline.txt +++ b/src/parser/test/test262_baseline.txt @@ -54,18 +54,6 @@ annexB/language/literals/numeric/non-octal-decimal-integer.js (default) Unexpected token ILLEGAL at (34, 17) to (34, 20) annexB/language/statements/for-in/nonstrict-initializer.js (default) Invalid left-hand side in for-in at (11, 7) to (11, 24) -built-ins/Function/prototype/toString/unicode.js (strict mode) - Unexpected token ILLEGAL at (12, 9) to (12, 10) -built-ins/Function/prototype/toString/unicode.js (default) - Unexpected token ILLEGAL at (12, 9) to (12, 10) -built-ins/RegExp/named-groups/non-unicode-property-names.js (strict mode) - Unexpected token ILLEGAL at (13, 51) to (13, 52) -built-ins/RegExp/named-groups/non-unicode-property-names.js (default) - Unexpected token ILLEGAL at (13, 51) to (13, 52) -built-ins/RegExp/named-groups/unicode-property-names.js (strict mode) - Unexpected token ILLEGAL at (12, 52) to (12, 53) -built-ins/RegExp/named-groups/unicode-property-names.js (default) - Unexpected token ILLEGAL at (12, 52) to (12, 53) language/block-scope/syntax/redeclaration/async-function-declaration-attempt-to-redeclare-with-async-function-declaration.js (strict mode) Missing parse error language/block-scope/syntax/redeclaration/async-function-declaration-attempt-to-redeclare-with-async-function-declaration.js (default) @@ -448,66 +436,6 @@ language/global-code/new.target.js (strict mode) Unexpected token . at (19, 3) to (19, 4) language/global-code/new.target.js (default) Unexpected token . at (19, 3) to (19, 4) -language/identifiers/part-digits-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 5) to (10, 6) -language/identifiers/part-digits-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 5) to (10, 6) -language/identifiers/part-digits-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 5) to (10, 6) -language/identifiers/part-digits-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 5) to (10, 6) -language/identifiers/start-escape-seq.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/start-escape-seq.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-dollar-sign-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-dollar-sign-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-dollar-sign-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-dollar-sign-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-underscore-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-underscore-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-underscore-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/val-underscore-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-lower-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-lower-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-lower-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-lower-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-upper-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-upper-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-upper-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-eng-alpha-upper-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-lower-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-lower-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-lower-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-lower-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-upper-via-escape-hex.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-upper-via-escape-hex.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-upper-via-escape-hex4.js (strict mode) - Unexpected token ILLEGAL at (10, 4) to (10, 5) -language/identifiers/vals-rus-alpha-upper-via-escape-hex4.js (default) - Unexpected token ILLEGAL at (10, 4) to (10, 5) language/import/dup-bound-names.js (strict mode) Missing parse error language/import/dup-bound-names.js (default) @@ -650,10 +578,6 @@ language/module-code/instn-resolve-order-src.js (strict mode) Missing parse error language/module-code/instn-resolve-order-src.js (default) Missing parse error -language/module-code/namespace/internals/own-property-keys-sort.js (strict mode) - Unexpected token ILLEGAL at (22, 14) to (22, 15) -language/module-code/namespace/internals/own-property-keys-sort.js (default) - Unexpected token ILLEGAL at (22, 14) to (22, 15) language/module-code/parse-err-hoist-lex-fun.js (strict mode) Missing parse error language/module-code/parse-err-hoist-lex-fun.js (default) @@ -798,18 +722,6 @@ language/statements/for/head-let-bound-names-in-stmt.js (default) Missing parse error language/statements/for/head-lhs-let.js (default) Unexpected token = at (25, 4) to (25, 5) -language/statements/function/S13_A7_T1.js (strict mode) - Unexpected token ILLEGAL at (55, 25) to (55, 26) -language/statements/function/S13_A7_T1.js (default) - Unexpected token ILLEGAL at (55, 25) to (55, 26) -language/statements/function/S14_A5_T1.js (strict mode) - Unexpected token ILLEGAL at (23, 9) to (23, 10) -language/statements/function/S14_A5_T1.js (default) - Unexpected token ILLEGAL at (23, 9) to (23, 10) -language/statements/function/S14_A5_T2.js (strict mode) - Unexpected token ILLEGAL at (23, 9) to (23, 10) -language/statements/function/S14_A5_T2.js (default) - Unexpected token ILLEGAL at (23, 9) to (23, 10) language/statements/function/use-strict-with-non-simple-param.js (strict mode) Missing parse error language/statements/function/use-strict-with-non-simple-param.js (default) @@ -826,16 +738,14 @@ language/statements/labeled/continue.js (strict mode) Missing parse error language/statements/labeled/continue.js (default) Missing parse error +language/statements/labeled/value-await-module-escaped.js (strict mode) + Missing parse error +language/statements/labeled/value-await-module-escaped.js (default) + Missing parse error language/statements/labeled/value-await-module.js (strict mode) Missing parse error language/statements/labeled/value-await-module.js (default) Missing parse error -language/statements/labeled/value-await-non-module-escaped.js (strict mode) - Unexpected token ILLEGAL at (15, 2) to (15, 3) -language/statements/labeled/value-await-non-module-escaped.js (default) - Unexpected token ILLEGAL at (15, 2) to (15, 3) -language/statements/labeled/value-yield-non-strict-escaped.js (default) - Unexpected token ILLEGAL at (11, 2) to (11, 3) language/statements/let/redeclaration-error-from-within-strict-mode-function.js (default) Missing parse error language/statements/let/syntax/attempt-to-redeclare-let-binding-with-function-declaration.js (strict mode) @@ -846,8 +756,6 @@ language/statements/let/syntax/attempt-to-redeclare-let-binding-with-var.js (str Missing parse error language/statements/let/syntax/attempt-to-redeclare-let-binding-with-var.js (default) Missing parse error -language/statements/let/syntax/escaped-let.js (default) - Unexpected token ILLEGAL at (22, 1) to (22, 2) language/statements/let/syntax/identifier-let-allowed-as-lefthandside-expression-non-strict.js (default) Unexpected token in at (10, 9) to (10, 11) language/statements/switch/syntax/redeclaration/async-function-declaration-attempt-to-redeclare-with-async-function-declaration.js (strict mode) @@ -1454,10 +1362,6 @@ language/statements/try/scope-catch-param-lex-open.js (default) Unexpected token [ at (16, 9) to (16, 10) language/statements/try/scope-catch-param-var-none.js (default) Unexpected token [ at (17, 9) to (17, 10) -language/statements/variable/S12.2_A4.js (strict mode) - Unexpected token ILLEGAL at (20, 4) to (20, 5) -language/statements/variable/S12.2_A4.js (default) - Unexpected token ILLEGAL at (20, 4) to (20, 5) language/statements/while/S12.6.2_A15.js (strict mode) Missing parse error language/statements/while/S12.6.2_A15.js (default) @@ -1468,13 +1372,11 @@ language/white-space/mongolian-vowel-separator.js (default) Missing parse error === Summary === -Passed: 54571 (98.67%) -Failed: 734 (1.33%) +Passed: 54619 (98.76%) +Failed: 686 (1.24%) Features: - Reflect: 244/246 (99.19%) Symbol.iterator: 1314/1324 (99.24%) - Symbol.toStringTag: 80/82 (97.56%) arrow-function: 22/24 (91.67%) async-functions: 157/217 (72.35%) async-iteration: 6984/7052 (99.04%) @@ -1483,4 +1385,3 @@ Features: generators: 1325/1343 (98.66%) let: 129/135 (95.56%) object-rest: 614/620 (99.03%) - regexp-named-groups: 36/40 (90.00%) diff --git a/src/parser/token.ml b/src/parser/token.ml index d5eaa3e79c1..233b1ce47ae 100644 --- a/src/parser/token.ml +++ b/src/parser/token.ml @@ -12,7 +12,7 @@ type t = | T_NUMBER of { kind: number_type; raw: string } | T_STRING of (Loc.t * string * string * bool) (* loc, value, raw, octal *) | T_TEMPLATE_PART of (Loc.t * template_part * bool) (* loc, value, is_tail *) - | T_IDENTIFIER of string + | T_IDENTIFIER of { loc: Loc.t; value: string; raw: string } | T_REGEXP of (Loc.t * string * string) (* /pattern/flags *) (* Syntax *) | T_LCURLY @@ -297,7 +297,7 @@ let value_of_token = function | T_NUMBER { raw; _ } -> raw | T_STRING (_, _, raw, _) -> raw | T_TEMPLATE_PART (_, { literal; _ }, _) -> literal - | T_IDENTIFIER raw -> raw + | T_IDENTIFIER { raw; _ } -> raw | T_REGEXP (_, pattern, flags) -> "/" ^ pattern ^ "/" ^ flags | T_LCURLY -> "{" | T_RCURLY -> "}"