feat: Support ES2023 and hashbangs (#556)

* feat: Support ES2023 and hashbangs

* Improve comment

* Update README with ES2023 feature support

* Update ecmaVersion test runner to include comments

By default, the `ecmaVersion` test runner was not asking the parser for
comments. Now, when the source filename contains `comment`, it sets
`comment: true` and includes comments in the expected result.

This extends the precedent set by filenames containing strings like
`"non-strict"`, `"edge-cases"`, `"modules"`, and a few others, all of
which put the test runner in slightly different modes.

* Define custom Hashbang comment type

Reviewers asked for the hashbang comment to have a type distinct from
`"Line"` [1].

ESLint already handles hashbangs internally, and removing its custom
handling would break backwards compatibility. The name is therefore up
to whatever non-ESLint users of Espree would expect. [2]

The hashbang grammar proposal already addressed this choice directly
[3], so this commit defines the type as `"Hashbang"` to match the
proposal.

[1]: #556 (comment)
[2]: #556 (comment)
[3]: https://github.com/tc39/proposal-hashbang#why-hashbang-instead-of-shebang

README.md

@@ -231,17 +231,11 @@ We are building on top of Acorn, however, so that we can contribute back and hel
 
 ### What ECMAScript features do you support?
 
-Espree supports all ECMAScript 2021 features and partially supports ECMAScript 2022 features.
+Espree supports all ECMAScript 2022 features and partially supports ECMAScript 2023 features.
 
-Because ECMAScript 2022 is still under development, we are implementing features as they are finalized. Currently, Espree supports:
+Because ECMAScript 2023 is still under development, we are implementing features as they are finalized. Currently, Espree supports:
 
-* [Class instance fields](https://github.com/tc39/proposal-class-fields)
-* [Class private instance methods and accessors](https://github.com/tc39/proposal-private-methods)
-* [Class static fields, static private methods and accessors](https://github.com/tc39/proposal-static-class-features)
-* [RegExp match indices](https://github.com/tc39/proposal-regexp-match-indices)
-* [Top-level await](https://github.com/tc39/proposal-top-level-await)
-* [Class static initialization blocks](https://github.com/tc39/proposal-class-static-block)
-* [Ergonomic brand checks for Private Fields](https://github.com/tc39/proposal-private-fields-in-in)
+* [Hashbang grammar](https://github.com/tc39/proposal-hashbang)
 
 See [finished-proposals.md](https://github.com/tc39/proposals/blob/master/finished-proposals.md) to know what features are finalized.
 

lib/espree.js

@@ -15,12 +15,23 @@ const ESPRIMA_FINISH_NODE = Symbol("espree's esprimaFinishNode");
  * @param {int} end The index at which the comment ends.
  * @param {Location} startLoc The location at which the comment starts.
  * @param {Location} endLoc The location at which the comment ends.
+ * @param {string} code The source code being parsed.
  * @returns {Object} The comment object.
  * @private
  */
-function convertAcornCommentToEsprimaComment(block, text, start, end, startLoc, endLoc) {
+function convertAcornCommentToEsprimaComment(block, text, start, end, startLoc, endLoc, code) {
+    let type;
+
+    if (block) {
+        type = "Block";
+    } else if (code.slice(start, start + 2) === "#!") {
+        type = "Hashbang";
+    } else {
+        type = "Line";
+    }
+
     const comment = {
-        type: block ? "Block" : "Line",
+        type,
         value: text
     };
 
@@ -65,6 +76,25 @@ export default () => Parser => {
                     ? new TokenTranslator(tokTypes, code)
                     : null;
 
+            /*
+             * Data that is unique to Espree and is not represented internally
+             * in Acorn.
+             *
+             * For ES2023 hashbangs, Espree will call `onComment()` during the
+             * constructor, so we must define state before having access to
+             * `this`.
+             */
+            const state = {
+                originalSourceType: originalSourceType || options.sourceType,
+                tokens: tokenTranslator ? [] : null,
+                comments: options.comment === true ? [] : null,
+                impliedStrict: ecmaFeatures.impliedStrict === true && options.ecmaVersion >= 5,
+                ecmaVersion: options.ecmaVersion,
+                jsxAttrValueToken: false,
+                lastToken: null,
+                templateElements: []
+            };
+
             // Initialize acorn parser.
             super({
 
@@ -83,38 +113,28 @@ export default () => Parser => {
                     if (tokenTranslator) {
 
                         // Use `tokens`, `ecmaVersion`, and `jsxAttrValueToken` in the state.
-                        tokenTranslator.onToken(token, this[STATE]);
+                        tokenTranslator.onToken(token, state);
                     }
                     if (token.type !== tokTypes.eof) {
-                        this[STATE].lastToken = token;
+                        state.lastToken = token;
                     }
                 },
 
                 // Collect comments
                 onComment: (block, text, start, end, startLoc, endLoc) => {
-                    if (this[STATE].comments) {
-                        const comment = convertAcornCommentToEsprimaComment(block, text, start, end, startLoc, endLoc);
+                    if (state.comments) {
+                        const comment = convertAcornCommentToEsprimaComment(block, text, start, end, startLoc, endLoc, code);
 
-                        this[STATE].comments.push(comment);
+                        state.comments.push(comment);
                     }
                 }
             }, code);
 
             /*
-             * Data that is unique to Espree and is not represented internally in
-             * Acorn. We put all of this data into a symbol property as a way to
-             * avoid potential naming conflicts with future versions of Acorn.
+             * We put all of this data into a symbol property as a way to avoid
+             * potential naming conflicts with future versions of Acorn.
              */
-            this[STATE] = {
-                originalSourceType: originalSourceType || options.sourceType,
-                tokens: tokenTranslator ? [] : null,
-                comments: options.comment === true ? [] : null,
-                impliedStrict: ecmaFeatures.impliedStrict === true && this.options.ecmaVersion >= 5,
-                ecmaVersion: this.options.ecmaVersion,
-                jsxAttrValueToken: false,
-                lastToken: null,
-                templateElements: []
-            };
+            this[STATE] = state;
         }
 
         tokenize() {

lib/options.js

@@ -10,14 +10,15 @@
 const SUPPORTED_VERSIONS = [
     3,
     5,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11,
-    12,
-    13
+    6, // 2015
+    7, // 2016
+    8, // 2017
+    9, // 2018
+    10, // 2019
+    11, // 2020
+    12, // 2021
+    13, // 2022
+    14 // 2023
 ];
 
 /**

tests/fixtures/ecma-version/14/hashbang/invalid-comment-not-at-beginning.result.js

@@ -0,0 +1,6 @@
+export default {
+    "index": 15,
+    "lineNumber": 2,
+    "column": 2,
+    "message": "Unexpected character '!'"
+};

tests/fixtures/ecma-version/14/hashbang/invalid-comment-not-at-beginning.src.js

@@ -0,0 +1,2 @@
+'use strict';
+#!/usr/bin/env node

tests/fixtures/ecma-version/14/hashbang/valid-comment-at-beginning.result.js

@@ -0,0 +1,40 @@
+export default {
+    "type": "Program",
+    "loc": {
+        "start": {
+            "line": 1,
+            "column": 0
+        },
+        "end": {
+            "line": 2,
+            "column": 0
+        }
+    },
+    "range": [
+        0,
+        20
+    ],
+    "body": [],
+    "sourceType": "script",
+    "comments": [
+        {
+            "type": "Hashbang",
+            "value": "/usr/bin/env node",
+            "range": [
+                0,
+                19
+            ],
+            "loc": {
+                "start": {
+                    "line": 1,
+                    "column": 0
+                },
+                "end": {
+                    "line": 1,
+                    "column": 19
+                }
+            }
+        }
+    ],
+    "tokens": []
+};

tests/fixtures/ecma-version/14/hashbang/valid-comment-at-beginning.src.js

@@ -0,0 +1 @@
+#!/usr/bin/env node

tests/lib/ecma-version.js

@@ -68,6 +68,9 @@ describe("ecmaVersion", () => {
 
                 it("should parse correctly when sourceType is script", async () => {
                     config.ecmaVersion = Number(version);
+                    if (filename.includes("comment")) {
+                        config.comment = true;
+                    }
 
                     const absolutePath = path.resolve(__dirname, FIXTURES_DIR, filename.slice(1));
                     // eslint-disable-next-line node/no-unsupported-features/es-syntax
@@ -107,6 +110,9 @@ describe("ecmaVersion", () => {
 
                     config.ecmaVersion = Number(version);
                     config.sourceType = "module";
+                    if (filename.includes("comment")) {
+                        config.comment = true;
+                    }
 
                     // set sourceType of program node to module
                     if (expected.type === "Program") {

tests/lib/supported-ecmaversions.js

@@ -17,15 +17,15 @@ import * as espree from "../../espree.js";
 
 describe("latestEcmaVersion", () => {
     it("should return the latest supported ecmaVersion", () => {
-        assert.strictEqual(espree.latestEcmaVersion, 13);
+        assert.strictEqual(espree.latestEcmaVersion, 14);
     });
 });
 
 describe("supportedEcmaVersions", () => {
     it("should return an array of all supported versions", () => {
         assert.deepStrictEqual(
             espree.supportedEcmaVersions,
-            [3, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+            [3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
         );
     });
 });

tools/update-ecma-version-tests.js

@@ -76,12 +76,14 @@ function main() {
         const moduleOnly = !scriptOnly && name.includes("modules");
         const expectedToBeError = name.includes("/invalid-");
         const expectedToBeOK = name.includes("/valid-");
+        const comment = name.includes("comment");
         const sourceFilePath = `${path.resolve(rootDir, name)}.src.js`;
         const resultFilePath = `${path.resolve(rootDir, name)}.result.js`;
         const moduleResultFilePath = `${path.resolve(rootDir, name)}.module-result.js`;
         const relSourceFilePath = path.relative(process.cwd(), sourceFilePath);
         const code = shelljs.cat(sourceFilePath);
         const parserOptions = {
+            comment,
             loc: true,
             range: true,
             tokens: true,