diff --git a/package.json b/package.json index 93074e7..6f8f638 100644 --- a/package.json +++ b/package.json @@ -26,12 +26,16 @@ ], "devDependencies": { "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", "@types/node": "^20.0.0", "acorn": "^8.0.0", "acorn-jsx": "^5.0.0", "c8": "^7.0.0", "micromark": "^3.0.0", "micromark-build": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", "prettier": "^2.0.0", "remark-cli": "^11.0.0", "remark-preset-wooorm": "^9.0.0", diff --git a/packages/micromark-extension-mdx-expression/dev/lib/syntax.js b/packages/micromark-extension-mdx-expression/dev/lib/syntax.js index d340080..564720c 100644 --- a/packages/micromark-extension-mdx-expression/dev/lib/syntax.js +++ b/packages/micromark-extension-mdx-expression/dev/lib/syntax.js @@ -27,12 +27,12 @@ * `micromark-extension-mdx-jsx` to prohobit empty attribute values). */ -import {ok as assert} from 'uvu/assert' import {factoryMdxExpression} from 'micromark-factory-mdx-expression' import {factorySpace} from 'micromark-factory-space' import {markdownLineEnding, markdownSpace} from 'micromark-util-character' import {codes} from 'micromark-util-symbol/codes.js' import {types} from 'micromark-util-symbol/types.js' +import {ok as assert} from 'uvu/assert' /** * Create an extension for `micromark` to enable MDX expression syntax. @@ -111,6 +111,7 @@ export function mdxExpression(options) { */ function start(code) { // To do: in `markdown-rs`, constructs need to parse the indent themselves. + // This should also be introduced in `micromark-js`. assert(code === codes.leftCurlyBrace, 'expected `{`') return before(code) } diff --git a/packages/micromark-factory-mdx-expression/dev/index.js b/packages/micromark-factory-mdx-expression/dev/index.js index 8f65f24..04af7aa 100644 --- a/packages/micromark-factory-mdx-expression/dev/index.js +++ b/packages/micromark-factory-mdx-expression/dev/index.js @@ -10,30 +10,29 @@ /** * @typedef MdxSignalOk + * Good result. * @property {'ok'} type + * Type. * @property {Program | undefined} estree + * Value. * * @typedef MdxSignalNok + * Bad result. * @property {'nok'} type + * Type. * @property {VFileMessage} message - * - * @typedef MdxSignalEof - * Currently not given back. - * @property {'eof'} type - * @property {VFileMessage} message + * Value. * * @typedef {MdxSignalOk | MdxSignalNok} MdxSignal */ -import {ok as assert} from 'uvu/assert' -import {markdownLineEnding, markdownSpace} from 'micromark-util-character' +import {markdownLineEnding} from 'micromark-util-character' +import {eventsToAcorn} from 'micromark-util-events-to-acorn' import {codes} from 'micromark-util-symbol/codes.js' import {types} from 'micromark-util-symbol/types.js' -import {constants} from 'micromark-util-symbol/constants.js' -import {factorySpace} from 'micromark-factory-space' import {positionFromEstree} from 'unist-util-position-from-estree' +import {ok as assert} from 'uvu/assert' import {VFileMessage} from 'vfile-message' -import {eventsToAcorn} from 'micromark-util-events-to-acorn' /** * @this {TokenizeContext} @@ -232,34 +231,21 @@ export function factoryMdxExpression( ) } - if (markdownSpace(code)) { - // Idea: investigate if we’d need to use more complex stripping. - // Take this example: - // - // ```markdown - // > aaa d - // > `} /> eee - // ``` - // - // Currently, the “paragraph” starts at `> | aaa`, so for the next line - // here we split it into `>␠|␠␠␠␠|␠d` (prefix, this indent here, - // expression data). - // The intention above is likely for the split to be as `>␠␠|␠␠␠␠|d`, - // which is impossible, but we can mimick it with `>␠|␠␠␠␠␠|d`. - // - // To improve the situation, we could take `tokenizer.line_start` at - // the start of the expression and move past whitespace. - // For future lines, we’d move at most to - // `line_start_shifted.column + 4`. - return factorySpace( - effects, - before, - types.linePrefix, - constants.tabSize - )(code) - } - + // Idea: investigate if we’d need to use more complex stripping. + // Take this example: + // + // ```markdown + // > aaa d + // > `} /> eee + // ``` + // + // The block quote takes one space from each line, the paragraph doesn’t. + // The intent above is *perhaps* for the split to be as `>␠␠|␠␠␠␠|d`, + // Currently, we *don’t* do anything at all, it’s `>␠|␠␠␠␠␠|d` instead. + // + // Note: we used to have some handling here, and `markdown-rs` still does, + // which should be removed. return before(code) } } diff --git a/packages/micromark-factory-mdx-expression/package.json b/packages/micromark-factory-mdx-expression/package.json index b75c6c8..5012e46 100644 --- a/packages/micromark-factory-mdx-expression/package.json +++ b/packages/micromark-factory-mdx-expression/package.json @@ -39,7 +39,7 @@ "default": "./index.js" }, "dependencies": { - "micromark-factory-space": "^1.0.0", + "@types/estree": "^1.0.0", "micromark-util-character": "^1.0.0", "micromark-util-events-to-acorn": "^1.0.0", "micromark-util-symbol": "^1.0.0", diff --git a/packages/micromark-util-events-to-acorn/dev/index.js b/packages/micromark-util-events-to-acorn/dev/index.js index 9fd6678..e25654d 100644 --- a/packages/micromark-util-events-to-acorn/dev/index.js +++ b/packages/micromark-util-events-to-acorn/dev/index.js @@ -5,8 +5,10 @@ * @typedef {import('acorn').Token} Token * @typedef {import('estree').Node} EstreeNode * @typedef {import('estree').Program} Program + * @typedef {import('micromark-util-types').Chunk} Chunk * @typedef {import('micromark-util-types').Event} Event - * @typedef {import('micromark-util-types').Point} Point + * @typedef {import('micromark-util-types').Point} MicromarkPoint + * @typedef {import('unist').Point} UnistPoint */ /** @@ -34,7 +36,7 @@ * Typically `acorn`, object with `parse` and `parseExpressionAt` fields. * @property {AcornOptions | null | undefined} [acornOptions] * Configuration for `acorn`. - * @property {Point | null | undefined} [start] + * @property {MicromarkPoint | null | undefined} [start] * Place where events start. * @property {string | null | undefined} [prefix=''] * Text to place before events. @@ -55,12 +57,20 @@ * @property {boolean} swallow * Whether the error, if there is one, can be swallowed and more JavaScript * could be valid. + * + * @typedef {[number, MicromarkPoint]} Stop + * + * @typedef Collection + * @property {string} value + * @property {Array} stops */ -import {ok as assert} from 'uvu/assert' import {visit} from 'estree-util-visit' +import {codes} from 'micromark-util-symbol/codes.js' +import {values} from 'micromark-util-symbol/values.js' +import {types} from 'micromark-util-symbol/types.js' +import {ok as assert} from 'uvu/assert' import {VFileMessage} from 'vfile-message' -import {location} from 'vfile-location' /** * Parse a list of micromark events with acorn. @@ -83,18 +93,11 @@ export function eventsToAcorn(events, options) { const tokens = [] const onComment = acornOptions.onComment const onToken = acornOptions.onToken - /** @type {Array} */ - const chunks = [] - /** @type {Record} */ - const lines = {} - let index = -1 let swallow = false /** @type {AcornNode | undefined} */ let estree /** @type {AcornError | undefined} */ let exception - /** @type {number} */ - let startLine /** @type {AcornOptions} */ const acornConfig = Object.assign({}, acornOptions, { onComment: comments, @@ -105,28 +108,25 @@ export function eventsToAcorn(events, options) { acornConfig.onToken = tokens } - // We use `events` to detect everything, however, it could be empty. - // In that case, we need `options.start` to make sense of positional info. - if (options.start) { - startLine = options.start.line - lines[startLine] = options.start - } - - while (++index < events.length) { - const [kind, token, context] = events[index] + const collection = collect(events, [ + types.lineEnding, + // To do: these should be passed by users in parameters. + 'expressionChunk', // From tests. + 'mdxFlowExpressionChunk', // Flow chunk. + 'mdxTextExpressionChunk', // Text chunk. + // JSX: + 'mdxJsxTextTagExpressionAttributeValue', + 'mdxJsxTextTagAttributeValueExpressionValue', + 'mdxJsxFlowTagExpressionAttributeValue', + 'mdxJsxFlowTagAttributeValueExpressionValue', + // ESM: + 'mdxjsEsmData' + ]) - // Assume only void events (and `enter` followed immediately by an `exit`). - if (kind === 'exit') { - chunks.push(context.sliceSerialize(token)) - setPoint(token.start) - setPoint(token.end) - } - } + const source = collection.value - const source = chunks.join('') const value = prefix + source + suffix const isEmptyExpression = options.expression && empty(source) - const place = location(source) if (isEmptyExpression && !options.allowEmpty) { throw new VFileMessage( @@ -145,6 +145,8 @@ export function eventsToAcorn(events, options) { const error = /** @type {AcornError} */ (error_) const point = parseOffsetToUnistPoint(error.pos) error.message = String(error.message).replace(/ \(\d+:\d+\)$/, '') + // Always defined in our unist points that come from micromark. + assert(point.offset, 'expected `offset`') error.pos = point.offset error.loc = {line: point.line, column: point.column - 1} exception = error @@ -177,6 +179,8 @@ export function eventsToAcorn(events, options) { const error = /** @type {AcornError} */ ( new Error('Unexpected content after expression') ) + // Always defined in our unist points that come from micromark. + assert(point.offset, 'expected `offset`') error.pos = point.offset error.loc = {line: point.line, column: point.column - 1} exception = error @@ -269,6 +273,9 @@ export function eventsToAcorn(events, options) { assert('end' in nodeOrToken, 'expected `end` in node or token from acorn') const pointStart = parseOffsetToUnistPoint(nodeOrToken.start) const pointEnd = parseOffsetToUnistPoint(nodeOrToken.end) + // Always defined in our unist points that come from micromark. + assert(pointStart.offset, 'expected `offset`') + assert(pointEnd.offset, 'expected `offset`') nodeOrToken.start = pointStart.offset nodeOrToken.end = pointEnd.offset nodeOrToken.loc = { @@ -291,7 +298,7 @@ export function eventsToAcorn(events, options) { * value. * * @param {number} acornOffset - * @returns {Point} + * @returns {UnistPoint} */ function parseOffsetToUnistPoint(acornOffset) { let sourceOffset = acornOffset - prefix.length @@ -302,31 +309,21 @@ export function eventsToAcorn(events, options) { sourceOffset = source.length } - const pointInSource = place.toPoint(sourceOffset) - assert( - typeof startLine === 'number', - 'expected `startLine` to be found or given ' - ) - assert(typeof pointInSource.line === 'number') - assert(typeof pointInSource.column === 'number') - const line = startLine + (pointInSource.line - 1) - assert(line in lines, 'expected line to be defined') - const column = lines[line].column + (pointInSource.column - 1) - const offset = lines[line].offset + (pointInSource.column - 1) - return /** @type {Point} */ ({line, column, offset}) - } + let point = relativeToPoint(collection.stops, sourceOffset) - /** @param {Point} point */ - function setPoint(point) { - // Not passed by `micromark-extension-mdxjs-esm` - /* c8 ignore next 3 */ - if (!startLine || point.line < startLine) { - startLine = point.line + if (!point) { + assert( + options.start, + 'empty expressions are need `options.start` being passed' + ) + point = { + line: options.start.line, + column: options.start.column, + offset: options.start.offset + } } - if (!(point.line in lines) || lines[point.line].offset > point.offset) { - lines[point.line] = point - } + return point } } @@ -345,3 +342,136 @@ function empty(value) { .replace(/\/\/[^\r\n]*(\r\n|\n|\r)/g, '') ) } + +// Port from . +/** + * @param {Array} events + * @param {Array} names + * @returns {Collection} + */ +function collect(events, names) { + /** @type {Collection} */ + const result = {value: '', stops: []} + let index = -1 + + while (++index < events.length) { + const event = events[index] + + // Assume void. + if (event[0] === 'enter' && names.includes(event[1].type)) { + const chunks = event[2].sliceStream(event[1]) + + // Drop virtual spaces. + while (chunks.length > 0 && chunks[0] === codes.virtualSpace) { + chunks.shift() + } + + const value = serializeChunks(chunks) + result.stops.push([result.value.length, event[1].start]) + result.value += value + result.stops.push([result.value.length, event[1].end]) + } + } + + return result +} + +// Port from . +/** + * Turn a relative offset into an absolute offset. + * + * @param {Array} stops + * @param {number} relative + * @returns {UnistPoint | undefined} + */ +function relativeToPoint(stops, relative) { + let index = 0 + + while (index < stops.length && stops[index][0] <= relative) { + index += 1 + } + + // There are no points: that only occurs if there was an empty string. + if (index === 0) { + return undefined + } + + const [stopRelative, stopAbsolute] = stops[index - 1] + const rest = relative - stopRelative + return { + line: stopAbsolute.line, + column: stopAbsolute.column + rest, + offset: stopAbsolute.offset + rest + } +} + +// Copy from +// To do: expose that? +/** + * Get the string value of a slice of chunks. + * + * @param {Array} chunks + * @returns {string} + */ +function serializeChunks(chunks) { + let index = -1 + /** @type {Array} */ + const result = [] + /** @type {boolean | undefined} */ + let atTab + + while (++index < chunks.length) { + const chunk = chunks[index] + /** @type {string} */ + let value + + if (typeof chunk === 'string') { + value = chunk + } else + switch (chunk) { + case codes.carriageReturn: { + value = values.cr + + break + } + + case codes.lineFeed: { + value = values.lf + + break + } + + case codes.carriageReturnLineFeed: { + value = values.cr + values.lf + + break + } + + case codes.horizontalTab: { + value = values.ht + + break + } + + /* c8 ignore next 6 */ + case codes.virtualSpace: { + if (atTab) continue + value = values.space + + break + } + + default: { + assert(typeof chunk === 'number', 'expected number') + // Currently only replacement character. + // eslint-disable-next-line unicorn/prefer-code-point + value = String.fromCharCode(chunk) + } + } + + atTab = chunk === codes.horizontalTab + result.push(value) + } + + return result.join('') +} diff --git a/packages/micromark-util-events-to-acorn/package.json b/packages/micromark-util-events-to-acorn/package.json index 222e5e2..28e6de8 100644 --- a/packages/micromark-util-events-to-acorn/package.json +++ b/packages/micromark-util-events-to-acorn/package.json @@ -41,10 +41,11 @@ "dependencies": { "@types/acorn": "^4.0.0", "@types/estree": "^1.0.0", + "@types/unist": "^2.0.0", "estree-util-visit": "^1.0.0", + "micromark-util-symbol": "^1.0.0", "micromark-util-types": "^1.0.0", "uvu": "^0.5.0", - "vfile-location": "^4.0.0", "vfile-message": "^3.0.0" }, "scripts": { diff --git a/test/index.js b/test/index.js index fde6148..f297dbc 100644 --- a/test/index.js +++ b/test/index.js @@ -1,6 +1,8 @@ /** * @typedef {import('acorn').Comment} Comment * @typedef {import('acorn').Token} Token + * @typedef {import('estree').Node} Node + * @typedef {import('estree').Program} Program * @typedef {import('micromark-util-types').CompileContext} CompileContext * @typedef {import('micromark-util-types').Extension} Extension * @typedef {import('micromark-util-types').Handle} Handle @@ -16,10 +18,12 @@ import assert from 'node:assert/strict' import test from 'node:test' import {Parser} from 'acorn' import acornJsx from 'acorn-jsx' +import {visit} from 'estree-util-visit' import {micromark} from 'micromark' -import {codes} from 'micromark-util-symbol/codes.js' import {mdxExpression} from 'micromark-extension-mdx-expression' import {factoryMdxExpression} from 'micromark-factory-mdx-expression' +import {markdownLineEnding} from 'micromark-util-character' +import {codes} from 'micromark-util-symbol/codes.js' const acorn = Parser.extend(acornJsx()) @@ -1145,12 +1149,7 @@ test('should add correct positional info on acorn tokens', function () { ] }) - for (const d of micromarkTokens) { - // @ts-expect-error: we add offsets, as we have them. - delete d.loc?.start.offset - // @ts-expect-error: we add offsets. - delete d.loc?.end.offset - } + removeOffsetsFromTokens(micromarkTokens) assert.deepEqual( JSON.parse(JSON.stringify(micromarkTokens)), @@ -1186,22 +1185,318 @@ test('should add correct positional info on acorn tokens with spread', function ] }) + removeOffsetsFromTokens(micromarkTokens) + // Remove `a`, `=`, `{` acornTokens.splice(0, 3) // Remove `}`. acornTokens.pop() - for (const d of micromarkTokens) { - // @ts-expect-error: we add offsets, as we have them. - delete d.loc?.start.offset - // @ts-expect-error: we add offsets. - delete d.loc?.end.offset + assert.deepEqual( + JSON.parse(JSON.stringify(micromarkTokens)), + JSON.parse(JSON.stringify(acornTokens)) + ) +}) + +test('should use correct positional info when tabs are used', function () { + const micromarkExample = 'ab {`\n\t`}' + const acornExample = 'a = `\n\t` ' + /** @type {Array} */ + const micromarkTokens = [] + /** @type {Array} */ + const acornTokens = [] + /** @type {Program | undefined} */ + let program + + const acornNode = /** @type {Node} */ ( + acorn.parseExpressionAt(acornExample, 0, { + ecmaVersion: 'latest', + onToken: acornTokens, + locations: true, + ranges: true + }) + ) + + micromark(micromarkExample, { + extensions: [ + createExtensionFromFactoryOptions( + acorn, + {ecmaVersion: 'latest', onToken: micromarkTokens}, + true, + false, + false, + true + ) + ], + htmlExtensions: [{enter: {expression}}] + }) + + if (program) removeOffsets(program) + removeOffsetsFromTokens(micromarkTokens) + + // Remove: `a`, `=` + acornTokens.splice(0, 2) + + assert.deepEqual( + JSON.parse(JSON.stringify(micromarkTokens)), + JSON.parse(JSON.stringify(acornTokens)) + ) + + assert(acornNode.type === 'AssignmentExpression') + + assert.deepEqual( + JSON.parse(JSON.stringify(program)), + JSON.parse( + JSON.stringify({ + type: 'Program', + start: 4, + end: 8, + body: [ + { + type: 'ExpressionStatement', + expression: acornNode.right, + start: 4, + end: 8, + loc: {start: {line: 1, column: 4}, end: {line: 2, column: 2}}, + range: [4, 8] + } + ], + sourceType: 'module', + comments: [], + loc: {start: {line: 1, column: 4}, end: {line: 2, column: 2}}, + range: [4, 8] + }) + ) + ) + + /** + * @this {CompileContext} + * @type {Handle} + */ + function expression(token) { + assert('estree' in token) + // @ts-expect-error: fine. + program = token.estree } +}) + +test('should use correct positional when there are virtual spaces due to a block quote', function () { + // Note: we drop the entire tab in this case, even though it represents 3 + // spaces, where the first is eaten by the block quote. + // I believe it would be too complex for users to understand that two spaces + // are passed to acorn and present in template strings. + const micromarkExample = '> ab {`\n>\t`}' + const acornExample = '`\n`' + /** @type {Array} */ + const micromarkTokens = [] + /** @type {Array} */ + const acornTokens = [] + /** @type {Program | undefined} */ + let program + + acorn.parseExpressionAt(acornExample, 0, { + ecmaVersion: 'latest', + onToken: acornTokens, + locations: true, + ranges: true + }) + + micromark(micromarkExample, { + extensions: [ + createExtensionFromFactoryOptions( + acorn, + {ecmaVersion: 'latest', onToken: micromarkTokens}, + true, + false, + false, + true + ) + ], + htmlExtensions: [{enter: {expression}}] + }) + + if (program) removeOffsets(program) + removeOffsetsFromTokens(micromarkTokens) + + assert(acornTokens.length === 3) + // `` ` `` + acornTokens[0].start = 6 + assert(acornTokens[0].loc) + acornTokens[0].loc.start.column = 6 + acornTokens[0].end = 7 + acornTokens[0].loc.end.column = 7 + acornTokens[0].range = [6, 7] + // `template` + acornTokens[1].start = 7 + assert(acornTokens[1].loc) + acornTokens[1].loc.start.column = 7 + acornTokens[1].end = 10 + acornTokens[1].loc.end.column = 2 + acornTokens[1].range = [7, 10] + // `` ` `` + acornTokens[2].start = 10 + assert(acornTokens[2].loc) + acornTokens[2].loc.start.column = 2 + acornTokens[2].end = 11 + acornTokens[2].loc.end.column = 3 + acornTokens[2].range = [10, 11] assert.deepEqual( JSON.parse(JSON.stringify(micromarkTokens)), JSON.parse(JSON.stringify(acornTokens)) ) + + assert.deepEqual( + JSON.parse(JSON.stringify(program)), + JSON.parse( + JSON.stringify({ + type: 'Program', + start: 6, + end: 11, + body: [ + { + type: 'ExpressionStatement', + expression: { + type: 'TemplateLiteral', + start: 6, + end: 11, + expressions: [], + quasis: [ + { + type: 'TemplateElement', + start: 7, + end: 10, + value: {raw: '\n', cooked: '\n'}, + tail: true, + loc: { + start: {line: 1, column: 7}, + end: {line: 2, column: 2} + }, + range: [7, 10] + } + ], + loc: {start: {line: 1, column: 6}, end: {line: 2, column: 3}}, + range: [6, 11] + }, + start: 6, + end: 11, + loc: {start: {line: 1, column: 6}, end: {line: 2, column: 3}}, + range: [6, 11] + } + ], + sourceType: 'module', + comments: [], + loc: {start: {line: 1, column: 6}, end: {line: 2, column: 3}}, + range: [6, 11] + }) + ) + ) + + /** + * @this {CompileContext} + * @type {Handle} + */ + function expression(token) { + assert('estree' in token) + // @ts-expect-error: fine. + program = token.estree + } +}) + +test('should keep the correct number of spaces in a blockquote (flow)', function () { + /** @type {Program | undefined} */ + let program + + micromark('> {`\n> alpha\n> bravo\n> charlie\n> delta\n> `}', { + extensions: [ + createExtensionFromFactoryOptions(acorn, {ecmaVersion: 'latest'}, true) + ], + htmlExtensions: [{enter: {expression}}] + }) + + assert(program) + const statement = program.body[0] + assert(statement.type === 'ExpressionStatement') + assert(statement.expression.type === 'TemplateLiteral') + const quasi = statement.expression.quasis[0] + assert(quasi) + const value = quasi.value.cooked + assert.equal(value, '\nalpha\n bravo\n charlie\n delta\n') + + /** + * @this {CompileContext} + * @type {Handle} + */ + function expression(token) { + assert('estree' in token) + // @ts-expect-error: fine. + program = token.estree + } +}) + +test('should keep the correct number of spaces in a blockquote (text)', function () { + /** @type {Program | undefined} */ + let program + + micromark( + '> alpha {`\n> bravo\n> charlie\n> delta\n> echo\n> `} foxtrot.', + { + extensions: [ + createExtensionFromFactoryOptions(acorn, {ecmaVersion: 'latest'}, true) + ], + htmlExtensions: [{enter: {expression}}] + } + ) + + assert(program) + const statement = program.body[0] + assert(statement.type === 'ExpressionStatement') + assert(statement.expression.type === 'TemplateLiteral') + const quasi = statement.expression.quasis[0] + assert(quasi) + const value = quasi.value.cooked + assert.equal(value, '\nbravo\n charlie\n delta\n echo\n') + + /** + * @this {CompileContext} + * @type {Handle} + */ + function expression(token) { + assert('estree' in token) + // @ts-expect-error: fine. + program = token.estree + } +}) + +test('should support `\\0` and `\\r` in expressions', function () { + /** @type {Program | undefined} */ + let program + + micromark('{`a\0b\rc\nd\r\ne`}', { + extensions: [ + createExtensionFromFactoryOptions(acorn, {ecmaVersion: 'latest'}, true) + ], + htmlExtensions: [{enter: {expression}}] + }) + + assert(program) + const statement = program.body[0] + assert(statement.type === 'ExpressionStatement') + assert(statement.expression.type === 'TemplateLiteral') + const quasi = statement.expression.quasis[0] + assert(quasi) + const value = quasi.value.cooked + assert.equal(value, 'a�b\nc\nd\ne') + + /** + * @this {CompileContext} + * @type {Handle} + */ + function expression(token) { + assert('estree' in token) + // @ts-expect-error: fine. + program = token.estree + } }) /** @@ -1228,13 +1523,53 @@ function createExtensionFromFactoryOptions( allowEmpty, allowLazy ) { - return {text: {[codes.leftCurlyBrace]: {tokenize}}} + return { + flow: { + [codes.leftCurlyBrace]: {tokenize: tokenizeFlow, concrete: true} + }, + text: {[codes.leftCurlyBrace]: {tokenize: tokenizeText}} + } + + /** + * @this {TokenizeContext} + * @type {Tokenizer} + */ + function tokenizeFlow(effects, ok, nok) { + const self = this + + return start + + /** @type {State} */ + function start(code) { + return factoryMdxExpression.call( + self, + effects, + after, + 'expression', + 'expressionMarker', + 'expressionChunk', + acorn, + acornOptions, + addResult, + spread, + allowEmpty + )(code) + } + + // Note: trailing whitespace not supported. + /** @type {State} */ + function after(code) { + return code === codes.eof || markdownLineEnding(code) + ? ok(code) + : nok(code) + } + } /** * @this {TokenizeContext} * @type {Tokenizer} */ - function tokenize(effects, ok) { + function tokenizeText(effects, ok) { const self = this return start @@ -1258,3 +1593,30 @@ function createExtensionFromFactoryOptions( } } } + +/** + * @param {Node} node + * @returns {void} + */ +function removeOffsets(node) { + visit(node, (d) => { + assert(d.loc, 'expected `loc`') + // @ts-expect-error: we add offsets, as we have them. + delete d.loc.start.offset + // @ts-expect-error: we add offsets. + delete d.loc.end.offset + }) +} + +/** + * @param {Array} tokens + * @returns {void} + */ +function removeOffsetsFromTokens(tokens) { + for (const d of tokens) { + // @ts-expect-error: we add offsets, as we have them. + delete d.loc?.start.offset + // @ts-expect-error: we add offsets. + delete d.loc?.end.offset + } +}