From 76bf1f95083777d25f9f3c5ad8c9e0f0ad21a7dd Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Mon, 15 Jul 2019 12:19:27 +0200 Subject: [PATCH 1/2] Fix non-ASCII whitespace in inline code Closes GH-321. --- .../remark-parse/lib/tokenize/code-inline.js | 2 +- .../util/is-markdown-whitespace-character.js | 26 +++++ .../input/code-inline-whitespace.text | 2 + .../fixtures/tree/code-inline-whitespace.json | 107 ++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 packages/remark-parse/lib/util/is-markdown-whitespace-character.js create mode 100644 test/fixtures/input/code-inline-whitespace.text create mode 100644 test/fixtures/tree/code-inline-whitespace.json diff --git a/packages/remark-parse/lib/tokenize/code-inline.js b/packages/remark-parse/lib/tokenize/code-inline.js index eef27e65e..044e06c10 100644 --- a/packages/remark-parse/lib/tokenize/code-inline.js +++ b/packages/remark-parse/lib/tokenize/code-inline.js @@ -1,7 +1,7 @@ 'use strict' -var whitespace = require('is-whitespace-character') var locate = require('../locate/code-inline') +var whitespace = require('../util/is-markdown-whitespace-character') module.exports = inlineCode inlineCode.locator = locate diff --git a/packages/remark-parse/lib/util/is-markdown-whitespace-character.js b/packages/remark-parse/lib/util/is-markdown-whitespace-character.js new file mode 100644 index 000000000..2cec57406 --- /dev/null +++ b/packages/remark-parse/lib/util/is-markdown-whitespace-character.js @@ -0,0 +1,26 @@ +'use strict' + +module.exports = whitespace + +var tab = 9 // '\t' +var lineFeed = 10 // '\n' +var lineTabulation = 11 // '\v' +var formFeed = 12 // '\f' +var carriageReturn = 13 // '\r' +var space = 32 // ' ' + +function whitespace(char) { + var code = typeof char === 'number' ? char : char.charCodeAt(0) + + switch (code) { + case tab: + case lineFeed: + case lineTabulation: + case formFeed: + case carriageReturn: + case space: + return true + default: + return false + } +} diff --git a/test/fixtures/input/code-inline-whitespace.text b/test/fixtures/input/code-inline-whitespace.text new file mode 100644 index 000000000..67b693153 --- /dev/null +++ b/test/fixtures/input/code-inline-whitespace.text @@ -0,0 +1,2 @@ +Markdown strips regular whitespaces: ` hello ` +However it preserves non-breaking whitespaces: `   hello   ` diff --git a/test/fixtures/tree/code-inline-whitespace.json b/test/fixtures/tree/code-inline-whitespace.json new file mode 100644 index 000000000..37c33fa44 --- /dev/null +++ b/test/fixtures/tree/code-inline-whitespace.json @@ -0,0 +1,107 @@ +{ + "type": "root", + "children": [ + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "Markdown strips regular whitespaces: ", + "position": { + "start": { + "line": 1, + "column": 1, + "offset": 0 + }, + "end": { + "line": 1, + "column": 38, + "offset": 37 + }, + "indent": [] + } + }, + { + "type": "inlineCode", + "value": "hello", + "position": { + "start": { + "line": 1, + "column": 38, + "offset": 37 + }, + "end": { + "line": 1, + "column": 51, + "offset": 50 + }, + "indent": [] + } + }, + { + "type": "text", + "value": "\nHowever it preserves non-breaking whitespaces: ", + "position": { + "start": { + "line": 1, + "column": 51, + "offset": 50 + }, + "end": { + "line": 2, + "column": 48, + "offset": 98 + }, + "indent": [ + 1 + ] + } + }, + { + "type": "inlineCode", + "value": "   hello   ", + "position": { + "start": { + "line": 2, + "column": 48, + "offset": 98 + }, + "end": { + "line": 2, + "column": 61, + "offset": 111 + }, + "indent": [] + } + } + ], + "position": { + "start": { + "line": 1, + "column": 1, + "offset": 0 + }, + "end": { + "line": 2, + "column": 61, + "offset": 111 + }, + "indent": [ + 1 + ] + } + } + ], + "position": { + "start": { + "line": 1, + "column": 1, + "offset": 0 + }, + "end": { + "line": 3, + "column": 1, + "offset": 112 + } + } +} From 6e28f1f80b4b89862df2320b45d03b81e71a7569 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Mon, 15 Jul 2019 12:26:15 +0200 Subject: [PATCH 2/2] Fix coverage --- .../util/is-markdown-whitespace-character.js | 1 + .../input/code-inline-whitespace.text | 1 + .../fixtures/tree/code-inline-whitespace.json | 57 +++++++++++++++---- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/packages/remark-parse/lib/util/is-markdown-whitespace-character.js b/packages/remark-parse/lib/util/is-markdown-whitespace-character.js index 2cec57406..ff6f4bbd4 100644 --- a/packages/remark-parse/lib/util/is-markdown-whitespace-character.js +++ b/packages/remark-parse/lib/util/is-markdown-whitespace-character.js @@ -10,6 +10,7 @@ var carriageReturn = 13 // '\r' var space = 32 // ' ' function whitespace(char) { + /* istanbul ignore next - `number` handling for future */ var code = typeof char === 'number' ? char : char.charCodeAt(0) switch (code) { diff --git a/test/fixtures/input/code-inline-whitespace.text b/test/fixtures/input/code-inline-whitespace.text index 67b693153..675bfd85c 100644 --- a/test/fixtures/input/code-inline-whitespace.text +++ b/test/fixtures/input/code-inline-whitespace.text @@ -1,2 +1,3 @@ Markdown strips regular whitespaces: ` hello ` +Including tabs: ` hello ` However it preserves non-breaking whitespaces: `   hello   ` diff --git a/test/fixtures/tree/code-inline-whitespace.json b/test/fixtures/tree/code-inline-whitespace.json index 37c33fa44..e2a74787e 100644 --- a/test/fixtures/tree/code-inline-whitespace.json +++ b/test/fixtures/tree/code-inline-whitespace.json @@ -40,7 +40,7 @@ }, { "type": "text", - "value": "\nHowever it preserves non-breaking whitespaces: ", + "value": "\nIncluding tabs: ", "position": { "start": { "line": 1, @@ -49,8 +49,44 @@ }, "end": { "line": 2, + "column": 17, + "offset": 67 + }, + "indent": [ + 1 + ] + } + }, + { + "type": "inlineCode", + "value": "hello", + "position": { + "start": { + "line": 2, + "column": 17, + "offset": 67 + }, + "end": { + "line": 2, + "column": 26, + "offset": 76 + }, + "indent": [] + } + }, + { + "type": "text", + "value": "\nHowever it preserves non-breaking whitespaces: ", + "position": { + "start": { + "line": 2, + "column": 26, + "offset": 76 + }, + "end": { + "line": 3, "column": 48, - "offset": 98 + "offset": 124 }, "indent": [ 1 @@ -62,14 +98,14 @@ "value": "   hello   ", "position": { "start": { - "line": 2, + "line": 3, "column": 48, - "offset": 98 + "offset": 124 }, "end": { - "line": 2, + "line": 3, "column": 61, - "offset": 111 + "offset": 137 }, "indent": [] } @@ -82,11 +118,12 @@ "offset": 0 }, "end": { - "line": 2, + "line": 3, "column": 61, - "offset": 111 + "offset": 137 }, "indent": [ + 1, 1 ] } @@ -99,9 +136,9 @@ "offset": 0 }, "end": { - "line": 3, + "line": 4, "column": 1, - "offset": 112 + "offset": 138 } } }