Fix non-ASCII whitespace in inline code

Closes GH-321. Closes GH-424. Reviewed-by: Marouane Fazouane <fazouanem3@gmail.com> Reviewed-by: Christian Murphy <christian.murphy.42@gmail.com>
remarkjs · Jul 15, 2019 · 475c72b · 475c72b
1 parent f89fd77
commit 475c72b
Show file tree

Hide file tree

Showing 4 changed files with 175 additions and 1 deletion.
diff --git a/packages/remark-parse/lib/tokenize/code-inline.js b/packages/remark-parse/lib/tokenize/code-inline.js
@@ -1,7 +1,7 @@
 'use strict'
 
-var whitespace = require('is-whitespace-character')
 var locate = require('../locate/code-inline')
+var whitespace = require('../util/is-markdown-whitespace-character')
 
 module.exports = inlineCode
 inlineCode.locator = locate

diff --git a/packages/remark-parse/lib/util/is-markdown-whitespace-character.js b/packages/remark-parse/lib/util/is-markdown-whitespace-character.js
@@ -0,0 +1,27 @@
+'use strict'
+
+module.exports = whitespace
+
+var tab = 9 // '\t'
+var lineFeed = 10 // '\n'
+var lineTabulation = 11 // '\v'
+var formFeed = 12 // '\f'
+var carriageReturn = 13 // '\r'
+var space = 32 // ' '
+
+function whitespace(char) {
+  /* istanbul ignore next - `number` handling for future */
+  var code = typeof char === 'number' ? char : char.charCodeAt(0)
+
+  switch (code) {
+    case tab:
+    case lineFeed:
+    case lineTabulation:
+    case formFeed:
+    case carriageReturn:
+    case space:
+      return true
+    default:
+      return false
+  }
+}
diff --git a/test/fixtures/input/code-inline-whitespace.text b/test/fixtures/input/code-inline-whitespace.text
@@ -0,0 +1,3 @@
+Markdown strips regular whitespaces: `   hello   `
+Including tabs: `	hello	`
+However it preserves non-breaking whitespaces: `   hello   `
diff --git a/test/fixtures/tree/code-inline-whitespace.json b/test/fixtures/tree/code-inline-whitespace.json
@@ -0,0 +1,144 @@
+{
+  "type": "root",
+  "children": [
+    {
+      "type": "paragraph",
+      "children": [
+        {
+          "type": "text",
+          "value": "Markdown strips regular whitespaces: ",
+          "position": {
+            "start": {
+              "line": 1,
+              "column": 1,
+              "offset": 0
+            },
+            "end": {
+              "line": 1,
+              "column": 38,
+              "offset": 37
+            },
+            "indent": []
+          }
+        },
+        {
+          "type": "inlineCode",
+          "value": "hello",
+          "position": {
+            "start": {
+              "line": 1,
+              "column": 38,
+              "offset": 37
+            },
+            "end": {
+              "line": 1,
+              "column": 51,
+              "offset": 50
+            },
+            "indent": []
+          }
+        },
+        {
+          "type": "text",
+          "value": "\nIncluding tabs: ",
+          "position": {
+            "start": {
+              "line": 1,
+              "column": 51,
+              "offset": 50
+            },
+            "end": {
+              "line": 2,
+              "column": 17,
+              "offset": 67
+            },
+            "indent": [
+              1
+            ]
+          }
+        },
+        {
+          "type": "inlineCode",
+          "value": "hello",
+          "position": {
+            "start": {
+              "line": 2,
+              "column": 17,
+              "offset": 67
+            },
+            "end": {
+              "line": 2,
+              "column": 26,
+              "offset": 76
+            },
+            "indent": []
+          }
+        },
+        {
+          "type": "text",
+          "value": "\nHowever it preserves non-breaking whitespaces: ",
+          "position": {
+            "start": {
+              "line": 2,
+              "column": 26,
+              "offset": 76
+            },
+            "end": {
+              "line": 3,
+              "column": 48,
+              "offset": 124
+            },
+            "indent": [
+              1
+            ]
+          }
+        },
+        {
+          "type": "inlineCode",
+          "value": "   hello   ",
+          "position": {
+            "start": {
+              "line": 3,
+              "column": 48,
+              "offset": 124
+            },
+            "end": {
+              "line": 3,
+              "column": 61,
+              "offset": 137
+            },
+            "indent": []
+          }
+        }
+      ],
+      "position": {
+        "start": {
+          "line": 1,
+          "column": 1,
+          "offset": 0
+        },
+        "end": {
+          "line": 3,
+          "column": 61,
+          "offset": 137
+        },
+        "indent": [
+          1,
+          1
+        ]
+      }
+    }
+  ],
+  "position": {
+    "start": {
+      "line": 1,
+      "column": 1,
+      "offset": 0
+    },
+    "end": {
+      "line": 4,
+      "column": 1,
+      "offset": 138
+    }
+  }
+}