From 6a31359c3150dad37947161367704b3cb9d1f5b8 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Tue, 4 Apr 2017 23:29:22 -0700 Subject: [PATCH 01/10] Add tabbed literate test; modernize Markdown title heading --- test/literate.litcoffee | 3 +- test/literate_tabbed.litcoffee | 164 +++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 test/literate_tabbed.litcoffee diff --git a/test/literate.litcoffee b/test/literate.litcoffee index df40dff858..d3a56c93d4 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -1,5 +1,4 @@ -Literate CoffeeScript Test --------------------------- +# Literate CoffeeScript Test comment comment diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee new file mode 100644 index 0000000000..32fb218077 --- /dev/null +++ b/test/literate_tabbed.litcoffee @@ -0,0 +1,164 @@ +# Tabbed Literate CoffeeScript Test + +comment comment + + test "basic literate CoffeeScript parsing", -> + ok yes + +now with a... + + test "broken up indentation", -> + +... broken up ... + + do -> + +... nested block. + + ok yes + +Code must be separated from text by a blank line. + + test "code blocks must be preceded by a blank line", -> + +The next line is part of the text and will not be executed. + fail() + + ok yes + +Code in `backticks is not parsed` and... + + test "comments in indented blocks work", -> + do -> + do -> + # Regular comment. + + ### + Block comment. + ### + + ok yes + +Regular [Markdown](http://example.com/markdown) features, like links +and unordered lists, are fine: + + * I + + * Am + + * A + + * List + +Spaces work too: + + test "spaced code", -> + ok yes + +--- + + # keep track of whether code blocks are executed or not + executed = false + +

+ + executed = true # should not execute, this is just HTML para, not code! + +

+ + test "should ignore indented sections inside HTML", -> + eq executed, false + +--- + +* A list item with a code block: + + test "basic literate CoffeeScript parsing", -> + ok yes + +--- + +* Lorem ipsum dolor sit amet, consectetuer adipiscing elit. + Aliquam hendrerit mi posuere lectus. Vestibulum enim wisi, + viverra nec, fringilla in, laoreet vitae, risus. + +* Donec sit amet nisl. Aliquam semper ipsum sit amet velit. + Suspendisse id sem consectetuer libero luctus adipiscing. + +--- + +1. This is a list item with two paragraphs. Lorem ipsum dolor + sit amet, consectetuer adipiscing elit. Aliquam hendrerit + mi posuere lectus. + + Vestibulum enim wisi, viverra nec, fringilla in, laoreet + vitae, risus. Donec sit amet nisl. Aliquam semper ipsum + sit amet velit. + +2. Suspendisse id sem consectetuer libero luctus adipiscing. + +--- + +1. This is a list item with two paragraphs. Lorem ipsum dolor + sit amet, consectetuer adipiscing elit. Aliquam hendrerit + mi posuere lectus. + + Vestibulum enim wisi, viverra nec, fringilla in, laoreet + vitae, risus. Donec sit amet nisl. Aliquam semper ipsum + sit amet velit. + +2. Suspendisse id sem consectetuer libero luctus adipiscing. + +--- + +* A list item with a blockquote: + + > This is a blockquote + > inside a list item. + +--- + +This next one probably passes because a string is inoffensive in compiled js, also, can't get `marked` to parse it correctly, and not sure if empty line is permitted between title and reference + +This is [an example][id] reference-style link. +[id]: http://example.com/ + + "Optional Title Here" + +--- + + executed = no + +1986. What a great season. + executed = yes + +and test... + + test "should recognise indented code blocks in lists", -> + ok executed + +--- + + executed = no + +1986. What a great season. + + executed = yes + +and test... + + test "should recognise indented code blocks in lists with empty line as separator", -> + ok executed + +--- + + executed = no + +1986\. What a great season. + executed = yes + +and test... + + test "should ignore indented code in escaped list like number", -> + eq executed, no + From 17c7333ce32c1b0b5f2c70743981751b6a7c083b Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 00:10:35 -0700 Subject: [PATCH 02/10] =?UTF-8?q?Better=20parsing=20of=20Literate=20Coffee?= =?UTF-8?q?Script=20files,=20including=20now=20correct=20parsing=20of=20ta?= =?UTF-8?q?bbed=20.litcoffee=20files;=20and=20more=20accurate=20stack=20tr?= =?UTF-8?q?aces=20(assuming=20you=20don=E2=80=99t=20do=20your=20own=20word?= =?UTF-8?q?=20wrapping=20within=20list=20items)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/coffeescript/helpers.js | 30 +++++++++++++++++------------- src/helpers.coffee | 34 ++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/lib/coffeescript/helpers.js b/lib/coffeescript/helpers.js index 7027a43f99..4d18f26e4b 100644 --- a/lib/coffeescript/helpers.js +++ b/lib/coffeescript/helpers.js @@ -98,23 +98,27 @@ }; exports.invertLiterate = function(code) { - var generateRandomToken, i, item, len1, out, ref1, token; - generateRandomToken = function() { - return `${Math.random() * Date.now()}`; - }; - while (token === void 0 || code.indexOf(token) !== -1) { - token = generateRandomToken(); - } - code = code.replace("\t", token); - out = ""; + var i, len1, out, ref1, section; + code = code.replace(/^(\t)+/gm, function(match) { + var i, ref1, spaces, tab; + spaces = ''; + for (tab = i = 0, ref1 = match.length; 0 <= ref1 ? i <= ref1 : i >= ref1; tab = 0 <= ref1 ? ++i : --i) { + spaces += ' '; + } + return spaces; + }); + out = ''; ref1 = marked.lexer(code, {}); for (i = 0, len1 = ref1.length; i < len1; i++) { - item = ref1[i]; - if (item.type === 'code') { - out += `${item.text}\n`; + section = ref1[i]; + if (section.type === 'code') { + out += section.text + '\n\n'; + } else if (section.text != null) { + out += '# ' + section.text.trim().replace(/\n/g, '\n# ') + '\n\n'; + } else if (section.type === 'hr') { + out += '# ---\n\n'; } } - out.replace(token, "\t"); return out; }; diff --git a/src/helpers.coffee b/src/helpers.coffee index 2c2d823a57..7e1200f7fa 100644 --- a/src/helpers.coffee +++ b/src/helpers.coffee @@ -82,20 +82,26 @@ exports.some = Array::some ? (fn) -> # out all non-code blocks, producing a string of CoffeeScript code that can # be compiled “normally.” exports.invertLiterate = (code) -> - # Create a placeholder for tabs, that isn’t used anywhere in `code`, and then - # re-insert the tabs after code extraction. - generateRandomToken = -> - "#{Math.random() * Date.now()}" - while token is undefined or code.indexOf(token) isnt -1 - token = generateRandomToken() - - code = code.replace "\t", token - # Parse as markdown, discard everything except code blocks. - out = "" - for item in marked.lexer code, {} - out += "#{item.text}\n" if item.type is 'code' - # Put the tabs back in. - out.replace token, "\t" + # `marked` really doesn’t process tabs too well. All this code is headed + # into JavaScript anyway, so replace the tabs with two spaces to make + # `marked`’s job a little easier. + code = code.replace /^(\t)+/gm, (match) -> + spaces = '' + spaces += ' ' for tab in [0..match.length] + spaces + + out = '' + for section in marked.lexer code, {} + if section.type is 'code' + out += section.text + '\n\n' + # Keep the non-code sections (that have text) so as to hopefully + # preserve the correct line numbers. Lists can screw this up. For better + # stack traces, don’t wrap your own list items like our tests do but rather + # let them run free and rely on your editor to wrap your lines for you. + else if section.text? + out += '# ' + section.text.trim().replace(/\n/g, '\n# ') + '\n\n' + else if section.type is 'hr' + out += '# ---\n\n' out # Merge two jison-style location data objects together. From d9e1fd45ee7a0c50fbe1c136ebcbd87030f45d10 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:32:36 -0700 Subject: [PATCH 03/10] =?UTF-8?q?Swap=20Marked=20for=20MarkdownIt=20for=20?= =?UTF-8?q?parsing=20the=20Markdown=20of=20Literate=20CoffeeScript=20files?= =?UTF-8?q?;=20use=20MarkdownIt=E2=80=99s=20`map`=20property=20to=20preser?= =?UTF-8?q?ve=20correct=20line=20numbers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/coffeescript/helpers.js | 59 +++++++++++++++++-------------------- package.json | 2 +- src/helpers.coffee | 44 ++++++++------------------- 3 files changed, 40 insertions(+), 65 deletions(-) diff --git a/lib/coffeescript/helpers.js b/lib/coffeescript/helpers.js index 4d18f26e4b..dcbb5bed37 100644 --- a/lib/coffeescript/helpers.js +++ b/lib/coffeescript/helpers.js @@ -1,8 +1,8 @@ // Generated by CoffeeScript 2.0.0-alpha1 (function() { - var buildLocationData, extend, flatten, marked, ref, repeat, syntaxErrorToString; + var buildLocationData, extend, flatten, md, ref, repeat, syntaxErrorToString; - marked = require('marked'); + md = require('markdown-it')(); exports.starts = function(string, literal, start) { return literal === string.substr(start, literal.length); @@ -28,10 +28,10 @@ }; exports.compact = function(array) { - var i, item, len1, results; + var item, j, len1, results; results = []; - for (i = 0, len1 = array.length; i < len1; i++) { - item = array[i]; + for (j = 0, len1 = array.length; j < len1; j++) { + item = array[j]; if (item) { results.push(item); } @@ -65,10 +65,10 @@ }; exports.flatten = flatten = function(array) { - var element, flattened, i, len1; + var element, flattened, j, len1; flattened = []; - for (i = 0, len1 = array.length; i < len1; i++) { - element = array[i]; + for (j = 0, len1 = array.length; j < len1; j++) { + element = array[j]; if ('[object Array]' === Object.prototype.toString.call(element)) { flattened = flattened.concat(flatten(element)); } else { @@ -86,10 +86,10 @@ }; exports.some = (ref = Array.prototype.some) != null ? ref : function(fn) { - var e, i, len1, ref1; + var e, j, len1, ref1; ref1 = this; - for (i = 0, len1 = ref1.length; i < len1; i++) { - e = ref1[i]; + for (j = 0, len1 = ref1.length; j < len1; j++) { + e = ref1[j]; if (fn(e)) { return true; } @@ -98,28 +98,23 @@ }; exports.invertLiterate = function(code) { - var i, len1, out, ref1, section; - code = code.replace(/^(\t)+/gm, function(match) { - var i, ref1, spaces, tab; - spaces = ''; - for (tab = i = 0, ref1 = match.length; 0 <= ref1 ? i <= ref1 : i >= ref1; tab = 0 <= ref1 ? ++i : --i) { - spaces += ' '; + var out; + out = []; + md.renderer.rules = { + code_block: function(tokens, idx) { + var i, j, len1, line, lines, results, startLine; + startLine = tokens[idx].map[0]; + lines = tokens[idx].content.split('\n'); + results = []; + for (i = j = 0, len1 = lines.length; j < len1; i = ++j) { + line = lines[i]; + results.push(out[startLine + i] = line); + } + return results; } - return spaces; - }); - out = ''; - ref1 = marked.lexer(code, {}); - for (i = 0, len1 = ref1.length; i < len1; i++) { - section = ref1[i]; - if (section.type === 'code') { - out += section.text + '\n\n'; - } else if (section.text != null) { - out += '# ' + section.text.trim().replace(/\n/g, '\n# ') + '\n\n'; - } else if (section.type === 'hr') { - out += '# ---\n\n'; - } - } - return out; + }; + md.render(code); + return out.join('\n'); }; buildLocationData = function(first, last) { diff --git a/package.json b/package.json index 1c12a0fb88..d6e63245e6 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,6 @@ "underscore": "~1.8.3" }, "dependencies": { - "marked": "~0.3.6" + "markdown-it": "^8.3.1" } } diff --git a/src/helpers.coffee b/src/helpers.coffee index 7e1200f7fa..8485e6f01b 100644 --- a/src/helpers.coffee +++ b/src/helpers.coffee @@ -2,16 +2,7 @@ # the **Lexer**, **Rewriter**, and the **Nodes**. Merge objects, flatten # arrays, count characters, that sort of thing. -marked = require 'marked' -# marked.setOptions -# renderer: new marked.Renderer() -# gfm: true -# tables: true -# breaks: false -# pedantic: false -# sanitize: true -# smartLists: true -# smartypants: false +md = require('markdown-it')() # Peek at the beginning of a given string to see if it matches a sequence. exports.starts = (string, literal, start) -> @@ -80,29 +71,18 @@ exports.some = Array::some ? (fn) -> # Simple function for extracting code from Literate CoffeeScript by stripping # out all non-code blocks, producing a string of CoffeeScript code that can -# be compiled “normally.” +# be compiled “normally.” Uses [MarkdownIt](https://markdown-it.github.io/) +# to tell the difference between Markdown and code blocks. exports.invertLiterate = (code) -> - # `marked` really doesn’t process tabs too well. All this code is headed - # into JavaScript anyway, so replace the tabs with two spaces to make - # `marked`’s job a little easier. - code = code.replace /^(\t)+/gm, (match) -> - spaces = '' - spaces += ' ' for tab in [0..match.length] - spaces - - out = '' - for section in marked.lexer code, {} - if section.type is 'code' - out += section.text + '\n\n' - # Keep the non-code sections (that have text) so as to hopefully - # preserve the correct line numbers. Lists can screw this up. For better - # stack traces, don’t wrap your own list items like our tests do but rather - # let them run free and rely on your editor to wrap your lines for you. - else if section.text? - out += '# ' + section.text.trim().replace(/\n/g, '\n# ') + '\n\n' - else if section.type is 'hr' - out += '# ---\n\n' - out + out = [] + md.renderer.rules = + code_block: (tokens, idx) -> + startLine = tokens[idx].map[0] + lines = tokens[idx].content.split '\n' + for line, i in lines + out[startLine + i] = line + md.render code + out.join '\n' # Merge two jison-style location data objects together. # If `last` is not provided, this will simply return `first`. From 3585d2a7c182659105254632b2d9682a4b4c7d29 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:35:23 -0700 Subject: [PATCH 04/10] Literate CoffeeScript tests: remove trailing whitespace, fix spelling --- test/literate.litcoffee | 37 +++++++++++++++------------------- test/literate_tabbed.litcoffee | 4 ++-- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/test/literate.litcoffee b/test/literate.litcoffee index d3a56c93d4..d36edb6de0 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -4,15 +4,15 @@ comment comment test "basic literate CoffeeScript parsing", -> ok yes - + now with a... - + test "broken up indentation", -> - + ... broken up ... do -> - + ... nested block. ok yes @@ -24,36 +24,31 @@ Code must be separated from text by a blank line. The next line is part of the text and will not be executed. fail() - ok yes - + ok yes + Code in `backticks is not parsed` and... test "comments in indented blocks work", -> do -> do -> # Regular comment. - + ### Block comment. ### - + ok yes - -Regular [Markdown](http://example.com/markdown) features, like links + +Regular [Markdown](http://example.com/markdown) features, like links and unordered lists, are fine: - * I - + * I + * Am - - * A - - * List -Tabs work too: + * A - test "tabbed code", -> - ok yes + * List --- @@ -134,7 +129,7 @@ This is [an example][id] reference-style link. and test... - test "should recognise indented code blocks in lists", -> + test "should recognize indented code blocks in lists", -> ok executed --- @@ -147,7 +142,7 @@ and test... and test... - test "should recognise indented code blocks in lists with empty line as separator", -> + test "should recognize indented code blocks in lists with empty line as separator", -> ok executed --- diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee index 32fb218077..66b8931540 100644 --- a/test/literate_tabbed.litcoffee +++ b/test/literate_tabbed.litcoffee @@ -134,7 +134,7 @@ This is [an example][id] reference-style link. and test... - test "should recognise indented code blocks in lists", -> + test "should recognize indented code blocks in lists", -> ok executed --- @@ -147,7 +147,7 @@ and test... and test... - test "should recognise indented code blocks in lists with empty line as separator", -> + test "should recognize indented code blocks in lists with empty line as separator", -> ok executed --- From c166486232f9a87ef2adb3cbd6f900e290192d9d Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:35:52 -0700 Subject: [PATCH 05/10] Literate CoffeeScript tests: add block quote test --- test/literate.litcoffee | 8 ++++++++ test/literate_tabbed.litcoffee | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/test/literate.litcoffee b/test/literate.litcoffee index d36edb6de0..e23300df0d 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -157,3 +157,11 @@ and test... test "should ignore indented code in escaped list like number", -> eq executed, no +one last test! + + test "block quotes should render correctly", -> + quote = ''' + foo + and bar! + ''' + eq quote, 'foo\n and bar!' diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee index 66b8931540..1777733b56 100644 --- a/test/literate_tabbed.litcoffee +++ b/test/literate_tabbed.litcoffee @@ -162,3 +162,11 @@ and test... test "should ignore indented code in escaped list like number", -> eq executed, no +one last test! + + test "block quotes should render correctly", -> + quote = ''' + foo + and bar! + ''' + eq quote, 'foo\n\t\tand bar!' From a3d6f27c98530571eb2cfce5b9f807c341c7e042 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:36:18 -0700 Subject: [PATCH 06/10] Literate CoffeeScript (tabbed, at least) seems to need a consistent starting indentation --- test/literate_tabbed.litcoffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee index 1777733b56..ecb641e280 100644 --- a/test/literate_tabbed.litcoffee +++ b/test/literate_tabbed.litcoffee @@ -143,7 +143,7 @@ and test... 1986. What a great season. - executed = yes + executed = yes and test... From 5826ca90d517ff70998fc7895c3ea287caf3b199 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:40:09 -0700 Subject: [PATCH 07/10] Restore test --- test/literate.litcoffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/literate.litcoffee b/test/literate.litcoffee index e23300df0d..dd8eae8f13 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -50,6 +50,11 @@ and unordered lists, are fine: * List +Tabs work too: + + test "tabbed code", -> + ok yes + --- # keep track of whether code blocks are executed or not From b401f840fb9c80ca0b11b702f68e7ca2294834f7 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:43:31 -0700 Subject: [PATCH 08/10] Reference links work now in MarkdownIt --- test/literate.litcoffee | 6 +----- test/literate_tabbed.litcoffee | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/test/literate.litcoffee b/test/literate.litcoffee index dd8eae8f13..4eb1831498 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -118,12 +118,8 @@ Tabs work too: --- -This next one probably passes because a string is inoffensive in compiled js, also, can't get `marked` to parse it correctly, and not sure if empty line is permitted between title and reference - This is [an example][id] reference-style link. -[id]: http://example.com/ - - "Optional Title Here" +[id]: http://example.com/ "Optional Title Here" --- diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee index ecb641e280..246e6a2f40 100644 --- a/test/literate_tabbed.litcoffee +++ b/test/literate_tabbed.litcoffee @@ -118,12 +118,8 @@ Spaces work too: --- -This next one probably passes because a string is inoffensive in compiled js, also, can't get `marked` to parse it correctly, and not sure if empty line is permitted between title and reference - This is [an example][id] reference-style link. -[id]: http://example.com/ - - "Optional Title Here" +[id]: http://example.com/ "Optional Title Here" --- From b8eed42cc9a21036f6c6a17ee3fa9e2b91937642 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:48:41 -0700 Subject: [PATCH 09/10] Breaking change in Literate CoffeeScript: code blocks within HTML tags must be unindented --- test/literate.litcoffee | 5 +++-- test/literate_tabbed.litcoffee | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/literate.litcoffee b/test/literate.litcoffee index 4eb1831498..5b0de9d7d0 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -62,11 +62,12 @@ Tabs work too:

- executed = true # should not execute, this is just HTML para, not code! +if true + executed = true # should not execute, this is just HTML para, not code!

- test "should ignore indented sections inside HTML", -> + test "should ignore code blocks inside HTML", -> eq executed, false --- diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee index 246e6a2f40..a460a829f5 100644 --- a/test/literate_tabbed.litcoffee +++ b/test/literate_tabbed.litcoffee @@ -62,11 +62,12 @@ Spaces work too:

+if true executed = true # should not execute, this is just HTML para, not code!

- test "should ignore indented sections inside HTML", -> + test "should ignore code blocks inside HTML", -> eq executed, false --- From 770543870c074883eff2a821ec26db9a7ce045e9 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Wed, 5 Apr 2017 19:50:46 -0700 Subject: [PATCH 10/10] Breaking change in Literate CoffeeScript: code blocks within lists need a blank line separating them from the list item text --- test/literate.litcoffee | 12 ------------ test/literate_tabbed.litcoffee | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/test/literate.litcoffee b/test/literate.litcoffee index 5b0de9d7d0..5d1f3c5fdd 100644 --- a/test/literate.litcoffee +++ b/test/literate.litcoffee @@ -122,18 +122,6 @@ if true This is [an example][id] reference-style link. [id]: http://example.com/ "Optional Title Here" ---- - - executed = no - -1986. What a great season. - executed = yes - -and test... - - test "should recognize indented code blocks in lists", -> - ok executed - --- executed = no diff --git a/test/literate_tabbed.litcoffee b/test/literate_tabbed.litcoffee index a460a829f5..b3d73e4454 100644 --- a/test/literate_tabbed.litcoffee +++ b/test/literate_tabbed.litcoffee @@ -122,18 +122,6 @@ if true This is [an example][id] reference-style link. [id]: http://example.com/ "Optional Title Here" ---- - - executed = no - -1986. What a great season. - executed = yes - -and test... - - test "should recognize indented code blocks in lists", -> - ok executed - --- executed = no