Skip to content

Commit

Permalink
Add "ignore" directive, skip tokenizing inside section
Browse files Browse the repository at this point in the history 
This adds directive to not even tokenize a section.
Effectively treats the text inside as a comment.

Closes #384
  • Loading branch information
@bitwiseman
bitwiseman committed Jun 15, 2015
1 parent 5f8fd65 commit 6163246
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 11 deletions.
14 changes: 11 additions & 3 deletions js/lib/beautify.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1541,6 +1541,7 @@
var comment_pattern = /([^\n\r\u2028\u2029]*)/g;

var directives_pattern = /\/\*\sbeautify\s(\w+[:]\w+)+\s\*\//g;
var directives_end_ignore_pattern = /([\s\S]*?)((?:\/\*\sbeautify\signore:end\s\*\/)|$)/g;

var n_newlines, whitespace_before_token, in_html_comment, tokens, parser_pos;
var input_length;
Expand Down Expand Up @@ -1749,16 +1750,23 @@
block_comment_pattern.lastIndex = parser_pos;
var comment_match = block_comment_pattern.exec(input);
comment = '/*' + comment_match[0];
parser_pos += comment.length - 2;
return [comment, 'TK_BLOCK_COMMENT', get_directives(comment)];
parser_pos += comment_match[0].length;
var directives = get_directives(comment);
if (directives && directives['ignore'] === 'start') {
directives_end_ignore_pattern.lastIndex = parser_pos;
comment_match = directives_end_ignore_pattern.exec(input)
comment += comment_match[0];
parser_pos += comment_match[0].length;
}
return [comment, 'TK_BLOCK_COMMENT', directives];
}
// peek for comment // ...
if (input.charAt(parser_pos) === '/') {
parser_pos += 1;
comment_pattern.lastIndex = parser_pos;
var comment_match = comment_pattern.exec(input);
comment = '//' + comment_match[0];
parser_pos += comment.length - 2;
parser_pos += comment_match[0].length;
return [comment, 'TK_COMMENT'];
}

Expand Down
38 changes: 37 additions & 1 deletion js/test/beautify-javascript-tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ function run_javascript_tests(test_obj, Urlencoded, js_beautify, html_beautify,
bt(
'var a = {\n' +
' /* beautify preserve:start */\n' +
' one : 1,\n' +
' one : 1\n' +
' two : 2,\n' +
' three : 3,\n' +
' ten : 10\n' +
Expand All @@ -678,6 +678,42 @@ function run_javascript_tests(test_obj, Urlencoded, js_beautify, html_beautify,
' ten : 10\n' +
'/* beautify preserve:end */\n' +
'};');
bt('/* beautify ignore:start */\n/* beautify ignore:end */');
bt('/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */');
bt('var a = 1;\n/* beautify ignore:start */\n var a = 1;\n/* beautify ignore:end */');
bt('/* beautify ignore:start */ {asdklgh;y;+++;dd2d}/* beautify ignore:end */');
bt(
'var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */',
'var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */');
bt(
'var a = 1;\n /* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */',
'var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */');
bt(
'var a = {\n' +
' /* beautify ignore:start */\n' +
' one : 1\n' +
' two : 2,\n' +
' three : {\n' +
' ten : 10\n' +
' /* beautify ignore:end */\n' +
'};');
bt(
'var a = {\n' +
'/* beautify ignore:start */\n' +
' one : 1\n' +
' two : 2,\n' +
' three : {\n' +
' ten : 10\n' +
'/* beautify ignore:end */\n' +
'};',
'var a = {\n' +
' /* beautify ignore:start */\n' +
' one : 1\n' +
' two : 2,\n' +
' three : {\n' +
' ten : 10\n' +
'/* beautify ignore:end */\n' +
'};');


// jslint and space after anon function - (f = " ", c = "")
Expand Down
17 changes: 12 additions & 5 deletions python/jsbeautifier/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1105,14 +1105,14 @@ def handle_operator(self, current_token):
def handle_block_comment(self, current_token):
if self.output.raw:
self.output.add_raw_token(current_token)
if current_token.directives and current_token.directives['preserve'] == 'end':
if current_token.directives and current_token.directives.get('preserve') == 'end':
self.output.raw = False
return

if current_token.directives:
self.print_newline(preserve_statement_flags = True)
self.print_token(current_token)
if current_token.directives['preserve'] == 'start':
if current_token.directives.get('preserve') == 'start':
self.output.raw = True

self.print_newline(preserve_statement_flags = True)
Expand Down Expand Up @@ -1394,6 +1394,7 @@ def __init__ (self, input, opts, indent_string):
self.comment_pattern = re.compile(self.acorn.six.u('([^\n\r\u2028\u2029]*)'))

self.directives_pattern = re.compile('\/\*\sbeautify\s(\w+[:]\w+)+\s\*\/')
self.directives_end_ignore_pattern = re.compile('([\s\S]*?)((?:\/\*\sbeautify\signore:end\s\*\/)|$)')


def tokenize(self):
Expand Down Expand Up @@ -1565,14 +1566,20 @@ def __tokenize_next(self):
self.parser_pos += 1
comment_match = self.block_comment_pattern.match(self.input, self.parser_pos)
comment = '/*' + comment_match.group(0)
self.parser_pos += len(comment) - 2;
return comment, 'TK_BLOCK_COMMENT', self.get_directives(comment)
self.parser_pos += len(comment_match.group(0))
directives = self.get_directives(comment)
if directives and directives.get('ignore') == 'start':
comment_match = self.directives_end_ignore_pattern.match(self.input, self.parser_pos)
comment += comment_match.group(0)
self.parser_pos += len(comment_match.group(0))

return comment, 'TK_BLOCK_COMMENT', directives

if self.input[self.parser_pos] == '/': # peek // comment
self.parser_pos += 1
comment_match = self.comment_pattern.match(self.input, self.parser_pos)
comment = '//' + comment_match.group(0)
self.parser_pos += len(comment) - 2;
self.parser_pos += len(comment_match.group(0));
return comment, 'TK_COMMENT'

if c == '`' or c == "'" or c == '"' or \
Expand Down
38 changes: 37 additions & 1 deletion python/jsbeautifier/tests/testjsbeautifier.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ def unicode_char(value):
bt(
'var a = {\n' +
' /* beautify preserve:start */\n' +
' one : 1,\n' +
' one : 1\n' +
' two : 2,\n' +
' three : 3,\n' +
' ten : 10\n' +
Expand All @@ -462,6 +462,42 @@ def unicode_char(value):
' ten : 10\n' +
'/* beautify preserve:end */\n' +
'};')
bt('/* beautify ignore:start */\n/* beautify ignore:end */')
bt('/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */')
bt('var a = 1;\n/* beautify ignore:start */\n var a = 1;\n/* beautify ignore:end */')
bt('/* beautify ignore:start */ {asdklgh;y;+++;dd2d}/* beautify ignore:end */')
bt(
'var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */',
'var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */')
bt(
'var a = 1;\n /* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */',
'var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */')
bt(
'var a = {\n' +
' /* beautify ignore:start */\n' +
' one : 1\n' +
' two : 2,\n' +
' three : {\n' +
' ten : 10\n' +
' /* beautify ignore:end */\n' +
'};')
bt(
'var a = {\n' +
'/* beautify ignore:start */\n' +
' one : 1\n' +
' two : 2,\n' +
' three : {\n' +
' ten : 10\n' +
'/* beautify ignore:end */\n' +
'};',
'var a = {\n' +
' /* beautify ignore:start */\n' +
' one : 1\n' +
' two : 2,\n' +
' three : {\n' +
' ten : 10\n' +
'/* beautify ignore:end */\n' +
'};')

# jslint and space after anon function - (f = " ", c = "")
self.options.jslint_happy = true
Expand Down
49 changes: 48 additions & 1 deletion test/data/javascript.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,7 +519,7 @@ exports.test_data = {
unchanged: [
'var a = {',
' /* beautify preserve:start */',
' one : 1,',
' one : 1',
' two : 2,',
' three : 3,',
' ten : 10',
Expand Down Expand Up @@ -548,6 +548,53 @@ exports.test_data = {
'/* beautify preserve:end */',
'};'
]
},
// ignore
{ unchanged: "/* beautify ignore:start */\n/* beautify ignore:end */" },
{ unchanged: "/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */" },
{ unchanged: "var a = 1;\n/* beautify ignore:start */\n var a = 1;\n/* beautify ignore:end */" },
{ unchanged: "/* beautify ignore:start */ {asdklgh;y;+++;dd2d}/* beautify ignore:end */" },
{
input_: "var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */",
output: "var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */"
},
{
input_: "var a = 1;\n /* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */",
output: "var a = 1;\n/* beautify ignore:start */\n var a,,,{ 1;\n/* beautify ignore:end */"
},
{
unchanged: [
'var a = {',
' /* beautify ignore:start */',
' one : 1',
' two : 2,',
' three : {',
' ten : 10',
' /* beautify ignore:end */',
'};'
]
},
{
input: [
'var a = {',
'/* beautify ignore:start */',
' one : 1',
' two : 2,',
' three : {',
' ten : 10',
'/* beautify ignore:end */',
'};'
],
output: [
'var a = {',
' /* beautify ignore:start */',
' one : 1',
' two : 2,',
' three : {',
' ten : 10',
'/* beautify ignore:end */',
'};'
]
}
]
},
Expand Down

0 comments on commit 6163246

Please sign in to comment.