Merge pull request #548 from magnusjt/master

Make trimBlocks also trim windows style line endings
mozilla · Oct 7, 2015 · 90e3913 · 90e3913
2 parents 30691f8 + b914fc5
commit 90e3913
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,8 @@ master (unreleased)
   [#537](https://github.com/mozilla/nunjucks/pull/537)
 * Fix error propagation. Thanks Tom Delmas. Merge of
   [#534](https://github.com/mozilla/nunjucks/pull/534).
+* trimBlocks now also trims windows style line endings. Thanks Magnus Tovslid. Merge of
+  [#548](https://github.com/mozilla/nunjucks/pull/548)
 
 
 v2.1.0 (Sep 21 2015)

diff --git a/src/lexer.js b/src/lexer.js
@@ -109,6 +109,16 @@ Tokenizer.prototype.nextToken = function() {
                 if(cur === '\n') {
                     // Skip newline
                     this.forward();
+                }else if(cur === '\r'){
+                    // Skip CRLF newline
+                    this.forward();
+                    cur = this.current();
+                    if(cur === '\n'){
+                        this.forward();
+                    }else{
+                        // Was not a CRLF, so go back
+                        this.back();
+                    }
                 }
             }
             return token(TOKEN_BLOCK_END, tok, lineno, colno);

diff --git a/tests/lexer.js b/tests/lexer.js
@@ -87,6 +87,35 @@
                       lexer.TOKEN_BLOCK_END);
         });
 
+        it('should trim windows-style CRLF line endings after blocks', function () {
+            tokens = lexer.lex('  {% if true %}\r\n    foo\r\n  {% endif %}\r\n', {trimBlocks: true});
+            hasTokens(tokens,
+                [lexer.TOKEN_DATA, '  '],
+                lexer.TOKEN_BLOCK_START,
+                lexer.TOKEN_SYMBOL,
+                lexer.TOKEN_BOOLEAN,
+                lexer.TOKEN_BLOCK_END,
+                [lexer.TOKEN_DATA, '    foo\r\n  '],
+                lexer.TOKEN_BLOCK_START,
+                lexer.TOKEN_SYMBOL,
+                lexer.TOKEN_BLOCK_END);
+        });
+
+        it('should not trim CR after blocks', function () {
+            tokens = lexer.lex('  {% if true %}\r    foo\r\n  {% endif %}\r', {trimBlocks: true});
+            hasTokens(tokens,
+                [lexer.TOKEN_DATA, '  '],
+                lexer.TOKEN_BLOCK_START,
+                lexer.TOKEN_SYMBOL,
+                lexer.TOKEN_BOOLEAN,
+                lexer.TOKEN_BLOCK_END,
+                [lexer.TOKEN_DATA, '\r    foo\r\n  '],
+                lexer.TOKEN_BLOCK_START,
+                lexer.TOKEN_SYMBOL,
+                lexer.TOKEN_BLOCK_END,
+                [lexer.TOKEN_DATA, '\r']);
+        });
+
         it('should lstrip and trim blocks', function () {
             tokens = lexer.lex('test\n {% if true %}\n  foo\n {% endif %}\n</div>', {
                 lstripBlocks: true,