Add unicode support for ID tokens

Fixes handlebars-lang#433 Fixes handlebars-lang#469
alejandroiglesias · May 27, 2013 · c1020a0 · c1020a0
1 parent bff71d7
commit c1020a0
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 3 deletions.
diff --git a/dist/handlebars.js b/dist/handlebars.js
@@ -644,7 +644,7 @@ case 33: return 5;
 break;
 }
 };
-lexer.rules = [/^(?:\\\\(?=(\{\{)))/,/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:[\s\S]*?--\}\})/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{!--)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[}/ ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:-?[0-9]+(?=[}\s]))/,/^(?:[a-zA-Z0-9_$:\-]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:\s+)/,/^(?:[a-zA-Z0-9_$\-\/\.]+)/,/^(?:$)/];
+lexer.rules = [/^(?:\\\\(?=(\{\{)))/,/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:[\s\S]*?--\}\})/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{!--)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[}\/ ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:-?[0-9]+(?=[}\s]))/,/^(?:[^\s!"#%-,\.\/;->@\[-\^`\{-~]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:\s+)/,/^(?:[a-zA-Z0-9_$\-\/\.]+)/,/^(?:$)/];
 lexer.conditions = {"mu":{"rules":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,33],"inclusive":false},"emu":{"rules":[3],"inclusive":false},"com":{"rules":[4],"inclusive":false},"par":{"rules":[31,32],"inclusive":false},"INITIAL":{"rules":[0,1,2,33],"inclusive":true}};
 return lexer;})()
 parser.lexer = lexer;

diff --git a/release-notes.md b/release-notes.md
@@ -4,6 +4,8 @@
 - [#515](https://github.com/wycats/handlebars.js/issues/515) - Add node require extensions support ([@jjclark1982](https://github.com/jjclark1982))
 - [#517](https://github.com/wycats/handlebars.js/issues/517) - Fix amd precompiler output with directories ([@blessenm](https://github.com/blessenm))
 - [#519](https://github.com/wycats/handlebars.js/issues/519) - Fix partials with . name ([@jamesgorrie](https://github.com/jamesgorrie))
+- [#433](https://github.com/wycats/handlebars.js/issues/433) - Add support for unicode ids
+- [#469](https://github.com/wycats/handlebars.js/issues/469) - Add support for `?` in ids
 - Docs updates
 
 [Commits](https://github.com/wycats/handlebars.js/compare/v1.0.11...master)

diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb
@@ -125,6 +125,10 @@ def path(*parts)
 
   it "parses simple mustaches" do
     ast_for("{{foo}}").should == root { mustache id("foo") }
+    ast_for("{{foo?}}").should == root { mustache id("foo?") }
+    ast_for("{{foo_}}").should == root { mustache id("foo_") }
+    ast_for("{{foo-}}").should == root { mustache id("foo-") }
+    ast_for("{{foo:}}").should == root { mustache id("foo:") }
   end
 
   it "parses simple mustaches with data" do

diff --git a/src/handlebars.l b/src/handlebars.l
@@ -32,7 +32,7 @@
 <mu>"{{"                         { return 'OPEN'; }
 
 <mu>"="                          { return 'EQUALS'; }
-<mu>"."/[}/ ]                    { return 'ID'; }
+<mu>"."/[}\/ ]                   { return 'ID'; }
 <mu>".."                         { return 'ID'; }
 <mu>[\/.]                        { return 'SEP'; }
 <mu>\s+                          { /*ignore whitespace*/ }
@@ -44,7 +44,18 @@
 <mu>"true"/[}\s]                 { return 'BOOLEAN'; }
 <mu>"false"/[}\s]                { return 'BOOLEAN'; }
 <mu>\-?[0-9]+/[}\s]              { return 'INTEGER'; }
-<mu>[a-zA-Z0-9_$:\-]+/[=}\s\/.]  { return 'ID'; }
+
+/*
+ID is the inverse of control characters.
+Control characters ranges:
+  [\s]          Whitespace
+  [!"#%-,\./]   !, ", #, %, &, ', (, ), *, +, ,, ., /,  Exceptions in range: $, -
+  [;->@]        ;, <, =, >, @,                          Exceptions in range: :, ?
+  [\[-\^`]      [, \, ], ^, `,                          Exceptions in range: _
+  [\{-~]        {, |, }, ~
+*/
+<mu>[^\s!"#%-,\.\/;->@\[-\^`\{-~]+/[=}\s\/.]   { return 'ID'; }
+
 <mu>'['[^\]]*']'                 { yytext = yytext.substr(1, yyleng-2); return 'ID'; }
 <mu>.                            { return 'INVALID'; }
 <par>\s+                         { /*ignore whitespace*/ }