Remove the latin1-wtf8 encoding mode

The default has been changed to 'none'. The testsuite has been rebuilt to reflect this; the encoding modes test in runner.js will still exercise all available encoding modes.
fstirlitz · Feb 22, 2020 · be3c3ee · be3c3ee
1 parent fb941d3
commit be3c3ee
Show file tree

Hide file tree

Showing 13 changed files with 84 additions and 121 deletions.
diff --git a/luaparse.js b/luaparse.js
@@ -87,7 +87,7 @@
     // '5.1', '5.2', '5.3').
     , luaVersion: '5.1'
     // Encoding mode: how to interpret code units higher than U+007F in input
-    , encodingMode: 'latin1-wtf8'
+    , encodingMode: 'none'
   };
 
   function encodeUTF8(codepoint, highMask) {
@@ -136,25 +136,6 @@
   }
 
   var encodingModes = {
-    // `latin1-wtf8`: assume input was UTF-8, output AST as if it was interpreted as latin1
-    'latin1-wtf8': {
-      fixup: function (s) {
-        return s.replace(/[\ud800-\udbff][\udc00-\udfff]|[^\x00-\x7f]/g, function (m) {
-          if (m.length === 1)
-            return encodeUTF8(m.charCodeAt(0));
-          return encodeUTF8(0x10000 + (((m.charCodeAt(0) & 0x3ff) << 10) | (m.charCodeAt(1) & 0x3ff)));
-        });
-      },
-      encodeByte: function (value) {
-        if (value === null)
-          return '';
-        return String.fromCharCode(value);
-      },
-      encodeUTF8: function (codepoint) {
-        return encodeUTF8(codepoint);
-      }
-    },
-
     // `pseudo-latin1` encoding mode: assume the input was decoded with the latin1 encoding
     // WARNING: latin1 does **NOT** mean cp1252 here like in the bone-headed WHATWG standard;
     // it means true ISO/IEC 8859-1 identity-mapped to Basic Latin and Latin-1 Supplement blocks

diff --git a/test/runner.js b/test/runner.js
@@ -571,24 +571,6 @@
 
   suite.addTest('Encoding modes', function () {
     var cases = [
-      { mode: 'latin1-wtf8', src: '"\\u{0}"', value: '\u0000' },
-      { mode: 'latin1-wtf8', src: '"\\u{80}"', value: '\u00c2\u0080' },
-      { mode: 'latin1-wtf8', src: '"\\u{d800}"', value: '\u00ed\u00a0\u0080' },
-      { mode: 'latin1-wtf8', src: '"\\u{20000}"', value: '\u00f0\u00a0\u0080\u0080' },
-      { mode: 'latin1-wtf8', src: '"\\x80"', value: '\u0080' },
-      { mode: 'latin1-wtf8', src: '"\\x61"', value: 'a' },
-      { mode: 'latin1-wtf8', src: '"\\128"', value: '\u0080' },
-      { mode: 'latin1-wtf8', src: '"\uf780"', value: '\u00ef\u009e\u0080' },
-      { mode: 'latin1-wtf8', src: '"\ud800"', value: '\u00ed\u00a0\u0080' },
-      { mode: 'latin1-wtf8', src: '"\u0080"', value: '\u00c2\u0080' },
-      { mode: 'latin1-wtf8', src: '[[\uf780]]', value: '\u00ef\u009e\u0080' },
-      { mode: 'latin1-wtf8', src: '[[\ud800]]', value: '\u00ed\u00a0\u0080' },
-      { mode: 'latin1-wtf8', src: '[[\u0080]]', value: '\u00c2\u0080' },
-      { mode: 'latin1-wtf8', src: '"a"', value: 'a' },
-      { mode: 'latin1-wtf8', src: '\uf780', name: '\xef\x9e\x80' },
-      { mode: 'latin1-wtf8', src: '\x80', name: '\xc2\x80' },
-      { mode: 'latin1-wtf8', src: 'a', name: 'a' },
-
       { mode: 'x-user-defined', src: '"\\u{0}"', value: '\u0000' },
       { mode: 'x-user-defined', src: '"\\u{80}"', value: '\uf7c2\uf780' },
       { mode: 'x-user-defined', src: '"\\u{d800}"', value: '\uf7ed\uf7a0\uf780' },

diff --git a/test/spec/assignments.js b/test/spec/assignments.js
@@ -2306,7 +2306,7 @@
                   },
                   "argument": {
                     "type": "StringLiteral",
-                    "value": "",
+                    "value": null,
                     "raw": "\"\"",
                     "loc": {
                       "start": {

diff --git a/test/spec/escapesequences.js b/test/spec/escapesequences.js
@@ -56,7 +56,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "bar\n\r\t\tbaz",
+                "value": null,
                 "raw": "\"bar\\n\\r\\t\tbaz\"",
                 "loc": {
                   "start": {
@@ -159,7 +159,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "barPbaz\u0007P\u0000foo",
+                "value": null,
                 "raw": "\"bar\\80baz\\a\\80\\0foo\"",
                 "loc": {
                   "start": {
@@ -262,7 +262,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "bar\f\u000b\bbaz",
+                "value": null,
                 "raw": "\"bar\\f\\v\\bbaz\"",
                 "loc": {
                   "start": {
@@ -365,7 +365,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "bar\f\u000b\baz",
+                "value": null,
                 "raw": "\"bar\f\u000b\baz\"",
                 "loc": {
                   "start": {
@@ -468,7 +468,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "\\",
+                "value": null,
                 "raw": "'\\\\'",
                 "loc": {
                   "start": {
@@ -571,7 +571,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "'",
+                "value": null,
                 "raw": "'\\''",
                 "loc": {
                   "start": {
@@ -674,7 +674,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "[",
+                "value": null,
                 "raw": "'\\['",
                 "loc": {
                   "start": {
@@ -777,7 +777,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "{",
+                "value": null,
                 "raw": "'\\123'",
                 "loc": {
                   "start": {
@@ -880,7 +880,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "ÿ",
+                "value": null,
                 "raw": "'\\255'",
                 "loc": {
                   "start": {
@@ -991,7 +991,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "bar\\f\\v\\bbaz",
+                "value": null,
                 "raw": "[[bar\\f\\v\\bbaz]]",
                 "loc": {
                   "start": {
@@ -1094,7 +1094,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "\\",
+                "value": null,
                 "raw": "[[\\]]",
                 "loc": {
                   "start": {
@@ -1197,7 +1197,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "z x",
+                "value": null,
                 "raw": "'\\z \\x'",
                 "loc": {
                   "start": {
@@ -1311,7 +1311,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "#",
+                "value": null,
                 "raw": "'\\x23'",
                 "loc": {
                   "start": {
@@ -1454,7 +1454,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "barbaz",
+                "value": null,
                 "raw": "\"bar\\z    baz\"",
                 "loc": {
                   "start": {
@@ -1564,7 +1564,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "\u0000",
+                "value": null,
                 "raw": "'\\u{0}'",
                 "loc": {
                   "start": {
@@ -1674,7 +1674,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "ð\u009f\u0092©",
+                "value": null,
                 "raw": "'\\u{1f4a9}'",
                 "loc": {
                   "start": {
@@ -1784,7 +1784,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "\u007fÂ\u0080",
+                "value": null,
                 "raw": "'\\u{7f}\\u{80}'",
                 "loc": {
                   "start": {
@@ -1894,7 +1894,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "ß¿à \u0080",
+                "value": null,
                 "raw": "'\\u{7ff}\\u{800}'",
                 "loc": {
                   "start": {
@@ -2004,7 +2004,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "ï¿¿ð\u009f\u0092©",
+                "value": null,
                 "raw": "'\\u{ffff}\\u{1f4a9}'",
                 "loc": {
                   "start": {
@@ -2114,7 +2114,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "ô\u008f¿¿",
+                "value": null,
                 "raw": "'\\u{10ffff}'",
                 "loc": {
                   "start": {
@@ -2246,7 +2246,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "ð\u009f\u0092©",
+                "value": null,
                 "raw": "'\\u{000000001f4a9}'",
                 "loc": {
                   "start": {

diff --git a/test/spec/expressions.js b/test/spec/expressions.js
@@ -52,7 +52,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "foo",
+                "value": null,
                 "raw": "[[foo]]",
                 "loc": {
                   "start": {
@@ -669,7 +669,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "foo",
+                "value": null,
                 "raw": "\"foo\"",
                 "loc": {
                   "start": {
@@ -772,7 +772,7 @@
             "init": [
               {
                 "type": "StringLiteral",
-                "value": "foo",
+                "value": null,
                 "raw": "[[foo]]",
                 "loc": {
                   "start": {
@@ -1621,7 +1621,7 @@
                 },
                 "index": {
                   "type": "StringLiteral",
-                  "value": "foo",
+                  "value": null,
                   "raw": "\"foo\"",
                   "loc": {
                     "start": {
@@ -2780,7 +2780,7 @@
                     "type": "TableValue",
                     "value": {
                       "type": "StringLiteral",
-                      "value": "-",
+                      "value": null,
                       "raw": "'-'",
                       "loc": {
                         "start": {
@@ -2919,7 +2919,7 @@
                     "type": "TableValue",
                     "value": {
                       "type": "StringLiteral",
-                      "value": "not",
+                      "value": null,
                       "raw": "'not'",
                       "loc": {
                         "start": {

diff --git a/test/spec/for.js b/test/spec/for.js
@@ -1110,7 +1110,7 @@
             "iterators": [
               {
                 "type": "StringLiteral",
-                "value": "foo",
+                "value": null,
                 "raw": "\"foo\"",
                 "loc": {
                   "start": {