diff --git a/lib/object.js b/lib/object.js index 12689700..7415f45a 100644 --- a/lib/object.js +++ b/lib/object.js @@ -91,7 +91,7 @@ var dataToString = function(asUTF8) { } if (!asUTF8 && !this.options.binary) { - result = out.utf8encode(result); + result = utils.transformTo("string", out.utf8encode(result)); } return result; }; @@ -342,11 +342,11 @@ var generateCompressedObjectFrom = function(file, compression) { */ var generateZipParts = function(name, file, compressedObject, offset) { var data = compressedObject.compressedContent, - utfEncodedFileName = this.utf8encode(file.name), + utfEncodedFileName = utils.transformTo("string", utf8.utf8encode(file.name)), comment = file.comment || "", - utfEncodedComment = this.utf8encode(comment), - useUTF8ForFileName = utfEncodedFileName !== file.name, - useUTF8ForComment = utfEncodedComment !== comment, + utfEncodedComment = utils.transformTo("string", utf8.utf8encode(comment)), + useUTF8ForFileName = utfEncodedFileName.length !== file.name.length, + useUTF8ForComment = utfEncodedComment.length !== comment.length, o = file.options, dosTime, dosDate, @@ -640,7 +640,7 @@ var out = { localDirLength = 0, centralDirLength = 0, writer, i, - utfEncodedComment = this.utf8encode(options.comment || this.comment || ""); + utfEncodedComment = utils.transformTo("string", this.utf8encode(options.comment || this.comment || "")); // first, generate all the zip parts. for (var name in this.files) { @@ -738,7 +738,7 @@ var out = { * This method will be removed in a future version without replacement. */ utf8encode: function (string) { - return utf8.utf8encode(string); + return utils.transformTo("string", utf8.utf8encode(string)); }, /** diff --git a/lib/utf8.js b/lib/utf8.js index d8b9f114..546fac8e 100644 --- a/lib/utf8.js +++ b/lib/utf8.js @@ -15,95 +15,216 @@ if ( } /** - * http://www.webtoolkit.info/javascript-utf8.html + * The following functions come from pako, from pako/lib/utils/strings + * released under the MIT license, see pako https://github.com/nodeca/pako/ */ -exports.utf8encode = function utf8encode(string) { - // TextEncoder + Uint8Array to binary string is faster than checking every bytes on long strings. - // http://jsperf.com/utf8encode-vs-textencoder - // On short strings (file names for example), the TextEncoder API is (currently) slower. - if (textEncoder) { - var u8 = textEncoder.encode(string); - return utils.transformTo("string", u8); + +// Table with utf8 lengths (calculated by first byte of sequence) +// Note, that 5 & 6-byte values and some 4-byte values can not be represented in JS, +// because max possible codepoint is 0x10ffff +var _utf8len = new Array(256); +for (var i=0; i<256; i++) { + _utf8len[i] = (i >= 252 ? 6 : i >= 248 ? 5 : i >= 240 ? 4 : i >= 224 ? 3 : i >= 192 ? 2 : 1); +} +_utf8len[254]=_utf8len[254]=1; // Invalid sequence start + +// convert string to array (typed, when possible) +var string2buf = function (str) { + var buf, c, c2, m_pos, i, str_len = str.length, buf_len = 0; + + // count binary size + for (m_pos = 0; m_pos < str_len; m_pos++) { + c = str.charCodeAt(m_pos); + if ((c & 0xfc00) === 0xd800 && (m_pos+1 < str_len)) { + c2 = str.charCodeAt(m_pos+1); + if ((c2 & 0xfc00) === 0xdc00) { + c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00); + m_pos++; + } + } + buf_len += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4; } - if (support.nodebuffer) { - return utils.transformTo("string", nodeBuffer(string, "utf-8")); + + // allocate buffer + if (support.uint8array) { + buf = new Uint8Array(buf_len); + } else { + buf = new Array(buf_len); } - // array.join may be slower than string concatenation but generates less objects (less time spent garbage collecting). - // See also http://jsperf.com/array-direct-assignment-vs-push/31 - var result = [], - resIndex = 0; + // convert + for (i=0, m_pos = 0; i < buf_len; m_pos++) { + c = str.charCodeAt(m_pos); + if ((c & 0xfc00) === 0xd800 && (m_pos+1 < str_len)) { + c2 = str.charCodeAt(m_pos+1); + if ((c2 & 0xfc00) === 0xdc00) { + c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00); + m_pos++; + } + } + if (c < 0x80) { + /* one byte */ + buf[i++] = c; + } else if (c < 0x800) { + /* two bytes */ + buf[i++] = 0xC0 | (c >>> 6); + buf[i++] = 0x80 | (c & 0x3f); + } else if (c < 0x10000) { + /* three bytes */ + buf[i++] = 0xE0 | (c >>> 12); + buf[i++] = 0x80 | (c >>> 6 & 0x3f); + buf[i++] = 0x80 | (c & 0x3f); + } else { + /* four bytes */ + buf[i++] = 0xf0 | (c >>> 18); + buf[i++] = 0x80 | (c >>> 12 & 0x3f); + buf[i++] = 0x80 | (c >>> 6 & 0x3f); + buf[i++] = 0x80 | (c & 0x3f); + } + } - for (var n = 0; n < string.length; n++) { + return buf; +}; - var c = string.charCodeAt(n); +// Calculate max possible position in utf8 buffer, +// that will not break sequence. If that's not possible +// - (very small limits) return max size as is. +// +// buf[] - utf8 bytes array +// max - length limit (mandatory); +var utf8border = function(buf, max) { + var pos; - if (c < 128) { - result[resIndex++] = String.fromCharCode(c); - } - else if ((c > 127) && (c < 2048)) { - result[resIndex++] = String.fromCharCode((c >> 6) | 192); - result[resIndex++] = String.fromCharCode((c & 63) | 128); + max = max || buf.length; + if (max > buf.length) { max = buf.length; } + + // go back from last position, until start of sequence found + pos = max-1; + while (pos >= 0 && (buf[pos] & 0xC0) === 0x80) { pos--; } + + // Fuckup - very small and broken sequence, + // return max, because we should return something anyway. + if (pos < 0) { return max; } + + // If we came to start of buffer - that means vuffer is too small, + // return max too. + if (pos === 0) { return max; } + + return (pos + _utf8len[buf[pos]] > max) ? pos : max; +}; + +// convert array to string +var buf2string = function (buf) { + var str, i, out, c, c_len; + var len = buf.length; + + // Reserve max possible length (2 words per char) + // NB: by unknown reasons, Array is significantly faster for + // String.fromCharCode.apply than Uint16Array. + var utf16buf = new Array(len*2); + + for (out=0, i=0; i 4) { utf16buf[out++] = 0xfffd; i += c_len-1; continue; } + + // apply mask on first byte + c &= c_len === 2 ? 0x1f : c_len === 3 ? 0x0f : 0x07; + // join the rest + while (c_len > 1 && i < len) { + c = (c << 6) | (buf[i++] & 0x3f); + c_len--; } - else { - result[resIndex++] = String.fromCharCode((c >> 12) | 224); - result[resIndex++] = String.fromCharCode(((c >> 6) & 63) | 128); - result[resIndex++] = String.fromCharCode((c & 63) | 128); + + // terminated by end of string? + if (c_len > 1) { utf16buf[out++] = 0xfffd; continue; } + + if (c < 0x10000) { + utf16buf[out++] = c; + } else { + c -= 0x10000; + utf16buf[out++] = 0xd800 | ((c >> 10) & 0x3ff); + utf16buf[out++] = 0xdc00 | (c & 0x3ff); } + } + // shrinkBuf(utf16buf, out) + if (utf16buf.length !== out) { + if(utf16buf.subarray) { + utf16buf = utf16buf.subarray(0, out); + } else { + utf16buf.length = out; + } } - return result.join(""); + // return String.fromCharCode.apply(null, utf16buf); + return utils.applyFromCharCode(utf16buf); }; +// That's all for the pako functions. + + /** - * http://www.webtoolkit.info/javascript-utf8.html + * Transform a javascript string into an array (typed if possible) of bytes, + * UTF-8 encoded. + * @param {String} str the string to encode + * @return {Array|Uint8Array|Buffer} the UTF-8 encoded string. */ -exports.utf8decode = function utf8decode(input) { - var result = [], - resIndex = 0; - var type = utils.getTypeOf(input); - var isArray = type !== "string"; - var i = 0; - var c = 0, - c1 = 0, - c2 = 0, - c3 = 0; +exports.utf8encode = function utf8encode(str) { + // TextEncoder + Uint8Array to binary string is faster than checking every bytes on long strings. + // http://jsperf.com/utf8encode-vs-textencoder + // On short strings (file names for example), the TextEncoder API is (currently) slower. + if (textEncoder) { + return textEncoder.encode(str); + } + if (support.nodebuffer) { + return nodeBuffer(str, "utf-8"); + } + return string2buf(str); +}; + + +/** + * Transform a bytes array (or a representation) representing an UTF-8 encoded + * string into a javascript string. + * @param {Array|Uint8Array|Buffer} buf the data de decode + * @return {String} the decoded string. + */ +exports.utf8decode = function utf8decode(buf) { // check if we can use the TextDecoder API // see http://encoding.spec.whatwg.org/#api if (textDecoder) { return textDecoder.decode( - utils.transformTo("uint8array", input) + utils.transformTo("uint8array", buf) ); } if (support.nodebuffer) { - return utils.transformTo("nodebuffer", input).toString("utf-8"); + return utils.transformTo("nodebuffer", buf).toString("utf-8"); } - while (i < input.length) { - - c = isArray ? input[i] : input.charCodeAt(i); - - if (c < 128) { - result[resIndex++] = String.fromCharCode(c); - i++; + buf = utils.transformTo(support.uint8array ? "uint8array" : "array", buf); + + // return buf2string(buf); + // Chrome prefers to work with "small" chunks of data + // for the method buf2string. + // Firefox and Chrome has their own shortcut, IE doesn't seem to really care. + var result = [], k = 0, len = buf.length, chunk = 65536; + while (k < len) { + var nextBoundary = utf8border(buf, Math.min(k + chunk, len)); + if (support.uint8array) { + result.push(buf2string(buf.subarray(k, nextBoundary))); + } else { + result.push(buf2string(buf.slice(k, nextBoundary))); } - else if ((c > 191) && (c < 224)) { - c2 = isArray ? input[i + 1] : input.charCodeAt(i + 1); - result[resIndex++] = String.fromCharCode(((c & 31) << 6) | (c2 & 63)); - i += 2; - } - else { - c2 = isArray ? input[i + 1] : input.charCodeAt(i + 1); - c3 = isArray ? input[i + 2] : input.charCodeAt(i + 2); - result[resIndex++] = String.fromCharCode(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63)); - i += 3; - } - + k = nextBoundary; } - return result.join(""); + }; // vim: set shiftwidth=4 softtabstop=4: diff --git a/lib/utils.js b/lib/utils.js index 91c5d8fc..1e3bb5b6 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -123,6 +123,9 @@ function arrayLikeToString(array) { return result.join(""); } +exports.applyFromCharCode = arrayLikeToString; + + /** * Copy the data from an array-like to an other array-like. * @param {Array|ArrayBuffer|Uint8Array|Buffer} arrayFrom the origin array. diff --git a/test/ref/pile_of_poo.zip b/test/ref/pile_of_poo.zip new file mode 100644 index 00000000..1ff8df8c Binary files /dev/null and b/test/ref/pile_of_poo.zip differ diff --git a/test/test.js b/test/test.js index bd0e1e6d..63eeffed 100644 --- a/test/test.js +++ b/test/test.js @@ -161,6 +161,28 @@ testZipFile("Zip text file with UTF-8 characters in filename", "ref/utf8_in_name equal(reload(actual), actual, "Generated ZIP can be parsed"); }); +// zip -X -0 pile_of_poo.zip Iñtërnâtiônàlizætiøn☃💩.txt +testZipFile("Zip text file and UTF-8, Pile Of Poo test", "ref/pile_of_poo.zip", function(expected) { + var zip = new JSZip(); + // this is the string "Iñtërnâtiônàlizætiøn☃💩", + // see http://mathiasbynens.be/notes/javascript-unicode + // but escaped, to avoid troubles + // thanks http://mothereff.in/js-escapes#1I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%C3%A0liz%C3%A6ti%C3%B8n%E2%98%83%F0%9F%92%A9 + var text = 'I\xF1t\xEBrn\xE2ti\xF4n\xE0liz\xE6ti\xF8n\u2603\uD83D\uDCA9'; + zip.file(text + ".txt", text + "\n"); + var actual = zip.generate({type:"string"}); + + equal(reload(actual), actual, "Generated ZIP can be parsed"); + + ok(new JSZip(expected).file(text + ".txt"), "JSZip finds the unicode file name on the external file"); + ok(new JSZip(actual).file(text + ".txt"), "JSZip finds the unicode file name on its own file"); + var textFromExpected = new JSZip(expected).file(text + ".txt").asText(); + var textFromActual = new JSZip(actual).file(text + ".txt").asText(); + + equal(textFromExpected, text + "\n", "JSZip can decode the external file"); + equal(textFromActual, text + "\n", "JSZip can decode its own file"); +}); + testZipFile("Zip text file with date", "ref/text.zip", function(expected) { var zip = new JSZip(); zip.file("Hello.txt", "Hello World\n", {date : new Date("July 17, 2009 14:36:57")});