Integrate dicer into the repo (#14)

.gitattributes

@@ -0,0 +1,2 @@
+* text=false
+*.header -crlf

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12, 14, 15, 16, 17]
+        node-version: [10, 12, 14, 16, 17]
         os: [macos-latest, ubuntu-latest]
 
     steps:

deps/dicer/LICENSE

@@ -0,0 +1,19 @@
+Copyright Brian White. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

deps/dicer/bench/dicer-bench-multipart-parser.js

@@ -0,0 +1,62 @@
+var Dicer = require('../lib/Dicer'),
+    boundary = '-----------------------------168072824752491622650073',
+    d = new Dicer({ boundary: boundary }),
+    mb = 100,
+    buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
+    callbacks =
+      { partBegin: -1,
+        partEnd: -1,
+        headerField: -1,
+        headerValue: -1,
+        partData: -1,
+        end: -1,
+      };
+
+
+d.on('part', function(p) {
+  callbacks.partBegin++;
+  p.on('header', function(header) {
+    /*for (var h in header)
+      console.log('Part header: k: ' + inspect(h) + ', v: ' + inspect(header[h]));*/
+  });
+  p.on('data', function(data) {
+    callbacks.partData++;
+    //console.log('Part data: ' + inspect(data.toString()));
+  });
+  p.on('end', function() {
+    //console.log('End of part\n');
+    callbacks.partEnd++;
+  });
+});
+d.on('end', function() {
+  //console.log('End of parts');
+  callbacks.end++;
+});
+
+var start = +new Date(),
+    nparsed = d.write(buffer),
+    duration = +new Date - start,
+    mbPerSec = (mb / (duration / 1000)).toFixed(2);
+
+console.log(mbPerSec+' mb/sec');
+
+//assert.equal(nparsed, buffer.length);
+
+function createMultipartBuffer(boundary, size) {
+  var head =
+        '--'+boundary+'\r\n'
+      + 'content-disposition: form-data; name="field1"\r\n'
+      + '\r\n'
+    , tail = '\r\n--'+boundary+'--\r\n'
+    , buffer = Buffer.allocUnsafe(size);
+
+  buffer.write(head, 0, 'ascii');
+  buffer.write(tail, buffer.length - tail.length, 'ascii');
+  return buffer;
+}
+
+process.on('exit', function() {
+  /*for (var k in callbacks) {
+    assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
+  }*/
+});

deps/dicer/bench/formidable-bench-multipart-parser.js

@@ -0,0 +1,70 @@
+var assert = require('assert');
+require('../node_modules/formidable/test/common');
+var multipartParser = require('../node_modules/formidable/lib/multipart_parser'),
+    MultipartParser = multipartParser.MultipartParser,
+    parser = new MultipartParser(),
+    boundary = '-----------------------------168072824752491622650073',
+    mb = 100,
+    buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
+    callbacks =
+      { partBegin: -1,
+        partEnd: -1,
+        headerField: -1,
+        headerValue: -1,
+        partData: -1,
+        end: -1,
+      };
+
+
+parser.initWithBoundary(boundary);
+parser.onHeaderField = function() {
+  callbacks.headerField++;
+};
+
+parser.onHeaderValue = function() {
+  callbacks.headerValue++;
+};
+
+parser.onPartBegin = function() {
+  callbacks.partBegin++;
+};
+
+parser.onPartData = function() {
+  callbacks.partData++;
+};
+
+parser.onPartEnd = function() {
+  callbacks.partEnd++;
+};
+
+parser.onEnd = function() {
+  callbacks.end++;
+};
+
+var start = +new Date(),
+    nparsed = parser.write(buffer),
+    duration = +new Date - start,
+    mbPerSec = (mb / (duration / 1000)).toFixed(2);
+
+console.log(mbPerSec+' mb/sec');
+
+//assert.equal(nparsed, buffer.length);
+
+function createMultipartBuffer(boundary, size) {
+  var head =
+        '--'+boundary+'\r\n'
+      + 'content-disposition: form-data; name="field1"\r\n'
+      + '\r\n'
+    , tail = '\r\n--'+boundary+'--\r\n'
+    , buffer = Buffer.allocUnsafe(size);
+
+  buffer.write(head, 'ascii', 0);
+  buffer.write(tail, 'ascii', buffer.length - tail.length);
+  return buffer;
+}
+
+process.on('exit', function() {
+  /*for (var k in callbacks) {
+    assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
+  }*/
+});

deps/dicer/bench/multipartser-bench-multipart-parser.js

@@ -0,0 +1,56 @@
+var assert = require('assert');
+var multipartser = require('multipartser'),
+    boundary = '-----------------------------168072824752491622650073',
+    parser = multipartser(),
+    mb = 100,
+    buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
+    callbacks =
+      { partBegin: -1,
+        partEnd: -1,
+        headerField: -1,
+        headerValue: -1,
+        partData: -1,
+        end: -1,
+      };
+
+parser.boundary( boundary );
+
+parser.on( 'part', function ( part ) {
+});
+
+parser.on( 'end', function () {
+  //console.log( 'completed parsing' );
+});
+
+parser.on( 'error', function ( error ) {
+  console.error( error );
+});
+
+var start = +new Date(),
+    nparsed = parser.data(buffer),
+    nend = parser.end(),
+    duration = +new Date - start,
+    mbPerSec = (mb / (duration / 1000)).toFixed(2);
+
+console.log(mbPerSec+' mb/sec');
+
+//assert.equal(nparsed, buffer.length);
+
+function createMultipartBuffer(boundary, size) {
+  var head =
+        '--'+boundary+'\r\n'
+      + 'content-disposition: form-data; name="field1"\r\n'
+      + '\r\n'
+    , tail = '\r\n--'+boundary+'--\r\n'
+    , buffer = Buffer.allocUnsafe(size);
+
+  buffer.write(head, 'ascii', 0);
+  buffer.write(tail, 'ascii', buffer.length - tail.length);
+  return buffer;
+}
+
+process.on('exit', function() {
+  /*for (var k in callbacks) {
+    assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
+  }*/
+});

deps/dicer/bench/multiparty-bench-multipart-parser.js

@@ -0,0 +1,76 @@
+var assert = require('assert'),
+    Form = require('multiparty').Form,
+    boundary = '-----------------------------168072824752491622650073',
+    mb = 100,
+    buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
+    callbacks =
+      { partBegin: -1,
+        partEnd: -1,
+        headerField: -1,
+        headerValue: -1,
+        partData: -1,
+        end: -1,
+      };
+
+var form = new Form({ boundary: boundary });
+
+hijack('onParseHeaderField', function() {
+  callbacks.headerField++;
+});
+
+hijack('onParseHeaderValue', function() {
+  callbacks.headerValue++;
+});
+
+hijack('onParsePartBegin', function() {
+  callbacks.partBegin++;
+});
+
+hijack('onParsePartData', function() {
+  callbacks.partData++;
+});
+
+hijack('onParsePartEnd', function() {
+  callbacks.partEnd++;
+});
+
+form.on('finish', function() {
+  callbacks.end++;
+});
+
+var start = new Date();
+form.write(buffer, function(err) {
+  var duration = new Date() - start;
+  assert.ifError(err);
+  var mbPerSec = (mb / (duration / 1000)).toFixed(2);
+  console.log(mbPerSec+' mb/sec');
+});
+
+//assert.equal(nparsed, buffer.length);
+
+function createMultipartBuffer(boundary, size) {
+  var head =
+        '--'+boundary+'\r\n'
+      + 'content-disposition: form-data; name="field1"\r\n'
+      + '\r\n'
+    , tail = '\r\n--'+boundary+'--\r\n'
+    , buffer = Buffer.allocUnsafe(size);
+
+  buffer.write(head, 'ascii', 0);
+  buffer.write(tail, 'ascii', buffer.length - tail.length);
+  return buffer;
+}
+
+process.on('exit', function() {
+  /*for (var k in callbacks) {
+    assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
+  }*/
+});
+
+function hijack(name, fn) {
+  var oldFn = form[name];
+  form[name] = function() {
+    fn();
+    return oldFn.apply(this, arguments);
+  };
+}

deps/dicer/bench/parted-bench-multipart-parser.js

@@ -0,0 +1,63 @@
+// A special, edited version of the multipart parser from parted is needed here
+// because otherwise it attempts to do some things above and beyond just parsing
+// -- like saving to disk and whatnot
+
+var assert = require('assert');
+var Parser = require('./parted-multipart'),
+    boundary = '-----------------------------168072824752491622650073',
+    parser = new Parser('boundary=' + boundary),
+    mb = 100,
+    buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
+    callbacks =
+      { partBegin: -1,
+        partEnd: -1,
+        headerField: -1,
+        headerValue: -1,
+        partData: -1,
+        end: -1,
+      };
+
+
+parser.on('header', function() {
+  //callbacks.headerField++;
+});
+
+parser.on('data', function() {
+  //callbacks.partBegin++;
+});
+
+parser.on('part', function() {
+
+});
+
+parser.on('end', function() {
+  //callbacks.end++;
+});
+
+var start = +new Date(),
+    nparsed = parser.write(buffer),
+    duration = +new Date - start,
+    mbPerSec = (mb / (duration / 1000)).toFixed(2);
+
+console.log(mbPerSec+' mb/sec');
+
+//assert.equal(nparsed, buffer.length);
+
+function createMultipartBuffer(boundary, size) {
+  var head =
+        '--'+boundary+'\r\n'
+      + 'content-disposition: form-data; name="field1"\r\n'
+      + '\r\n'
+    , tail = '\r\n--'+boundary+'--\r\n'
+    , buffer = Buffer.allocUnsafe(size);
+
+  buffer.write(head, 'ascii', 0);
+  buffer.write(tail, 'ascii', buffer.length - tail.length);
+  return buffer;
+}
+
+process.on('exit', function() {
+  /*for (var k in callbacks) {
+    assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
+  }*/
+});