Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for utf8 chars #7

Merged
merged 13 commits into from
Jan 20, 2016
18 changes: 18 additions & 0 deletions bench.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
'use strict'

var split = require('./')
var bench = require('fastbench')
var fs = require('fs')

function benchSplit (cb) {
fs.createReadStream('package.json')
.pipe(split())
.on('end', cb)
.resume()
}

var run = bench([
benchSplit
], 10000)

run(run)
48 changes: 23 additions & 25 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,48 +14,47 @@ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

'use strict';
'use strict'

var through = require('through2')
var StringDecoder = require('string_decoder').StringDecoder

function transform(chunk, enc, cb) {
var list = chunk.toString('utf8').split(this.matcher)
, remaining = list.pop()
, i
function transform (chunk, enc, cb) {
this._last += this._decoder.write(chunk)

if (list.length >= 1) {
push(this, this.mapper((this._last + list.shift())))
} else {
remaining = this._last + remaining
}
var list = this._last.split(this.matcher)

this._last = list.pop()

for (i = 0; i < list.length; i++) {
for (var i = 0; i < list.length; i++) {
push(this, this.mapper(list[i]))
}

this._last = remaining

cb()
}

function flush(cb) {
if (this._last)
function flush (cb) {
// forward any gibberish left in there
this._last += this._decoder.end()

if (this._last) {
push(this, this.mapper(this._last))
}

cb()
}

function push(self, val) {
if (val !== undefined)
function push (self, val) {
if (val !== undefined) {
self.push(val)
}
}

function noop(incoming) {
function noop (incoming) {
return incoming
}

function split(matcher, mapper, options) {

function split (matcher, mapper, options) {
// Set defaults for any arguments not supplied.
matcher = matcher || /\r?\n/
mapper = mapper || noop
Expand All @@ -68,9 +67,8 @@ function split(matcher, mapper, options) {
if (typeof matcher === 'function') {
mapper = matcher
matcher = /\r?\n/
}
// If options is only argument.
else if (typeof matcher === 'object' && !(matcher instanceof RegExp)) {
} else if (typeof matcher === 'object' && !(matcher instanceof RegExp)) {
options = matcher
matcher = /\r?\n/
}
Expand All @@ -82,9 +80,8 @@ function split(matcher, mapper, options) {
options = mapper
mapper = matcher
matcher = /\r?\n/
}
// If matcher and options are arguments.
else if (typeof mapper === 'object') {
} else if (typeof mapper === 'object') {
options = mapper
mapper = noop
}
Expand All @@ -93,9 +90,10 @@ function split(matcher, mapper, options) {
var stream = through(options, transform, flush)

// this stream is in objectMode only in the readable part
stream._readableState.objectMode = true;
stream._readableState.objectMode = true

stream._last = ''
stream._decoder = new StringDecoder('utf8')
stream.matcher = matcher
stream.mapper = mapper

Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"description": "split a Text Stream into a Line Stream, using Stream 3",
"main": "index.js",
"scripts": {
"test": "tap test.js"
"test": "standard && tap -b test.js"
},
"pre-commit": [
"test"
Expand All @@ -21,7 +21,9 @@
"license": "ISC",
"devDependencies": {
"callback-stream": "^1.1.0",
"fastbench": "^1.0.0",
"pre-commit": "^1.1.2",
"standard": "^5.4.1",
"tap": "^5.0.0"
},
"dependencies": {
Expand Down
Loading