streams: add stream.compose

Refs: nodejs#32020

doc/api/stream.md

@@ -1859,6 +1859,93 @@ run().catch(console.error);
 after the `callback` has been invoked. In the case of reuse of streams after
 failure, this can cause event listener leaks and swallowed errors.
 
+### `stream.compose(...streams)`
+<!-- YAML
+added: REPLACEME
+-->
+
+* `streams` {Stream[]|Iterable[]|AsyncIterable[]|Function[]}
+* Returns: {stream.Duplex}
+
+Combines two or more streams into a `Duplex` stream that writes to the
+first stream and reads from the last. Each provided stream is piped into
+the next, using `stream.pipeline`. If any of the streams error then all
+are destroyed, including the outer `Duplex` stream.
+
+Because `stream.compose` returns a new stream that in turn can (and
+should) be piped into other streams, it enables composition. In contrast,
+when passing streams to `stream.pipeline`, typically the first stream is
+a readable stream and the last a writable stream, forming a closed
+circuit.
+
+If passed a `Function` is must be a factory method taking a `source`
+`Iterable`.
+
+```mjs
+import { compose, Transform } from 'stream';
+
+const removeSpaces = new Transform({
+  transform(chunk, encoding, callback) {
+    callback(null, String(chunk).replace(' ', ''));
+  }
+});
+
+async function* toUpper(source) {
+  for await (const chunk of source) {
+    yield String(chunk).toUpperCase();
+  }
+}
+
+let res = '';
+for await (const buf of compose(removeSpaces, toUpper).end('hello world')) {
+  res += buf;
+}
+
+console.log(res); // prints 'HELLOWORLD'
+```
+
+`stream.compose` can be used to convert async iterables, generators and
+functions into streams.
+
+* `AsyncIterable` converts into a readable `Duplex`.  Cannot yield
+  `null`.
+* `AsyncGeneratorFunction` converts into a readable/writable transform `Duplex`.
+  Must take a source `AsyncIterable` as first parameter. Cannot yield
+  `null`.
+* `AsyncFunction` converts into a writable `Duplex`. Must return
+  either `null` or `undefined`.
+
+```mjs
+import { compose } from 'stream';
+import { finished } from 'stream/promises';
+
+// Convert AsyncIterable into readable Duplex.
+const s1 = compose(async function*() {
+  yield 'Hello';
+  yield 'World';
+}());
+
+// Convert AsyncGenerator into transform Duplex.
+const s2 = compose(async function*(source) {
+  for await (const chunk of source) {
+    yield String(chunk).toUpperCase();
+  }
+});
+
+let res = '';
+
+// Convert AsyncFunction into writable Duplex.
+const s3 = compose(async function(source) {
+  for await (const chunk of source) {
+    res += chunk;
+  }
+});
+
+await finished(compose(s1, s2, s3));
+
+console.log(res); // prints 'HELLOWORLD'
+```
+
 ### `stream.Readable.from(iterable, [options])`
 <!-- YAML
 added:

lib/internal/streams/compose.js

@@ -0,0 +1,290 @@
+'use strict';
+
+const pipeline = require('internal/streams/pipeline');
+const Duplex = require('internal/streams/duplex');
+const { createDeferredPromise } = require('internal/util');
+const { destroyer } = require('internal/streams/destroy');
+const from = require('internal/streams/from');
+const {
+  isStream,
+  isIterable,
+  isReadable,
+  isReadableNodeStream,
+  isWritable,
+  isWritableNodeStream
+} = require('internal/streams/utils');
+const {
+  PromiseResolve,
+} = primordials;
+const {
+  AbortError,
+  codes: {
+    ERR_INVALID_ARG_TYPE,
+    ERR_INVALID_ARG_VALUE,
+    ERR_INVALID_RETURN_VALUE,
+    ERR_MISSING_ARGS,
+  },
+} = require('internal/errors');
+const assert = require('internal/assert');
+
+// This is needed for pre node 17.
+class ComposeDuplex extends Duplex {
+  constructor(options) {
+    super(options);
+
+    // https://github.com/nodejs/node/pull/34385
+
+    if (options?.readable === false) {
+      this._readableState.readable = false;
+      this._readableState.ended = true;
+      this._readableState.endEmitted = true;
+    }
+
+    if (options?.writable === false) {
+      this._writableState.writable = false;
+      this._writableState.ending = true;
+      this._writableState.ended = true;
+      this._writableState.finished = true;
+    }
+  }
+}
+
+module.exports = function compose(...streams) {
+  if (streams.length === 0) {
+    throw new ERR_MISSING_ARGS('streams');
+  }
+
+  if (streams.length === 1) {
+    return makeDuplex(streams[0], 'streams[0]');
+  }
+
+  const orgStreams = [...streams];
+
+  if (typeof streams[0] === 'function') {
+    streams[0] = makeDuplex(streams[0], 'streams[0]');
+  }
+
+  if (typeof streams[streams.length - 1] === 'function') {
+    const idx = streams.length - 1;
+    streams[idx] = makeDuplex(streams[idx], `streams[${idx}]`);
+  }
+
+  for (let n = 0; n < streams.length; ++n) {
+    if (!isStream(streams[n])) {
+      // TODO (ronag): Add checks for non streams.
+      continue;
+    }
+    if (n < streams.length - 1 && !isReadableNodeStream(streams[n])) {
+      throw new ERR_INVALID_ARG_VALUE(
+        `streams[${n}]`,
+        orgStreams[n],
+        'must be readable'
+      );
+    }
+    if (n > 0 && !isWritableNodeStream(streams[n])) {
+      throw new ERR_INVALID_ARG_VALUE(
+        `streams[${n}]`,
+        orgStreams[n],
+        'must be writable'
+      );
+    }
+  }
+
+  let ondrain;
+  let onfinish;
+  let onreadable;
+  let onclose;
+  let d;
+
+  function onfinished(err) {
+    const cb = onclose;
+    onclose = null;
+
+    if (cb) {
+      cb(err);
+    } else if (err) {
+      d.destroy(err);
+    }
+  }
+
+  const head = streams[0];
+  const tail = pipeline(streams, onfinished);
+
+  const writable = isWritable(head);
+  const readable = isReadable(tail);
+
+  // TODO (ronag): Avoid double buffering.
+  // Implement Writable/Readable/Duplex traits.
+  // See, https://github.com/nodejs/node/pull/33515.
+  d = new ComposeDuplex({
+    highWaterMark: 1,
+    writableObjectMode: !!head?.writableObjectMode,
+    readableObjectMode: !!tail?.writableObjectMode,
+    writable,
+    readable,
+  });
+
+  if (writable) {
+    d._write = function(chunk, encoding, callback) {
+      if (head.write(chunk, encoding)) {
+        callback();
+      } else {
+        ondrain = callback;
+      }
+    };
+
+    d._final = function(callback) {
+      head.end();
+      onfinish = callback;
+    };
+
+    head.on('drain', function() {
+      if (ondrain) {
+        const cb = ondrain;
+        ondrain = null;
+        cb();
+      }
+    });
+
+    tail.on('finish', function() {
+      if (onfinish) {
+        const cb = onfinish;
+        onfinish = null;
+        cb();
+      }
+    });
+  }
+
+  if (readable) {
+    tail.on('readable', function() {
+      if (onreadable) {
+        const cb = onreadable;
+        onreadable = null;
+        cb();
+      }
+    });
+
+    tail.on('end', function() {
+      d.push(null);
+    });
+
+    d._read = function() {
+      while (true) {
+        const buf = tail.read();
+
+        if (buf === null) {
+          onreadable = d._read;
+          return;
+        }
+
+        if (!d.push(buf)) {
+          return;
+        }
+      }
+    };
+  }
+
+  if (!readable && !writable) {
+    tail.on('finish', function() {
+      d.destroy();
+    });
+  }
+
+  d._destroy = function(err, callback) {
+    if (!err && onclose !== null) {
+      err = new AbortError();
+    }
+
+    onreadable = null;
+    ondrain = null;
+    onfinish = null;
+
+    if (onclose === null) {
+      callback(err);
+    } else {
+      onclose = callback;
+      destroyer(tail, err);
+    }
+  };
+
+  return d;
+};
+
+function makeDuplex(stream, name) {
+  let ret;
+  if (typeof stream === 'function') {
+    assert(stream.length > 0);
+
+    const { value, write, final } = fromAsyncGen(stream);
+
+    if (isIterable(value)) {
+      ret = from(ComposeDuplex, value, {
+        objectMode: true,
+        highWaterMark: 1,
+        write,
+        final
+      });
+    } else if (typeof value?.then === 'function') {
+      const promise = PromiseResolve(value)
+        .then((val) => {
+          if (val != null) {
+            throw new ERR_INVALID_RETURN_VALUE('nully', name, val);
+          }
+        })
+        .catch((err) => {
+          destroyer(ret, err);
+        });
+
+      ret = new ComposeDuplex({
+        objectMode: true,
+        highWaterMark: 1,
+        readable: false,
+        write,
+        final(cb) {
+          final(() => promise.then(cb, cb));
+        }
+      });
+    } else {
+      throw new ERR_INVALID_RETURN_VALUE(
+        'Iterable, AsyncIterable or AsyncFunction', name, value);
+    }
+  } else if (isStream(stream)) {
+    ret = stream;
+  } else if (isIterable(stream)) {
+    ret = from(ComposeDuplex, stream, {
+      objectMode: true,
+      highWaterMark: 1,
+      writable: false
+    });
+  } else {
+    throw new ERR_INVALID_ARG_TYPE(
+      name,
+      ['Stream', 'Iterable', 'AsyncIterable', 'Function'],
+      stream)
+    ;
+  }
+  return ret;
+}
+
+function fromAsyncGen(fn) {
+  let { promise, resolve } = createDeferredPromise();
+  const value = fn(async function*() {
+    while (true) {
+      const { chunk, done, cb } = await promise;
+      process.nextTick(cb);
+      if (done) return;
+      yield chunk;
+      ({ promise, resolve } = createDeferredPromise());
+    }
+  }());
+
+  return {
+    value,
+    write(chunk, encoding, cb) {
+      resolve({ chunk, done: false, cb });
+    },
+    final(cb) {
+      resolve({ done: true, cb });
+    }
+  };
+}

lib/internal/streams/pipeline.js

@@ -295,9 +295,6 @@ function pipeline(...streams) {
     }
   }
 
-  // TODO(ronag): Consider returning a Duplex proxy if the first argument
-  // is a writable. Would improve composability.
-  // See, https://github.com/nodejs/node/issues/32020
   return ret;
 }