Support web-streams (WHATWG standard streams)

lib/ReadStreamTokenizer.ts

@@ -1,17 +1,13 @@
 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { EndOfStreamError, StreamReader } from 'peek-readable';
-import { Readable } from 'node:stream';
+import { EndOfStreamError, IStreamReader } from 'peek-readable';
 import { IFileInfo, IReadChunkOptions } from './types.js';
 
 const maxBufferSize = 256000;
 
 export class ReadStreamTokenizer extends AbstractTokenizer {
 
-  private streamReader: StreamReader;
-
-  public constructor(stream: Readable, fileInfo?: IFileInfo) {
+  public constructor(private streamReader: IStreamReader, fileInfo?: IFileInfo) {
     super(fileInfo);
-    this.streamReader = new StreamReader(stream);
   }
 
   /**

lib/core.ts

@@ -2,6 +2,8 @@ import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { Readable } from 'node:stream';
 import { BufferTokenizer } from './BufferTokenizer.js';
 import { IFileInfo } from './types.js';
+import { StreamReader, WebStreamReader} from "peek-readable";
+import type { ReadableStream } from 'node:stream/web';
 export { EndOfStreamError } from 'peek-readable';
 export { ITokenizer, IFileInfo } from './types.js';
 export { IToken, IGetToken } from '@tokenizer/token';
@@ -15,7 +17,19 @@ export { IToken, IGetToken } from '@tokenizer/token';
  */
 export function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer {
   fileInfo = fileInfo ? fileInfo : {};
-  return new ReadStreamTokenizer(stream, fileInfo);
+  return new ReadStreamTokenizer(new StreamReader(stream), fileInfo);
+}
+
+/**
+ * Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
+ * Will set fileSize, if provided given Stream has set the .path property/
+ * @param webStream - Read from Node.js Stream.Readable
+ * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @returns ReadStreamTokenizer
+ */
+export function fromWebStream(webStream: ReadableStream<Uint8Array>, fileInfo?: IFileInfo): ReadStreamTokenizer {
+  fileInfo = fileInfo ? fileInfo : {};
+  return new ReadStreamTokenizer(new WebStreamReader(webStream), fileInfo);
 }
 
 /**

lib/index.ts

@@ -4,7 +4,7 @@ import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import * as core from './core.js';
 
 export { fromFile } from './FileTokenizer.js';
-export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core.js';
+export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo } from './core.js';
 export { IToken, IGetToken } from '@tokenizer/token';
 
 /**