feat: data chunking and stream parsing tests

experimental/storage/test/data-chunk/buffer-chunk.spec.ts

@@ -0,0 +1,47 @@
+import { chunkFromBuffer } from "../../src/data-chunk/buffer-helper";
+import { DataPart } from "../../src/data-chunk/yield-chunk";
+
+describe(chunkFromBuffer.name, () => {
+  const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive";
+
+  describe("String chunking", () => {
+    it("should come back with small sub buffers", async (done) => {
+      const length = 10;
+      const chunker = chunkFromBuffer(Buffer.from(INPUT_STRING), length);
+      const result = await chunker.next();
+      const dataChunk = result.value as DataPart;
+      expect(dataChunk.PartNumber).toEqual(1);
+      expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(0, length));
+
+      done();
+    });
+
+    it("should come back with many chunks on subsequent calls", async (done) => {
+      const length = 10;
+      const chunker = chunkFromBuffer(Buffer.from(INPUT_STRING), length);
+
+      let result = await chunker.next();
+      let dataChunk = result.value as DataPart;
+      expect(dataChunk.PartNumber).toEqual(1);
+      expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(0, length));
+
+      result = await chunker.next();
+      dataChunk = result.value as DataPart;
+      expect(dataChunk.PartNumber).toEqual(2);
+      expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(length, length * 2));
+
+      done();
+    });
+
+    it("should have the last sub buffer be the remainder length", async (done) => {
+      const length = 30;
+      const chunker = chunkFromBuffer(Buffer.from(INPUT_STRING), length);
+      let result = await chunker.next();
+      result = await chunker.next();
+      const dataChunk = result.value as DataPart;
+      expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(length));
+
+      done();
+    });
+  });
+});

experimental/storage/test/data-chunk/readable-helper.spec.ts

@@ -0,0 +1,160 @@
+import { Readable } from "stream";
+import { chunkFromReadable } from "../../src/data-chunk/readable-helper";
+import { DataPart } from "../../src/data-chunk/yield-chunk";
+const fs = require("fs");
+const fileStream = fs.createReadStream(
+  "/Users/alexfors/Documents/workspace/sdk/alexfors/aws-sdk-js-v3/experimental/storage/test/data-chunk/sample.file"
+);
+
+describe(chunkFromReadable.name, () => {
+  const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive";
+
+  describe("Proper chunk creation", () => {
+    let chunks: IteratorResult<DataPart>[] = [];
+    const CHUNK_SIZE = 30;
+
+    const setup = async () => {
+      const chunks: IteratorResult<DataPart>[] = [];
+      const readableStream = Readable.from(INPUT_STRING);
+      const interation = chunkFromReadable(readableStream, CHUNK_SIZE);
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      return chunks;
+    };
+
+    beforeAll(async (done) => {
+      chunks = await setup();
+      done();
+    });
+
+    it("should properly give the first chunk", async (done) => {
+      expect(chunks[0].done).toEqual(false);
+      expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big ");
+      expect(chunks[0].value.Body.length).toEqual(30);
+      expect(chunks[0].value.PartNumber).toEqual(1);
+      done();
+    });
+
+    it("should properly give the second (smaller) chunk", async (done) => {
+      expect(chunks[1].done).toEqual(false);
+      expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive");
+      expect(chunks[1].value.Body.length).toEqual(28);
+      expect(chunks[1].value.PartNumber).toEqual(2);
+      done();
+    });
+
+    it("should properly end the interation", async (done) => {
+      expect(chunks[2].done).toEqual(true);
+      expect(chunks[2].value).toEqual(undefined);
+      done();
+    });
+  });
+
+  describe("Proper single chunk", () => {
+    let chunks: IteratorResult<DataPart>[] = [];
+    const CHUNK_SIZE = 1000;
+
+    const setup = async () => {
+      const chunks: IteratorResult<DataPart>[] = [];
+      const readableStream = Readable.from(INPUT_STRING);
+      const interation = chunkFromReadable(readableStream, CHUNK_SIZE);
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      return chunks;
+    };
+
+    beforeAll(async (done) => {
+      chunks = await setup();
+      done();
+    });
+
+    it("should properly give the first chunk as all the data", async (done) => {
+      expect(CHUNK_SIZE).toBeGreaterThan(INPUT_STRING.length);
+      expect(chunks[0].done).toEqual(false);
+      expect(chunks[0].value.Body.toString()).toEqual(INPUT_STRING);
+      expect(chunks[0].value.Body.length).toEqual(INPUT_STRING.length);
+      expect(chunks[0].value.PartNumber).toEqual(1);
+      done();
+    });
+
+    it("should properly end the interation", async (done) => {
+      expect(chunks[1].done).toEqual(true);
+      expect(chunks[1].value).toEqual(undefined);
+      done();
+    });
+  });
+
+  describe("Proper no data chunk", () => {
+    let chunks: IteratorResult<DataPart>[] = [];
+    const CHUNK_SIZE = 10;
+
+    const setup = async () => {
+      const chunks: IteratorResult<DataPart>[] = [];
+      const readableStream = Readable.from([]);
+      const interation = chunkFromReadable(readableStream, CHUNK_SIZE);
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      return chunks;
+    };
+
+    beforeAll(async (done) => {
+      chunks = await setup();
+      done();
+    });
+
+    it("should return an empty object", async (done) => {
+      expect(chunks[0].done).toEqual(false);
+      expect(chunks[0].value.Body.toString()).toEqual("");
+      done();
+    });
+
+    it("should properly end the interation", async (done) => {
+      expect(chunks[1].done).toEqual(true);
+      expect(chunks[1].value).toEqual(undefined);
+      done();
+    });
+  });
+
+  describe("Properly chunks files", () => {
+    let chunks: IteratorResult<DataPart>[] = [];
+    const CHUNK_SIZE = 30;
+
+    const setup = async () => {
+      const chunks: IteratorResult<DataPart>[] = [];
+      const interation = chunkFromReadable(fileStream, CHUNK_SIZE);
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      return chunks;
+    };
+
+    beforeAll(async (done) => {
+      chunks = await setup();
+      expect(1).toEqual(1);
+      done();
+    });
+
+    it("should properly give the first chunk", async (done) => {
+      expect(chunks[0].done).toEqual(false);
+      expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big ");
+      expect(chunks[0].value.Body.length).toEqual(30);
+      expect(chunks[0].value.PartNumber).toEqual(1);
+      done();
+    });
+
+    // it("should properly give the second (smaller) chunk", async (done) => {
+    //   expect(chunks[1].done).toEqual(false);
+    //   expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive");
+    //   expect(chunks[1].value.Body.length).toEqual(28);
+    //   expect(chunks[1].value.PartNumber).toEqual(2);
+    //   done();
+    // });
+
+    // it("should properly end the interation", async (done) => {
+    //   expect(chunks[2].done).toEqual(true);
+    //   expect(chunks[2].value).toEqual(undefined);
+    //   done();
+    // });
+  });
+});

experimental/storage/test/data-chunk/readable-stream-helper.spec.ts

@@ -0,0 +1,59 @@
+import { ReadableStream } from "web-streams-polyfill";
+import { chunkFromReadableStream } from "../../src/data-chunk/readable-stream-helper";
+import { DataPart } from "../../src/data-chunk/yield-chunk";
+
+describe(chunkFromReadableStream.name, () => {
+  const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive";
+
+  describe("Proper chunk creation", () => {
+    let chunks: IteratorResult<DataPart>[] = [];
+    const CHUNK_SIZE = 30;
+
+    const setup = async () => {
+      const customStringStream = new ReadableStream({
+        start(controller) {
+          let string = INPUT_STRING;
+          controller.enqueue(string);
+        },
+        pull(controller) {
+          controller.close();
+        },
+        cancel() {},
+      });
+
+      let chunks: IteratorResult<DataPart>[] = [];
+      let interation = chunkFromReadableStream(customStringStream, CHUNK_SIZE);
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      chunks.push(await interation.next());
+      return chunks;
+    };
+
+    beforeAll(async (done) => {
+      chunks = await setup();
+      done();
+    });
+
+    it("should properly give the first chunk", async (done) => {
+      expect(chunks[0].done).toEqual(false);
+      expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big ");
+      expect(chunks[0].value.Body.length).toEqual(30);
+      expect(chunks[0].value.PartNumber).toEqual(1);
+      done();
+    });
+
+    it("should properly give the second (smaller) chunk", async (done) => {
+      expect(chunks[1].done).toEqual(false);
+      expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive");
+      expect(chunks[1].value.Body.length).toEqual(28);
+      expect(chunks[1].value.PartNumber).toEqual(2);
+      done();
+    });
+
+    it("should properly end the interation", async (done) => {
+      expect(chunks[2].done).toEqual(true);
+      expect(chunks[2].value).toEqual(undefined);
+      done();
+    });
+  });
+});

experimental/storage/test/data-chunk/sample.file

@@ -0,0 +1 @@
+Farmer jack realized that big yellow quilts were expensive

experimental/storage/test/data-chunk/yield-chunk.spec.ts

@@ -0,0 +1,70 @@
+import { Buffer } from "buffer";
+import { DataPart, yieldChunk } from "../../src/data-chunk/yield-chunk";
+import { Readable } from "stream";
+import { ReadableStream } from "web-streams-polyfill";
+const fs = require("fs");
+const fileStream = fs.createReadStream("./test/data-chunk/sample.file");
+
+describe(yieldChunk.name, () => {
+  const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive";
+
+  const standardChunkTests = async (input: any) => {
+    const chunker = yieldChunk(input, 30);
+    let chunks: IteratorResult<DataPart>[] = [];
+    chunks.push(await chunker.next());
+    chunks.push(await chunker.next());
+    chunks.push(await chunker.next());
+
+    // test full, first chunk
+    expect(chunks[0].done).toEqual(false);
+    expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big ");
+    expect(chunks[0].value.Body.length).toEqual(30);
+    expect(chunks[0].value.PartNumber).toEqual(1);
+
+    // test small, final chunk
+    expect(chunks[1].done).toEqual(false);
+    expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive");
+    expect(chunks[1].value.Body.length).toEqual(28);
+    expect(chunks[1].value.PartNumber).toEqual(2);
+
+    // test chunk ends
+    expect(chunks[2].done).toEqual(true);
+    expect(chunks[2].value).toEqual(undefined);
+  };
+
+  it(`Readable: ${yieldChunk.name}`, async (done) => {
+    standardChunkTests(Readable.from(INPUT_STRING));
+    done();
+  });
+
+  it(`File Stream: ${yieldChunk.name}`, async (done) => {
+    standardChunkTests(fileStream);
+    done();
+  });
+
+  it(`ReadableStream: ${yieldChunk.name}`, async (done) => {
+    const customStringStream = new ReadableStream({
+      start(controller) {
+        let string = INPUT_STRING;
+        controller.enqueue(string);
+      },
+      pull(controller) {
+        controller.close();
+      },
+      cancel() {},
+    });
+
+    standardChunkTests(customStringStream);
+    done();
+  });
+
+  it(`Buffer: ${yieldChunk.name}`, async (done) => {
+    standardChunkTests(Buffer.from(INPUT_STRING));
+    done();
+  });
+
+  it(`String: ${yieldChunk.name}`, async (done) => {
+    standardChunkTests(INPUT_STRING);
+    done();
+  });
+});