From e9b0cad886b069e60bceca2b1e81fe3c1ccd925e Mon Sep 17 00:00:00 2001 From: Alex Forsyth Date: Fri, 2 Oct 2020 15:53:14 -0400 Subject: [PATCH] feat: data chunking and stream parsing tests --- .../test/data-chunk/buffer-chunk.spec.ts | 47 +++++ .../test/data-chunk/readable-helper.spec.ts | 160 ++++++++++++++++++ .../data-chunk/readable-stream-helper.spec.ts | 59 +++++++ .../storage/test/data-chunk/sample.file | 1 + .../test/data-chunk/yield-chunk.spec.ts | 70 ++++++++ 5 files changed, 337 insertions(+) create mode 100644 experimental/storage/test/data-chunk/buffer-chunk.spec.ts create mode 100644 experimental/storage/test/data-chunk/readable-helper.spec.ts create mode 100644 experimental/storage/test/data-chunk/readable-stream-helper.spec.ts create mode 100644 experimental/storage/test/data-chunk/sample.file create mode 100644 experimental/storage/test/data-chunk/yield-chunk.spec.ts diff --git a/experimental/storage/test/data-chunk/buffer-chunk.spec.ts b/experimental/storage/test/data-chunk/buffer-chunk.spec.ts new file mode 100644 index 0000000000000..d42087bbb05c6 --- /dev/null +++ b/experimental/storage/test/data-chunk/buffer-chunk.spec.ts @@ -0,0 +1,47 @@ +import { chunkFromBuffer } from "../../src/data-chunk/buffer-helper"; +import { DataPart } from "../../src/data-chunk/yield-chunk"; + +describe(chunkFromBuffer.name, () => { + const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive"; + + describe("String chunking", () => { + it("should come back with small sub buffers", async (done) => { + const length = 10; + const chunker = chunkFromBuffer(Buffer.from(INPUT_STRING), length); + const result = await chunker.next(); + const dataChunk = result.value as DataPart; + expect(dataChunk.PartNumber).toEqual(1); + expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(0, length)); + + done(); + }); + + it("should come back with many chunks on subsequent calls", async (done) => { + const length = 10; + const chunker = chunkFromBuffer(Buffer.from(INPUT_STRING), length); + + let result = await chunker.next(); + let dataChunk = result.value as DataPart; + expect(dataChunk.PartNumber).toEqual(1); + expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(0, length)); + + result = await chunker.next(); + dataChunk = result.value as DataPart; + expect(dataChunk.PartNumber).toEqual(2); + expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(length, length * 2)); + + done(); + }); + + it("should have the last sub buffer be the remainder length", async (done) => { + const length = 30; + const chunker = chunkFromBuffer(Buffer.from(INPUT_STRING), length); + let result = await chunker.next(); + result = await chunker.next(); + const dataChunk = result.value as DataPart; + expect(dataChunk.Body.toString()).toEqual(INPUT_STRING.substring(length)); + + done(); + }); + }); +}); diff --git a/experimental/storage/test/data-chunk/readable-helper.spec.ts b/experimental/storage/test/data-chunk/readable-helper.spec.ts new file mode 100644 index 0000000000000..7ff8ae3cae8e2 --- /dev/null +++ b/experimental/storage/test/data-chunk/readable-helper.spec.ts @@ -0,0 +1,160 @@ +import { Readable } from "stream"; +import { chunkFromReadable } from "../../src/data-chunk/readable-helper"; +import { DataPart } from "../../src/data-chunk/yield-chunk"; +const fs = require("fs"); +const fileStream = fs.createReadStream( + "/Users/alexfors/Documents/workspace/sdk/alexfors/aws-sdk-js-v3/experimental/storage/test/data-chunk/sample.file" +); + +describe(chunkFromReadable.name, () => { + const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive"; + + describe("Proper chunk creation", () => { + let chunks: IteratorResult[] = []; + const CHUNK_SIZE = 30; + + const setup = async () => { + const chunks: IteratorResult[] = []; + const readableStream = Readable.from(INPUT_STRING); + const interation = chunkFromReadable(readableStream, CHUNK_SIZE); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + return chunks; + }; + + beforeAll(async (done) => { + chunks = await setup(); + done(); + }); + + it("should properly give the first chunk", async (done) => { + expect(chunks[0].done).toEqual(false); + expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big "); + expect(chunks[0].value.Body.length).toEqual(30); + expect(chunks[0].value.PartNumber).toEqual(1); + done(); + }); + + it("should properly give the second (smaller) chunk", async (done) => { + expect(chunks[1].done).toEqual(false); + expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive"); + expect(chunks[1].value.Body.length).toEqual(28); + expect(chunks[1].value.PartNumber).toEqual(2); + done(); + }); + + it("should properly end the interation", async (done) => { + expect(chunks[2].done).toEqual(true); + expect(chunks[2].value).toEqual(undefined); + done(); + }); + }); + + describe("Proper single chunk", () => { + let chunks: IteratorResult[] = []; + const CHUNK_SIZE = 1000; + + const setup = async () => { + const chunks: IteratorResult[] = []; + const readableStream = Readable.from(INPUT_STRING); + const interation = chunkFromReadable(readableStream, CHUNK_SIZE); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + return chunks; + }; + + beforeAll(async (done) => { + chunks = await setup(); + done(); + }); + + it("should properly give the first chunk as all the data", async (done) => { + expect(CHUNK_SIZE).toBeGreaterThan(INPUT_STRING.length); + expect(chunks[0].done).toEqual(false); + expect(chunks[0].value.Body.toString()).toEqual(INPUT_STRING); + expect(chunks[0].value.Body.length).toEqual(INPUT_STRING.length); + expect(chunks[0].value.PartNumber).toEqual(1); + done(); + }); + + it("should properly end the interation", async (done) => { + expect(chunks[1].done).toEqual(true); + expect(chunks[1].value).toEqual(undefined); + done(); + }); + }); + + describe("Proper no data chunk", () => { + let chunks: IteratorResult[] = []; + const CHUNK_SIZE = 10; + + const setup = async () => { + const chunks: IteratorResult[] = []; + const readableStream = Readable.from([]); + const interation = chunkFromReadable(readableStream, CHUNK_SIZE); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + return chunks; + }; + + beforeAll(async (done) => { + chunks = await setup(); + done(); + }); + + it("should return an empty object", async (done) => { + expect(chunks[0].done).toEqual(false); + expect(chunks[0].value.Body.toString()).toEqual(""); + done(); + }); + + it("should properly end the interation", async (done) => { + expect(chunks[1].done).toEqual(true); + expect(chunks[1].value).toEqual(undefined); + done(); + }); + }); + + describe("Properly chunks files", () => { + let chunks: IteratorResult[] = []; + const CHUNK_SIZE = 30; + + const setup = async () => { + const chunks: IteratorResult[] = []; + const interation = chunkFromReadable(fileStream, CHUNK_SIZE); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + return chunks; + }; + + beforeAll(async (done) => { + chunks = await setup(); + expect(1).toEqual(1); + done(); + }); + + it("should properly give the first chunk", async (done) => { + expect(chunks[0].done).toEqual(false); + expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big "); + expect(chunks[0].value.Body.length).toEqual(30); + expect(chunks[0].value.PartNumber).toEqual(1); + done(); + }); + + // it("should properly give the second (smaller) chunk", async (done) => { + // expect(chunks[1].done).toEqual(false); + // expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive"); + // expect(chunks[1].value.Body.length).toEqual(28); + // expect(chunks[1].value.PartNumber).toEqual(2); + // done(); + // }); + + // it("should properly end the interation", async (done) => { + // expect(chunks[2].done).toEqual(true); + // expect(chunks[2].value).toEqual(undefined); + // done(); + // }); + }); +}); diff --git a/experimental/storage/test/data-chunk/readable-stream-helper.spec.ts b/experimental/storage/test/data-chunk/readable-stream-helper.spec.ts new file mode 100644 index 0000000000000..61cd83edc447e --- /dev/null +++ b/experimental/storage/test/data-chunk/readable-stream-helper.spec.ts @@ -0,0 +1,59 @@ +import { ReadableStream } from "web-streams-polyfill"; +import { chunkFromReadableStream } from "../../src/data-chunk/readable-stream-helper"; +import { DataPart } from "../../src/data-chunk/yield-chunk"; + +describe(chunkFromReadableStream.name, () => { + const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive"; + + describe("Proper chunk creation", () => { + let chunks: IteratorResult[] = []; + const CHUNK_SIZE = 30; + + const setup = async () => { + const customStringStream = new ReadableStream({ + start(controller) { + let string = INPUT_STRING; + controller.enqueue(string); + }, + pull(controller) { + controller.close(); + }, + cancel() {}, + }); + + let chunks: IteratorResult[] = []; + let interation = chunkFromReadableStream(customStringStream, CHUNK_SIZE); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + chunks.push(await interation.next()); + return chunks; + }; + + beforeAll(async (done) => { + chunks = await setup(); + done(); + }); + + it("should properly give the first chunk", async (done) => { + expect(chunks[0].done).toEqual(false); + expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big "); + expect(chunks[0].value.Body.length).toEqual(30); + expect(chunks[0].value.PartNumber).toEqual(1); + done(); + }); + + it("should properly give the second (smaller) chunk", async (done) => { + expect(chunks[1].done).toEqual(false); + expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive"); + expect(chunks[1].value.Body.length).toEqual(28); + expect(chunks[1].value.PartNumber).toEqual(2); + done(); + }); + + it("should properly end the interation", async (done) => { + expect(chunks[2].done).toEqual(true); + expect(chunks[2].value).toEqual(undefined); + done(); + }); + }); +}); diff --git a/experimental/storage/test/data-chunk/sample.file b/experimental/storage/test/data-chunk/sample.file new file mode 100644 index 0000000000000..5050ba8c72a4e --- /dev/null +++ b/experimental/storage/test/data-chunk/sample.file @@ -0,0 +1 @@ +Farmer jack realized that big yellow quilts were expensive \ No newline at end of file diff --git a/experimental/storage/test/data-chunk/yield-chunk.spec.ts b/experimental/storage/test/data-chunk/yield-chunk.spec.ts new file mode 100644 index 0000000000000..06a5d52bbf978 --- /dev/null +++ b/experimental/storage/test/data-chunk/yield-chunk.spec.ts @@ -0,0 +1,70 @@ +import { Buffer } from "buffer"; +import { DataPart, yieldChunk } from "../../src/data-chunk/yield-chunk"; +import { Readable } from "stream"; +import { ReadableStream } from "web-streams-polyfill"; +const fs = require("fs"); +const fileStream = fs.createReadStream("./test/data-chunk/sample.file"); + +describe(yieldChunk.name, () => { + const INPUT_STRING = "Farmer jack realized that big yellow quilts were expensive"; + + const standardChunkTests = async (input: any) => { + const chunker = yieldChunk(input, 30); + let chunks: IteratorResult[] = []; + chunks.push(await chunker.next()); + chunks.push(await chunker.next()); + chunks.push(await chunker.next()); + + // test full, first chunk + expect(chunks[0].done).toEqual(false); + expect(chunks[0].value.Body.toString()).toEqual("Farmer jack realized that big "); + expect(chunks[0].value.Body.length).toEqual(30); + expect(chunks[0].value.PartNumber).toEqual(1); + + // test small, final chunk + expect(chunks[1].done).toEqual(false); + expect(chunks[1].value.Body.toString()).toEqual("yellow quilts were expensive"); + expect(chunks[1].value.Body.length).toEqual(28); + expect(chunks[1].value.PartNumber).toEqual(2); + + // test chunk ends + expect(chunks[2].done).toEqual(true); + expect(chunks[2].value).toEqual(undefined); + }; + + it(`Readable: ${yieldChunk.name}`, async (done) => { + standardChunkTests(Readable.from(INPUT_STRING)); + done(); + }); + + it(`File Stream: ${yieldChunk.name}`, async (done) => { + standardChunkTests(fileStream); + done(); + }); + + it(`ReadableStream: ${yieldChunk.name}`, async (done) => { + const customStringStream = new ReadableStream({ + start(controller) { + let string = INPUT_STRING; + controller.enqueue(string); + }, + pull(controller) { + controller.close(); + }, + cancel() {}, + }); + + standardChunkTests(customStringStream); + done(); + }); + + it(`Buffer: ${yieldChunk.name}`, async (done) => { + standardChunkTests(Buffer.from(INPUT_STRING)); + done(); + }); + + it(`String: ${yieldChunk.name}`, async (done) => { + standardChunkTests(INPUT_STRING); + done(); + }); +});