From 0bb97bf89c5f09671d3f4b026bc4da53db14a8b1 Mon Sep 17 00:00:00 2001 From: streamich Date: Sat, 27 Jul 2024 12:58:09 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20add=20streaming=20read?= =?UTF-8?q?=20support=20ot=20CRUD=20FS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- src/crud/__tests__/testCrudfs.ts | 7 ++++--- src/crud/types.ts | 2 +- src/fsa-to-crud/FsaCrud.ts | 11 ++++++++--- src/node-to-crud/NodeCrud.ts | 26 +++++++++++++++++++++++--- yarn.lock | 10 +++++----- 6 files changed, 42 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index cac06be..2883b36 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "dependencies": { "@jsonjoy.com/json-pack": "^1.0.4", "@jsonjoy.com/util": "^1.3.0", - "memfs": "^4.9.4", + "memfs": "^4.10.0", "sonic-forest": "^1.0.3", "thingies": "^2.1.1", "tslib": "^2.6.3" diff --git a/src/crud/__tests__/testCrudfs.ts b/src/crud/__tests__/testCrudfs.ts index 49d39dd..debee74 100644 --- a/src/crud/__tests__/testCrudfs.ts +++ b/src/crud/__tests__/testCrudfs.ts @@ -1,5 +1,6 @@ import { of } from 'thingies'; import { fromStream } from '@jsonjoy.com/util/lib/streams/fromStream'; +import { bufferToUint8Array } from '@jsonjoy.com/util/lib/buffers/bufferToUint8Array'; import type { CrudApi, CrudCollectionEntry } from '../types'; export type Setup = () => { @@ -173,12 +174,12 @@ export const testCrudfs = (setup: Setup) => { }); }); - describe('.getFile()', () => { + describe('.getStream()', () => { test('can fetch an existing resource as stream', async () => { const { crud } = setup(); await crud.put(['foo'], 'bar', b('abc')); - const file = await crud.getFile(['foo'], 'bar'); - const blob = await fromStream(file.stream()); + const stream = await crud.getStream(['foo'], 'bar'); + const blob = bufferToUint8Array(await fromStream(stream) as any); expect(blob).toStrictEqual(b('abc')); }); }); diff --git a/src/crud/types.ts b/src/crud/types.ts index a5e0e54..0594e8c 100644 --- a/src/crud/types.ts +++ b/src/crud/types.ts @@ -25,7 +25,7 @@ export interface CrudApi { * @param id Id of the resource, document name. * @returns Blob content of the resource. */ - getFile: (collection: CrudCollection, id: string) => Promise; + getStream: (collection: CrudCollection, id: string) => Promise; /** * Deletes a resource. diff --git a/src/fsa-to-crud/FsaCrud.ts b/src/fsa-to-crud/FsaCrud.ts index 3818932..7774477 100644 --- a/src/fsa-to-crud/FsaCrud.ts +++ b/src/fsa-to-crud/FsaCrud.ts @@ -98,16 +98,21 @@ export class FsaCrud implements crud.CrudApi { await writable.close(); }; - public readonly getFile = async (collection: crud.CrudCollection, id: string): Promise => { + public async _get(collection: crud.CrudCollection, id: string): Promise { assertType(collection, 'get', 'crudfs'); assertName(id, 'get', 'crudfs'); const [, file] = await this.__resolve(collection, id); return await file.getFile(); + } + + public readonly getStream = async (collection: crud.CrudCollection, id: string): Promise => { + const file = await this._get(collection, id); + return file.stream(); }; public readonly get = async (collection: crud.CrudCollection, id: string): Promise => { - const blob = await this.getFile(collection, id); - const buffer = await blob.arrayBuffer(); + const file = await this._get(collection, id); + const buffer = await file.arrayBuffer(); return new Uint8Array(buffer); }; diff --git a/src/node-to-crud/NodeCrud.ts b/src/node-to-crud/NodeCrud.ts index 2a3eab2..3e906b8 100644 --- a/src/node-to-crud/NodeCrud.ts +++ b/src/node-to-crud/NodeCrud.ts @@ -4,7 +4,7 @@ import { FLAG } from 'memfs/lib/consts/FLAG'; import { newExistsError, newFile404Error, newFolder404Error, newMissingError } from '../fsa-to-crud/util'; import type { FsPromisesApi } from 'memfs/lib/node/types'; import type * as crud from '../crud/types'; -import type { IDirent, IFileHandle } from 'memfs/lib/node/types/misc'; +import type { IDirent, IFileHandle, TFlags } from 'memfs/lib/node/types/misc'; export interface NodeCrudOptions { readonly fs: FsPromisesApi; @@ -98,14 +98,34 @@ export class NodeCrud implements crud.CrudApi { } }; - public readonly get = async (collection: crud.CrudCollection, id: string): Promise => { + public async _file(collection: crud.CrudCollection, id: string, flags: TFlags): Promise { assertType(collection, 'get', 'crudfs'); assertName(id, 'get', 'crudfs'); const dir = await this.checkDir(collection); const filename = dir + id; const fs = this.fs; + return await fs.open(filename, flags); + }; + + public readonly getStream = async (collection: crud.CrudCollection, id: string): Promise => { + try { + const handle = await this._file(collection, id, FLAG.O_RDONLY); + return handle.readableWebStream(); + } catch (error) { + if (error && typeof error === 'object') { + switch (error.code) { + case 'ENOENT': + throw newFile404Error(collection, id); + } + } + throw error; + } + }; + + public readonly get = async (collection: crud.CrudCollection, id: string): Promise => { try { - const buf = (await fs.readFile(filename)) as Buffer; + const handle = await this._file(collection, id, FLAG.O_RDONLY); + const buf = await handle.readFile() as Buffer; return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); } catch (error) { if (error && typeof error === 'object') { diff --git a/yarn.lock b/yarn.lock index e9df911..a81655b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3156,13 +3156,13 @@ memfs@^3.4.3: dependencies: fs-monkey "^1.0.4" -memfs@^4.9.4: - version "4.9.4" - resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.9.4.tgz#803eb7f2091d1c6198ec9ba9b582505ad8699c9e" - integrity sha512-Xlj8b2rU11nM6+KU6wC7cuWcHQhVINWCUgdPS4Ar9nPxLaOya3RghqK7ALyDW2QtGebYAYs6uEdEVnwPVT942A== +memfs@^4.10.0: + version "4.10.0" + resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.10.0.tgz#f691ac909b679e0be6e29a6a15d8db4f9160a3f2" + integrity sha512-CkZUf+y0Ph/hu+mh0LPFrzDTPvGxUOjEqs4/DkIjFACbrCx8gT3VjqPY6EEtl5leC6JOkrm4wOlhQo8028Tj1g== dependencies: "@jsonjoy.com/json-pack" "^1.0.3" - "@jsonjoy.com/util" "^1.1.2" + "@jsonjoy.com/util" "^1.3.0" tree-dump "^1.0.1" tslib "^2.0.0"