Skip to content

Commit

Permalink
feat: 🎸 add streaming read support ot CRUD FS
Browse files Browse the repository at this point in the history
streamich committed Jul 27, 2024
1 parent f1d0abe commit 0bb97bf
Showing 6 changed files with 42 additions and 16 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -88,7 +88,7 @@
"dependencies": {
"@jsonjoy.com/json-pack": "^1.0.4",
"@jsonjoy.com/util": "^1.3.0",
"memfs": "^4.9.4",
"memfs": "^4.10.0",
"sonic-forest": "^1.0.3",
"thingies": "^2.1.1",
"tslib": "^2.6.3"
7 changes: 4 additions & 3 deletions src/crud/__tests__/testCrudfs.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { of } from 'thingies';
import { fromStream } from '@jsonjoy.com/util/lib/streams/fromStream';
import { bufferToUint8Array } from '@jsonjoy.com/util/lib/buffers/bufferToUint8Array';
import type { CrudApi, CrudCollectionEntry } from '../types';

export type Setup = () => {
@@ -173,12 +174,12 @@ export const testCrudfs = (setup: Setup) => {
});
});

describe('.getFile()', () => {
describe('.getStream()', () => {
test('can fetch an existing resource as stream', async () => {
const { crud } = setup();
await crud.put(['foo'], 'bar', b('abc'));
const file = await crud.getFile(['foo'], 'bar');
const blob = await fromStream(file.stream());
const stream = await crud.getStream(['foo'], 'bar');
const blob = bufferToUint8Array(await fromStream(stream) as any);
expect(blob).toStrictEqual(b('abc'));
});
});
2 changes: 1 addition & 1 deletion src/crud/types.ts
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ export interface CrudApi {
* @param id Id of the resource, document name.
* @returns Blob content of the resource.
*/
getFile: (collection: CrudCollection, id: string) => Promise<File>;
getStream: (collection: CrudCollection, id: string) => Promise<ReadableStream>;

/**
* Deletes a resource.
11 changes: 8 additions & 3 deletions src/fsa-to-crud/FsaCrud.ts
Original file line number Diff line number Diff line change
@@ -98,16 +98,21 @@ export class FsaCrud implements crud.CrudApi {
await writable.close();
};

public readonly getFile = async (collection: crud.CrudCollection, id: string): Promise<File> => {
public async _get(collection: crud.CrudCollection, id: string): Promise<File> {
assertType(collection, 'get', 'crudfs');
assertName(id, 'get', 'crudfs');
const [, file] = await this.__resolve(collection, id);
return await file.getFile();
}

public readonly getStream = async (collection: crud.CrudCollection, id: string): Promise<ReadableStream> => {
const file = await this._get(collection, id);
return file.stream();
};

public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
const blob = await this.getFile(collection, id);
const buffer = await blob.arrayBuffer();
const file = await this._get(collection, id);
const buffer = await file.arrayBuffer();
return new Uint8Array(buffer);
};

26 changes: 23 additions & 3 deletions src/node-to-crud/NodeCrud.ts
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ import { FLAG } from 'memfs/lib/consts/FLAG';
import { newExistsError, newFile404Error, newFolder404Error, newMissingError } from '../fsa-to-crud/util';
import type { FsPromisesApi } from 'memfs/lib/node/types';
import type * as crud from '../crud/types';
import type { IDirent, IFileHandle } from 'memfs/lib/node/types/misc';
import type { IDirent, IFileHandle, TFlags } from 'memfs/lib/node/types/misc';

export interface NodeCrudOptions {
readonly fs: FsPromisesApi;
@@ -98,14 +98,34 @@ export class NodeCrud implements crud.CrudApi {
}
};

public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
public async _file(collection: crud.CrudCollection, id: string, flags: TFlags): Promise<IFileHandle> {
assertType(collection, 'get', 'crudfs');
assertName(id, 'get', 'crudfs');
const dir = await this.checkDir(collection);
const filename = dir + id;
const fs = this.fs;
return await fs.open(filename, flags);
};

public readonly getStream = async (collection: crud.CrudCollection, id: string): Promise<ReadableStream> => {
try {
const handle = await this._file(collection, id, FLAG.O_RDONLY);
return handle.readableWebStream();
} catch (error) {
if (error && typeof error === 'object') {
switch (error.code) {
case 'ENOENT':
throw newFile404Error(collection, id);
}
}
throw error;
}
};

public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
try {
const buf = (await fs.readFile(filename)) as Buffer;
const handle = await this._file(collection, id, FLAG.O_RDONLY);
const buf = await handle.readFile() as Buffer;
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
} catch (error) {
if (error && typeof error === 'object') {
10 changes: 5 additions & 5 deletions yarn.lock
Original file line number Diff line number Diff line change
@@ -3156,13 +3156,13 @@ memfs@^3.4.3:
dependencies:
fs-monkey "^1.0.4"

memfs@^4.9.4:
version "4.9.4"
resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.9.4.tgz#803eb7f2091d1c6198ec9ba9b582505ad8699c9e"
integrity sha512-Xlj8b2rU11nM6+KU6wC7cuWcHQhVINWCUgdPS4Ar9nPxLaOya3RghqK7ALyDW2QtGebYAYs6uEdEVnwPVT942A==
memfs@^4.10.0:
version "4.10.0"
resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.10.0.tgz#f691ac909b679e0be6e29a6a15d8db4f9160a3f2"
integrity sha512-CkZUf+y0Ph/hu+mh0LPFrzDTPvGxUOjEqs4/DkIjFACbrCx8gT3VjqPY6EEtl5leC6JOkrm4wOlhQo8028Tj1g==
dependencies:
"@jsonjoy.com/json-pack" "^1.0.3"
"@jsonjoy.com/util" "^1.1.2"
"@jsonjoy.com/util" "^1.3.0"
tree-dump "^1.0.1"
tslib "^2.0.0"

0 comments on commit 0bb97bf

Please sign in to comment.