From d43c2211bbae1818bcde3a2f64746ef69111caff Mon Sep 17 00:00:00 2001 From: mariusheine Date: Mon, 22 Aug 2022 22:36:12 +0200 Subject: [PATCH] feat: add handling for paginated data and allow loading all pages at once (#77) * add handling of paginated requests/responses * run prettier * fix branch testing coverage * add test case for single entity to fix branch testing coverage * fix eslint issues * add option for chunking and add some comments to the code * fix typecheck error * fix unit tests * rename test cases a bit Co-authored-by: Anbraten * make unit test title more specific * restructured page handling * fix unit test Co-authored-by: Anbraten --- src/useFind.ts | 62 +++++++++++-- src/utils.ts | 7 +- test/useFind.test.ts | 201 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 261 insertions(+), 9 deletions(-) diff --git a/src/useFind.ts b/src/useFind.ts index 728b6f1..0255ca7 100644 --- a/src/useFind.ts +++ b/src/useFind.ts @@ -1,9 +1,10 @@ +import { AdapterService } from '@feathersjs/adapter-commons/lib'; import type { FeathersError } from '@feathersjs/errors'; -import type { Application, FeathersService, Params, ServiceMethods } from '@feathersjs/feathers'; +import type { Application, FeathersService, Paginated, Params, Query, ServiceMethods } from '@feathersjs/feathers'; import sift from 'sift'; import { getCurrentInstance, onBeforeUnmount, Ref, ref, watch } from 'vue'; -import { getId, ServiceModel, ServiceTypes } from './utils'; +import { getId, isPaginated, ServiceModel, ServiceTypes } from './utils'; function loadServiceEventHandlers< CustomApplication extends Application, @@ -87,12 +88,20 @@ export type UseFindFunc = < params?: Ref, ) => UseFind; +type Options = { + disableUnloadingEventHandlers: boolean; + loadAllPages: boolean; +}; + +const defaultOptions: Options = { disableUnloadingEventHandlers: false, loadAllPages: false }; + export default (feathers: CustomApplication) => , M = ServiceModel>( serviceName: T, params: Ref = ref({ paginate: false, query: {} }), - { disableUnloadingEventHandlers } = { disableUnloadingEventHandlers: false }, + options: Partial = {}, ): UseFind => { + const { disableUnloadingEventHandlers, loadAllPages } = { ...defaultOptions, ...options }; // type cast is fine here (source: https://github.com/vuejs/vue-next/issues/2136#issuecomment-693524663) const data = ref([]) as Ref; const isLoading = ref(false); @@ -100,8 +109,11 @@ export default (feathers: CustomApplicati const service = feathers.service(serviceName as string); const unloadEventHandlers = loadServiceEventHandlers(service, params, data); + let unloaded = false; + + const currentFindCall = ref(0); - const find = async () => { + const find = async (call: number) => { isLoading.value = true; error.value = undefined; @@ -112,10 +124,44 @@ export default (feathers: CustomApplicati } try { + const originalParams: Params = params.value; + const originalQuery: Query & { $limit?: number } = originalParams.query || {}; // TODO: the typecast below is necessary due to the prerelease state of feathers v5. The problem there is // that the AdapterService interface is not yet updated and is not compatible with the ServiceMethods interface. - const res = await (service as unknown as ServiceMethods).find(params.value); - data.value = Array.isArray(res) ? res : [res]; + const res = await (service as unknown as ServiceMethods | AdapterService).find(originalParams); + if (call !== currentFindCall.value) { + // stop handling response since there already is a new find call running within this composition + return; + } + if (isPaginated(res) && !loadAllPages) { + data.value = [...res.data]; + } else if (!isPaginated(res)) { + data.value = Array.isArray(res) ? res : [res]; + } else { + // extract data from page response + let loadedPage: Paginated = res; + let loadedItemsCount = loadedPage.data.length; + data.value = [...loadedPage.data]; + // limit might not be specified in the original query if default pagination from backend is applied, that's why we use this fallback pattern + const limit: number = originalQuery.$limit || loadedPage.data.length; + // if chunking is enabled we go on requesting all following pages until all data have been received + while (!unloaded && loadedPage.total > loadedItemsCount) { + // skip can be a string in cases where key based chunking/pagination is done e.g. in DynamoDb via `LastEvaluatedKey` + const skip: string | number = + typeof loadedPage.skip === 'string' ? loadedPage.skip : loadedPage.skip + limit; + // request next page + loadedPage = (await (service as unknown as ServiceMethods | AdapterService).find({ + ...originalParams, + query: { ...originalQuery, $skip: skip, $limit: limit }, + })) as Paginated; + if (call !== currentFindCall.value) { + // stop handling/requesting further pages since there already is a new find call running within this composition + return; + } + loadedItemsCount += loadedPage.data.length; + data.value = [...data.value, ...loadedPage.data]; + } + } } catch (_error) { error.value = _error as FeathersError; } @@ -124,10 +170,12 @@ export default (feathers: CustomApplicati }; const load = () => { - void find(); + currentFindCall.value = currentFindCall.value + 1; + void find(currentFindCall.value); }; const unload = () => { + unloaded = true; unloadEventHandlers(); feathers.off('connect', load); }; diff --git a/src/utils.ts b/src/utils.ts index dee23c4..e32ccd5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,5 @@ import type { AdapterService } from '@feathersjs/adapter-commons'; -import type { Application, Id, ServiceMethods } from '@feathersjs/feathers'; +import type { Application, Id, Paginated, ServiceMethods } from '@feathersjs/feathers'; export type PotentialIds = { id?: Id; @@ -29,3 +29,8 @@ export type ServiceModel< : ServiceTypes[T] extends ServiceMethods ? M2 : never; + +export function isPaginated(response: T | T[] | Paginated): response is Paginated { + const { total, limit, skip, data } = response as Paginated; + return total !== undefined && limit !== undefined && skip !== undefined && data !== undefined && Array.isArray(data); +} diff --git a/test/useFind.test.ts b/test/useFind.test.ts index db19d02..49c962c 100644 --- a/test/useFind.test.ts +++ b/test/useFind.test.ts @@ -1,5 +1,5 @@ import { FeathersError, GeneralError } from '@feathersjs/errors'; -import type { Application, Params } from '@feathersjs/feathers'; +import type { Application, Paginated, Params } from '@feathersjs/feathers'; import { flushPromises } from '@vue/test-utils'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { nextTick, ref } from 'vue'; @@ -423,6 +423,36 @@ describe('Find composition', () => { expect(findComposition && findComposition.error.value).toBeTruthy(); }); + it('should load single entity response', async () => { + expect.assertions(3); + + // given + const serviceFind = vi.fn(() => testModel); + + const feathersMock = { + service: () => ({ + find: serviceFind, + on: vi.fn(), + off: vi.fn(), + }), + on: vi.fn(), + off: vi.fn(), + } as unknown as Application; + const useFind = useFindOriginal(feathersMock); + + // when + let findComposition = null as UseFind | null; + mountComposition(() => { + findComposition = useFind('testModels'); + }); + await nextTick(); + + // then + expect(serviceFind).toHaveBeenCalledTimes(1); + expect(findComposition).toBeTruthy(); + expect(findComposition && findComposition.data.value).toStrictEqual([testModel]); + }); + describe('Event Handlers', () => { it('should listen to "create" events', () => { expect.assertions(2); @@ -935,4 +965,173 @@ describe('Find composition', () => { expect(serviceOff).toHaveBeenCalledTimes(4); // unload of: created, updated, patched, removed events }); }); + + describe('pagination', () => { + it('should handle paginated data', async () => { + expect.assertions(3); + + // given + let startItemIndex = 0; + const serviceFind = vi.fn(() => { + const page: Paginated = { + total: testModels.length, + skip: startItemIndex, + limit: 1, + data: testModels.slice(startItemIndex, startItemIndex + 1), + }; + startItemIndex++; + return page; + }); + + const feathersMock = { + service: () => ({ + find: serviceFind, + on: vi.fn(), + off: vi.fn(), + }), + on: vi.fn(), + off: vi.fn(), + } as unknown as Application; + const useFind = useFindOriginal(feathersMock); + + // when + let findComposition = null as UseFind | null; + mountComposition(() => { + findComposition = useFind('testModels'); + }); + await nextTick(); + + // then + expect(serviceFind).toHaveBeenCalledTimes(1); + expect(findComposition).toBeTruthy(); + expect(findComposition && findComposition.data.value).toStrictEqual(testModels.slice(0, 1)); + }); + + it('should load all data with chunking', async () => { + expect.assertions(3); + + // given + let startItemIndex = 0; + const serviceFind = vi.fn(() => { + const page: Paginated = { + total: testModels.length, + skip: startItemIndex, + limit: 1, + data: testModels.slice(startItemIndex, startItemIndex + 1), + }; + startItemIndex++; + return page; + }); + + const feathersMock = { + service: () => ({ + find: serviceFind, + on: vi.fn(), + off: vi.fn(), + }), + on: vi.fn(), + off: vi.fn(), + } as unknown as Application; + const useFind = useFindOriginal(feathersMock); + + // when + let findComposition = null as UseFind | null; + mountComposition(() => { + findComposition = useFind('testModels', undefined, { loadAllPages: true }); + }); + await nextTick(); + + // then + expect(serviceFind).toHaveBeenCalledTimes(2); + expect(findComposition).toBeTruthy(); + expect(findComposition && findComposition.data.value).toStrictEqual(testModels); + }); + + it('should load data with pagination using lastEvaluatedKey patterns', async () => { + expect.assertions(3); + + // given + const serviceFind = vi.fn((params?: Params) => { + const startItemIndex = testModels.findIndex(({ _id }) => _id === params?.query?.$skip) + 1; + const data = testModels.slice(startItemIndex, startItemIndex + 1); + const page: Paginated = { + total: testModels.length, + skip: data[data.length - 1]._id as unknown as number, + limit: 1, + data, + }; + return page; + }); + + const feathersMock = { + service: () => ({ + find: serviceFind, + on: vi.fn(), + off: vi.fn(), + }), + on: vi.fn(), + off: vi.fn(), + } as unknown as Application; + const useFind = useFindOriginal(feathersMock); + + // when + let findComposition = null as UseFind | null; + mountComposition(() => { + findComposition = useFind('testModels', undefined, { loadAllPages: true }); + }); + await nextTick(); + + // then + expect(serviceFind).toHaveBeenCalledTimes(2); + expect(findComposition).toBeTruthy(); + expect(findComposition && findComposition.data.value).toStrictEqual(testModels); + }); + + it('should stop further page requests if find was retriggered due to a change to params or connection reset', async () => { + expect.assertions(3); + + // given + let startItemIndex = 0; + let data = [additionalTestModel, ...testModels]; + const serviceFind = vi.fn(() => { + const page: Paginated = { + total: data.length, + skip: startItemIndex, + limit: 1, + data: data.slice(startItemIndex, startItemIndex + 1), + }; + startItemIndex++; + return page; + }); + const emitter = eventHelper(); + const feathersMock = { + service: () => ({ + find: serviceFind, + on: vi.fn(), + off: vi.fn(), + }), + on: emitter.on, + off: vi.fn(), + } as unknown as Application; + const useFind = useFindOriginal(feathersMock); + let findComposition = null as UseFind | null; + mountComposition(() => { + findComposition = useFind('testModels', undefined, { loadAllPages: true }); + }); + await nextTick(); + serviceFind.mockClear(); + data = testModels; + startItemIndex = 0; + + // when + emitter.emit('connect'); + await nextTick(); + await nextTick(); + + // then + expect(serviceFind).toHaveBeenCalledTimes(2); + expect(findComposition).toBeTruthy(); + expect(findComposition && findComposition.data.value).toStrictEqual(testModels); + }); + }); });