From 715aa7e1d8ea1797784d37ab706c12b133fca4f0 Mon Sep 17 00:00:00 2001 From: Alex Hinson Date: Wed, 3 Mar 2021 10:47:44 -0800 Subject: [PATCH] fix(@aws-amplify/datastore): return partial data when available (#7775) --- .../__tests__/__snapshots__/sync.test.ts.snap | 80 ++++++ packages/datastore/__tests__/sync.test.ts | 267 ++++++++++++++++++ .../datastore/src/sync/processors/sync.ts | 68 ++++- 3 files changed, 407 insertions(+), 8 deletions(-) create mode 100644 packages/datastore/__tests__/__snapshots__/sync.test.ts.snap create mode 100644 packages/datastore/__tests__/sync.test.ts diff --git a/packages/datastore/__tests__/__snapshots__/sync.test.ts.snap b/packages/datastore/__tests__/__snapshots__/sync.test.ts.snap new file mode 100644 index 00000000000..3fba0a3bbe0 --- /dev/null +++ b/packages/datastore/__tests__/__snapshots__/sync.test.ts.snap @@ -0,0 +1,80 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Sync jitteredRetry should return all data 1`] = ` +Object { + "data": Object { + "syncPosts": Object { + "items": Array [ + Object { + "id": "1", + "title": "Item 1", + }, + Object { + "id": "2", + "title": "Item 2", + }, + ], + }, + }, +} +`; + +exports[`Sync jitteredRetry should return partial data and send Hub event when datastorePartialData is set 1`] = ` +Object { + "data": Object { + "syncPosts": Object { + "items": Array [ + Object { + "id": "1", + "title": "Item 1", + }, + Object { + "id": "3", + "title": "Item 3", + }, + ], + }, + }, + "errors": Array [ + Object { + "message": "Item 2 error", + }, + ], +} +`; + +exports[`Sync jitteredRetry should throw error and NOT return data or send Hub event when datastorePartialData is not set 1`] = ` +Object { + "data": Object { + "syncPosts": Object { + "items": Array [ + Object { + "id": "1", + "title": "Item 1", + }, + null, + Object { + "id": "3", + "title": "Item 3", + }, + ], + }, + }, + "errors": Array [ + Object { + "message": "Item 2 error", + }, + ], +} +`; + +exports[`Sync jitteredRetry should throw error if no data is returned 1`] = ` +Object { + "data": null, + "errors": Array [ + Object { + "message": "General error", + }, + ], +} +`; diff --git a/packages/datastore/__tests__/sync.test.ts b/packages/datastore/__tests__/sync.test.ts new file mode 100644 index 00000000000..b1fe228fd30 --- /dev/null +++ b/packages/datastore/__tests__/sync.test.ts @@ -0,0 +1,267 @@ +// These tests should be replaced once SyncEngine.partialDataFeatureFlagEnabled is removed. + +const sessionStorageMock = (() => { + let store = {}; + + return { + getItem(key) { + return store[key] || null; + }, + setItem(key, value) { + store[key] = value.toString(); + }, + removeItem(key) { + delete store[key]; + }, + clear() { + store = {}; + }, + }; +})(); + +Object.defineProperty(window, 'sessionStorage', { + value: sessionStorageMock, +}); + +describe('Sync', () => { + describe('jitteredRetry', () => { + const defaultQuery = `query { + syncPosts { + items { + id + title + count + _version + _lastChangedAt + _deleted + } + nextToken + startedAt + } + }`; + const defaultVariables = {}; + const defaultOpName = 'syncPosts'; + const defaultModelDefinition = { name: 'Post' }; + + beforeEach(() => { + window.sessionStorage.clear(); + jest.resetModules(); + jest.resetAllMocks(); + }); + + it('should return all data', async () => { + window.sessionStorage.setItem('datastorePartialData', 'true'); + const resolveResponse = { + data: { + syncPosts: { + items: [ + { + id: '1', + title: 'Item 1', + }, + { + id: '2', + title: 'Item 2', + }, + ], + }, + }, + }; + + const SyncProcessor = jitteredRetrySyncProcessorSetup({ + resolveResponse, + }); + + const data = await SyncProcessor.jitteredRetry({ + query: defaultQuery, + variables: defaultVariables, + opName: defaultOpName, + modelDefinition: defaultModelDefinition, + }); + + expect(data).toMatchSnapshot(); + }); + + it('should return partial data and send Hub event when datastorePartialData is set', async () => { + window.sessionStorage.setItem('datastorePartialData', 'true'); + const rejectResponse = { + data: { + syncPosts: { + items: [ + { + id: '1', + title: 'Item 1', + }, + null, + { + id: '3', + title: 'Item 3', + }, + ], + }, + }, + errors: [ + { + message: 'Item 2 error', + }, + ], + }; + + const hubDispatchMock = jest.fn(); + const coreMocks = { + Hub: { + dispatch: hubDispatchMock, + listen: jest.fn(), + }, + }; + + const SyncProcessor = jitteredRetrySyncProcessorSetup({ + rejectResponse, + coreMocks, + }); + + const data = await SyncProcessor.jitteredRetry({ + query: defaultQuery, + variables: defaultVariables, + opName: defaultOpName, + modelDefinition: defaultModelDefinition, + }); + + expect(data).toMatchSnapshot(); + + expect(hubDispatchMock).toHaveBeenCalledWith('datastore', { + event: 'syncQueriesPartialSyncError', + data: { + errors: [ + { + message: 'Item 2 error', + }, + ], + modelName: 'Post', + }, + }); + }); + + it('should throw error and NOT return data or send Hub event when datastorePartialData is not set', async () => { + const rejectResponse = { + data: { + syncPosts: { + items: [ + { + id: '1', + title: 'Item 1', + }, + null, + { + id: '3', + title: 'Item 3', + }, + ], + }, + }, + errors: [ + { + message: 'Item 2 error', + }, + ], + }; + + const hubDispatchMock = jest.fn(); + const coreMocks = { + Hub: { + dispatch: hubDispatchMock, + listen: jest.fn(), + }, + }; + + const SyncProcessor = jitteredRetrySyncProcessorSetup({ + rejectResponse, + coreMocks, + }); + + try { + await SyncProcessor.jitteredRetry({ + query: defaultQuery, + variables: defaultVariables, + opName: defaultOpName, + modelDefinition: defaultModelDefinition, + }); + } catch (e) { + expect(e).toMatchSnapshot(); + } + }); + + it('should throw error if no data is returned', async () => { + window.sessionStorage.setItem('datastorePartialData', 'true'); + const rejectResponse = { + data: null, + errors: [ + { + message: 'General error', + }, + ], + }; + + const SyncProcessor = jitteredRetrySyncProcessorSetup({ + rejectResponse, + }); + + try { + await SyncProcessor.jitteredRetry({ + query: defaultQuery, + variables: defaultVariables, + opName: defaultOpName, + modelDefinition: defaultModelDefinition, + }); + } catch (e) { + expect(e).toMatchSnapshot(); + } + }); + }); +}); + +function jitteredRetrySyncProcessorSetup({ + rejectResponse, + resolveResponse, + coreMocks, +}: { + rejectResponse?: any; + resolveResponse?: any; + coreMocks?: object; +}) { + jest.mock('@aws-amplify/api', () => ({ + graphql: () => + new Promise((res, rej) => { + if (resolveResponse) { + res(resolveResponse); + } else if (rejectResponse) { + rej(rejectResponse); + } + }), + })); + + jest.mock('@aws-amplify/core', () => ({ + ...jest.requireActual('@aws-amplify/core'), + // No need to retry any thrown errors right now, + // so we're overriding jitteredExponentialRetry + jitteredExponentialRetry: (fn, args) => fn(...args), + ...coreMocks, + })); + + const SyncProcessorClass = require('../src/sync/processors/sync') + .SyncProcessor; + + const testInternalSchema = { + namespaces: {}, + version: '', + }; + + const SyncProcessor = new SyncProcessorClass( + testInternalSchema, + 1000, // default maxRecordsToSync + 10000, // default syncPageSize + null + ); + + return SyncProcessor; +} diff --git a/packages/datastore/src/sync/processors/sync.ts b/packages/datastore/src/sync/processors/sync.ts index e8e577ecfeb..7d660ce66b7 100644 --- a/packages/datastore/src/sync/processors/sync.ts +++ b/packages/datastore/src/sync/processors/sync.ts @@ -12,14 +12,15 @@ import { buildGraphQLOperation, predicateToGraphQLFilter } from '../utils'; import { jitteredExponentialRetry, ConsoleLogger as Logger, + Hub, } from '@aws-amplify/core'; import { ModelPredicateCreator } from '../../predicates'; -const logger = new Logger('DataStore'); - const DEFAULT_PAGINATION_LIMIT = 1000; const DEFAULT_MAX_RECORDS_TO_SYNC = 10000; +const logger = new Logger('DataStore'); + class SyncProcessor { private readonly typeQuery = new WeakMap(); @@ -90,7 +91,12 @@ class SyncProcessor { startedAt: number; }; }> - >await this.jitteredRetry(query, variables, opName); + >await this.jitteredRetry({ + query, + variables, + opName, + modelDefinition, + }); const { [opName]: opResult } = data; @@ -99,11 +105,27 @@ class SyncProcessor { return { nextToken: newNextToken, startedAt, items }; } - private async jitteredRetry( - query: string, - variables: { limit: number; lastSync: number; nextToken: string }, - opName: string - ): Promise< + // Partial data private feature flag. Not a public API. This will be removed in a future release. + private partialDataFeatureFlagEnabled() { + try { + const flag = sessionStorage.getItem('datastorePartialData'); + return Boolean(flag); + } catch (e) { + return false; + } + } + + private async jitteredRetry({ + query, + variables, + opName, + modelDefinition, + }: { + query: string; + variables: { limit: number; lastSync: number; nextToken: string }; + opName: string; + modelDefinition: SchemaModel; + }): Promise< GraphQLResult<{ [opName: string]: { items: T[]; @@ -120,6 +142,36 @@ class SyncProcessor { variables, }); } catch (error) { + if (this.partialDataFeatureFlagEnabled()) { + const hasItems = Boolean( + error && + error.data && + error.data[opName] && + error.data[opName].items + ); + + if (hasItems) { + const result = error; + result.data[opName].items = result.data[opName].items.filter( + item => item !== null + ); + + if (error.errors) { + Hub.dispatch('datastore', { + event: 'syncQueriesPartialSyncError', + data: { + errors: error.errors, + modelName: modelDefinition.name, + }, + }); + } + + return result; + } else { + throw error; + } + } + // If the error is unauthorized, filter out unauthorized items and return accessible items const unauthorized = (error.errors as [any]).some( err => err.errorType === 'Unauthorized'