From 6f0da3d7043edbad4fd14a69543f00faa88dfdab Mon Sep 17 00:00:00 2001 From: Mathieu Artu Date: Mon, 23 Sep 2024 09:16:24 +0200 Subject: [PATCH 1/2] feat(NOTIFY-1128): add batch PUT endpoint support --- .../user-storage/UserStorageController.ts | 28 ++++++ .../__fixtures__/mockResponses.ts | 10 +++ .../user-storage/__fixtures__/mockServices.ts | 20 ++++- .../controllers/user-storage/services.test.ts | 90 ++++++++++++++++++- .../src/controllers/user-storage/services.ts | 47 +++++++++- .../src/sdk/__fixtures__/mock-userstorage.ts | 9 +- .../src/sdk/user-storage.test.ts | 77 ++++++++++++++++ .../src/sdk/user-storage.ts | 55 ++++++++++++ .../src/shared/storage-schema.ts | 7 +- 9 files changed, 333 insertions(+), 10 deletions(-) diff --git a/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts b/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts index 3b868ea01f..6402711da5 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts @@ -25,6 +25,7 @@ import { createSHA256Hash } from '../../shared/encryption'; import type { UserStoragePathWithFeatureAndKey, UserStoragePathWithFeatureOnly, + UserStoragePathWithKeyOnly, } from '../../shared/storage-schema'; import type { NativeScrypt } from '../../shared/types/encryption'; import { createSnapSignMessageRequest } from '../authentication/auth-snap-requests'; @@ -42,6 +43,7 @@ import { } from './accounts/user-storage'; import { startNetworkSyncing } from './network-syncing/controller-integration'; import { + batchUpsertUserStorage, getUserStorage, getUserStorageAllFeatureEntries, upsertUserStorage, @@ -635,6 +637,32 @@ export default class UserStorageController extends BaseController< }); } + /** + * Allows storage of multiple user data entries for one specific feature. Data stored must be string formatted. + * Developers can extend the entry path and entry name through the `schema.ts` file. + * + * @param path - string in the form of `${feature}` that matches schema + * @param values - data to store, in the form of an array of `[entryKey, entryValue]` pairs + * @returns nothing. NOTE that an error is thrown if fails to store data. + */ + + public async performBatchSetStorage( + path: UserStoragePathWithFeatureOnly, + values: [UserStoragePathWithKeyOnly, string][], + ): Promise { + this.#assertProfileSyncingEnabled(); + + const { bearerToken, storageKey } = + await this.#getStorageKeyAndBearerToken(); + + await batchUpsertUserStorage(values, { + path, + bearerToken, + storageKey, + nativeScryptCrypto: this.#nativeScryptCrypto, + }); + } + /** * Retrieves the storage key, for internal use only! * diff --git a/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockResponses.ts b/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockResponses.ts index 94e9229215..4faba5dc6f 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockResponses.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockResponses.ts @@ -80,3 +80,13 @@ export const getMockUserStoragePutResponse = ( response: null, } satisfies MockResponse; }; + +export const getMockUserStorageBatchPutResponse = ( + path: UserStoragePathWithFeatureOnly = 'notifications', +) => { + return { + url: getMockUserStorageEndpoint(path), + requestMethod: 'PUT', + response: null, + } satisfies MockResponse; +}; diff --git a/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockServices.ts b/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockServices.ts index 57fbc9a944..b798ade0b8 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockServices.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/__fixtures__/mockServices.ts @@ -8,6 +8,7 @@ import { getMockUserStorageGetResponse, getMockUserStoragePutResponse, getMockUserStorageAllFeatureEntriesResponse, + getMockUserStorageBatchPutResponse, } from './mockResponses'; type MockReply = { @@ -52,10 +53,27 @@ export const mockEndpointGetUserStorage = async ( export const mockEndpointUpsertUserStorage = ( path: UserStoragePathWithFeatureAndKey = 'notifications.notification_settings', mockReply?: Pick, + expectCallback?: (requestBody: nock.Body) => Promise, ) => { const mockResponse = getMockUserStoragePutResponse(path); const mockEndpoint = nock(mockResponse.url) .put('') - .reply(mockReply?.status ?? 204); + .reply(mockReply?.status ?? 204, async (_, requestBody) => { + await expectCallback?.(requestBody); + }); + return mockEndpoint; +}; + +export const mockEndpointBatchUpsertUserStorage = ( + path: UserStoragePathWithFeatureOnly = 'notifications', + mockReply?: Pick, + callback?: (uri: string, requestBody: nock.Body) => Promise, +) => { + const mockResponse = getMockUserStorageBatchPutResponse(path); + const mockEndpoint = nock(mockResponse.url) + .put('') + .reply(mockReply?.status ?? 204, async (uri, requestBody) => { + return await callback?.(uri, requestBody); + }); return mockEndpoint; }; diff --git a/packages/profile-sync-controller/src/controllers/user-storage/services.test.ts b/packages/profile-sync-controller/src/controllers/user-storage/services.test.ts index f4a15317f5..7cc8e87e91 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/services.test.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/services.test.ts @@ -1,15 +1,19 @@ +import type { UserStoragePathWithKeyOnly } from 'src/shared/storage-schema'; + +import encryption, { createSHA256Hash } from '../../shared/encryption'; import { mockEndpointGetUserStorage, mockEndpointUpsertUserStorage, mockEndpointGetUserStorageAllFeatureEntries, + mockEndpointBatchUpsertUserStorage, } from './__fixtures__/mockServices'; import { - MOCK_ENCRYPTED_STORAGE_DATA, MOCK_STORAGE_DATA, MOCK_STORAGE_KEY, } from './__fixtures__/mockStorage'; import type { GetUserStorageResponse } from './services'; import { + batchUpsertUserStorage, getUserStorage, getUserStorageAllFeatureEntries, upsertUserStorage, @@ -132,8 +136,7 @@ describe('user-storage/services.ts - getUserStorageAllFeatureEntries() tests', ( describe('user-storage/services.ts - upsertUserStorage() tests', () => { const actCallUpsertUserStorage = async () => { - const encryptedData = await MOCK_ENCRYPTED_STORAGE_DATA(); - return await upsertUserStorage(encryptedData, { + return await upsertUserStorage(MOCK_STORAGE_DATA, { bearerToken: 'MOCK_BEARER_TOKEN', path: 'notifications.notification_settings', storageKey: MOCK_STORAGE_KEY, @@ -141,7 +144,23 @@ describe('user-storage/services.ts - upsertUserStorage() tests', () => { }; it('invokes upsert endpoint with no errors', async () => { - const mockUpsertUserStorage = mockEndpointUpsertUserStorage(); + const mockUpsertUserStorage = mockEndpointUpsertUserStorage( + 'notifications.notification_settings', + undefined, + async (requestBody) => { + if (typeof requestBody === 'string') { + return; + } + + const decryptedBody = await encryption.decryptString( + requestBody.data, + MOCK_STORAGE_KEY, + ); + + expect(decryptedBody).toBe(MOCK_STORAGE_DATA); + }, + ); + await actCallUpsertUserStorage(); expect(mockUpsertUserStorage.isDone()).toBe(true); @@ -159,3 +178,66 @@ describe('user-storage/services.ts - upsertUserStorage() tests', () => { mockUpsertUserStorage.done(); }); }); + +describe('user-storage/services.ts - batchUpsertUserStorage() tests', () => { + const dataToStore: [UserStoragePathWithKeyOnly, string][] = [ + ['0x123', MOCK_STORAGE_DATA], + ['0x456', MOCK_STORAGE_DATA], + ]; + + const actCallBatchUpsertUserStorage = async () => { + return await batchUpsertUserStorage(dataToStore, { + bearerToken: 'MOCK_BEARER_TOKEN', + path: 'accounts', + storageKey: MOCK_STORAGE_KEY, + }); + }; + + it('invokes upsert endpoint with no errors', async () => { + const mockUpsertUserStorage = mockEndpointBatchUpsertUserStorage( + 'accounts', + undefined, + async (_uri, requestBody) => { + if (typeof requestBody === 'string') { + return; + } + + const decryptedBody = await Promise.all( + Object.entries(requestBody.data).map( + async ([entryKey, entryValue]) => { + return [ + entryKey, + await encryption.decryptString(entryValue, MOCK_STORAGE_KEY), + ]; + }, + ), + ); + + const expectedBody = dataToStore.map(([entryKey, entryValue]) => [ + createSHA256Hash(String(entryKey) + MOCK_STORAGE_KEY), + entryValue, + ]); + + expect(decryptedBody).toStrictEqual(expectedBody); + }, + ); + + await actCallBatchUpsertUserStorage(); + + expect(mockUpsertUserStorage.isDone()).toBe(true); + }); + + it('throws error if unable to upsert user storage', async () => { + const mockUpsertUserStorage = mockEndpointBatchUpsertUserStorage( + 'accounts', + { + status: 500, + }, + ); + + await expect(actCallBatchUpsertUserStorage()).rejects.toThrow( + expect.any(Error), + ); + mockUpsertUserStorage.done(); + }); +}); diff --git a/packages/profile-sync-controller/src/controllers/user-storage/services.ts b/packages/profile-sync-controller/src/controllers/user-storage/services.ts index 49838b5532..bb3a82a9fe 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/services.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/services.ts @@ -1,10 +1,11 @@ import log from 'loglevel'; -import encryption from '../../shared/encryption'; +import encryption, { createSHA256Hash } from '../../shared/encryption'; import { Env, getEnvUrls } from '../../shared/env'; import type { UserStoragePathWithFeatureAndKey, UserStoragePathWithFeatureOnly, + UserStoragePathWithKeyOnly, } from '../../shared/storage-schema'; import { createEntryPath } from '../../shared/storage-schema'; import type { NativeScrypt } from '../../shared/types/encryption'; @@ -187,3 +188,47 @@ export async function upsertUserStorage( throw new Error('user-storage - unable to upsert data'); } } + +/** + * User Storage Service - Set multiple storage entries for one specific feature. + * You cannot use this method to set multiple features at once. + * + * @param data - data to store, in the form of an array of [entryKey, entryValue] pairs + * @param opts - storage options + */ +export async function batchUpsertUserStorage( + data: [UserStoragePathWithKeyOnly, string][], + opts: UserStorageAllFeatureEntriesOptions, +): Promise { + const { bearerToken, path, storageKey, nativeScryptCrypto } = opts; + + const encryptedData = await Promise.all( + data.map(async (d) => { + return [ + createSHA256Hash(d[0] + storageKey), + await encryption.encryptString( + d[1], + opts.storageKey, + nativeScryptCrypto, + ), + ]; + }), + ); + + const url = new URL(`${USER_STORAGE_ENDPOINT}/${path}`); + + const formattedData = Object.fromEntries(encryptedData); + + const res = await fetch(url.toString(), { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${bearerToken}`, + }, + body: JSON.stringify({ data: formattedData }), + }); + + if (!res.ok) { + throw new Error('user-storage - unable to batch upsert data'); + } +} diff --git a/packages/profile-sync-controller/src/sdk/__fixtures__/mock-userstorage.ts b/packages/profile-sync-controller/src/sdk/__fixtures__/mock-userstorage.ts index e777e2e99b..80b11706ed 100644 --- a/packages/profile-sync-controller/src/sdk/__fixtures__/mock-userstorage.ts +++ b/packages/profile-sync-controller/src/sdk/__fixtures__/mock-userstorage.ts @@ -62,12 +62,17 @@ export const handleMockUserStorageGetAllFeatureEntries = async ( return mockEndpoint; }; -export const handleMockUserStoragePut = (mockReply?: MockReply) => { +export const handleMockUserStoragePut = ( + mockReply?: MockReply, + callback?: (uri: string, requestBody: nock.Body) => Promise, +) => { const reply = mockReply ?? { status: 204 }; const mockEndpoint = nock(MOCK_STORAGE_URL) .persist() .put(/.*/u) - .reply(reply.status); + .reply(reply.status, async (uri, requestBody) => { + return await callback?.(uri, requestBody); + }); return mockEndpoint; }; diff --git a/packages/profile-sync-controller/src/sdk/user-storage.test.ts b/packages/profile-sync-controller/src/sdk/user-storage.test.ts index 8c5490b751..ea240e4a58 100644 --- a/packages/profile-sync-controller/src/sdk/user-storage.test.ts +++ b/packages/profile-sync-controller/src/sdk/user-storage.test.ts @@ -1,3 +1,6 @@ +import type { UserStoragePathWithKeyOnly } from 'src/shared/storage-schema'; + +import encryption, { createSHA256Hash } from '../shared/encryption'; import { Env } from '../shared/env'; import { arrangeAuthAPIs } from './__fixtures__/mock-auth'; import { @@ -89,6 +92,46 @@ describe('User Storage', () => { expect(responseAllFeatureEntries).toStrictEqual([data]); }); + it('batch set items', async () => { + const dataToStore: [UserStoragePathWithKeyOnly, string][] = [ + ['0x123', JSON.stringify(MOCK_NOTIFICATIONS_DATA)], + ['0x456', JSON.stringify(MOCK_NOTIFICATIONS_DATA)], + ]; + + const { auth } = arrangeAuth('SRP', MOCK_SRP); + const { userStorage } = arrangeUserStorage(auth); + + const mockPut = handleMockUserStoragePut( + undefined, + async (_, requestBody) => { + if (typeof requestBody === 'string') { + return; + } + + const decryptedBody = await Promise.all( + Object.entries(requestBody.data).map( + async ([entryKey, entryValue]) => { + return [ + entryKey, + await encryption.decryptString(entryValue, MOCK_STORAGE_KEY), + ]; + }, + ), + ); + + const expectedBody = dataToStore.map(([entryKey, entryValue]) => [ + createSHA256Hash(String(entryKey) + MOCK_STORAGE_KEY), + entryValue, + ]); + + expect(decryptedBody).toStrictEqual(expectedBody); + }, + ); + + await userStorage.batchSetItems('accounts', dataToStore); + expect(mockPut.isDone()).toBe(true); + }); + it('user storage: failed to set key', async () => { const { auth } = arrangeAuth('SRP', MOCK_SRP); const { userStorage } = arrangeUserStorage(auth); @@ -107,6 +150,23 @@ describe('User Storage', () => { ).rejects.toThrow(UserStorageError); }); + it('user storage: failed to batch set items', async () => { + const { auth } = arrangeAuth('SRP', MOCK_SRP); + const { userStorage } = arrangeUserStorage(auth); + + handleMockUserStoragePut({ + status: 401, + body: { + message: 'failed to insert storage entries', + error: 'generic-error', + }, + }); + + await expect( + userStorage.batchSetItems('notifications', []), + ).rejects.toThrow(UserStorageError); + }); + it('user storage: failed to get storage entry', async () => { const { auth } = arrangeAuth('SRP', MOCK_SRP); const { userStorage } = arrangeUserStorage(auth); @@ -124,6 +184,23 @@ describe('User Storage', () => { ).rejects.toThrow(UserStorageError); }); + it('user storage: failed to get storage entries', async () => { + const { auth } = arrangeAuth('SRP', MOCK_SRP); + const { userStorage } = arrangeUserStorage(auth); + + await handleMockUserStorageGetAllFeatureEntries({ + status: 401, + body: { + message: 'failed to get storage entries', + error: 'generic-error', + }, + }); + + await expect( + userStorage.getAllFeatureItems('notifications'), + ).rejects.toThrow(UserStorageError); + }); + it('user storage: key not found', async () => { const { auth } = arrangeAuth('SRP', MOCK_SRP); const { userStorage } = arrangeUserStorage(auth); diff --git a/packages/profile-sync-controller/src/sdk/user-storage.ts b/packages/profile-sync-controller/src/sdk/user-storage.ts index cbb4005b2f..cc1d8185dc 100644 --- a/packages/profile-sync-controller/src/sdk/user-storage.ts +++ b/packages/profile-sync-controller/src/sdk/user-storage.ts @@ -4,6 +4,7 @@ import { getEnvUrls } from '../shared/env'; import type { UserStoragePathWithFeatureAndKey, UserStoragePathWithFeatureOnly, + UserStoragePathWithKeyOnly, } from '../shared/storage-schema'; import { createEntryPath } from '../shared/storage-schema'; import type { IBaseAuth } from './authentication-jwt-bearer/types'; @@ -58,6 +59,13 @@ export class UserStorage { await this.#upsertUserStorage(path, value); } + async batchSetItems( + path: UserStoragePathWithFeatureOnly, + values: [UserStoragePathWithKeyOnly, string][], + ) { + await this.#batchUpsertUserStorage(path, values); + } + async getItem(path: UserStoragePathWithFeatureAndKey): Promise { return this.#getUserStorage(path); } @@ -123,6 +131,53 @@ export class UserStorage { } } + async #batchUpsertUserStorage( + path: UserStoragePathWithFeatureOnly, + data: [UserStoragePathWithKeyOnly, string][], + ): Promise { + try { + const headers = await this.#getAuthorizationHeader(); + const storageKey = await this.getStorageKey(); + + const encryptedData = await Promise.all( + data.map(async (d) => { + return [ + this.#createEntryKey(d[0], storageKey), + await encryption.encryptString(d[1], storageKey), + ]; + }), + ); + + const url = new URL(STORAGE_URL(this.env, path)); + + const response = await fetch(url.toString(), { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + ...headers, + }, + body: JSON.stringify({ data: Object.fromEntries(encryptedData) }), + }); + + if (!response.ok) { + const responseBody: ErrorMessage = await response.json().catch(() => ({ + message: 'unknown', + error: 'unknown', + })); + throw new Error( + `HTTP error message: ${responseBody.message}, error: ${responseBody.error}`, + ); + } + } catch (e) { + /* istanbul ignore next */ + const errorMessage = + e instanceof Error ? e.message : JSON.stringify(e ?? ''); + throw new UserStorageError( + `failed to batch upsert user storage for path '${path}'. ${errorMessage}`, + ); + } + } + async #getUserStorage( path: UserStoragePathWithFeatureAndKey, ): Promise { diff --git a/packages/profile-sync-controller/src/shared/storage-schema.ts b/packages/profile-sync-controller/src/shared/storage-schema.ts index eaebacdaf4..888ef93160 100644 --- a/packages/profile-sync-controller/src/shared/storage-schema.ts +++ b/packages/profile-sync-controller/src/shared/storage-schema.ts @@ -17,8 +17,8 @@ export const USER_STORAGE_SCHEMA = { type UserStorageSchema = typeof USER_STORAGE_SCHEMA; -type UserStorageFeatures = keyof UserStorageSchema; -type UserStorageFeatureKeys = +export type UserStorageFeatures = keyof UserStorageSchema; +export type UserStorageFeatureKeys = UserStorageSchema[Feature][0] extends typeof ALLOW_ARBITRARY_KEYS ? string : UserStorageSchema[Feature][number]; @@ -29,6 +29,9 @@ type UserStorageFeatureAndKey = { }; export type UserStoragePathWithFeatureOnly = keyof UserStorageSchema; +export type UserStoragePathWithKeyOnly = { + [K in UserStorageFeatures]: `${UserStorageFeatureKeys}`; +}[UserStoragePathWithFeatureOnly]; export type UserStoragePathWithFeatureAndKey = { [K in UserStorageFeatures]: `${K}.${UserStorageFeatureKeys}`; }[UserStoragePathWithFeatureOnly]; From 48b57731a92624e390343827017eb88d2d64439a Mon Sep 17 00:00:00 2001 From: Mathieu Artu Date: Mon, 23 Sep 2024 09:52:12 +0200 Subject: [PATCH 2/2] fix: various improvements --- .../src/controllers/user-storage/UserStorageController.ts | 2 +- .../src/controllers/user-storage/services.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts b/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts index 6402711da5..19e8824995 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/UserStorageController.ts @@ -639,7 +639,7 @@ export default class UserStorageController extends BaseController< /** * Allows storage of multiple user data entries for one specific feature. Data stored must be string formatted. - * Developers can extend the entry path and entry name through the `schema.ts` file. + * Developers can extend the entry path through the `schema.ts` file. * * @param path - string in the form of `${feature}` that matches schema * @param values - data to store, in the form of an array of `[entryKey, entryValue]` pairs diff --git a/packages/profile-sync-controller/src/controllers/user-storage/services.ts b/packages/profile-sync-controller/src/controllers/user-storage/services.ts index bb3a82a9fe..f3abe1bcb7 100644 --- a/packages/profile-sync-controller/src/controllers/user-storage/services.ts +++ b/packages/profile-sync-controller/src/controllers/user-storage/services.ts @@ -46,6 +46,8 @@ export type UserStorageAllFeatureEntriesOptions = UserStorageBaseOptions & { path: UserStoragePathWithFeatureOnly; }; +export type UserStorageBatchUpsertOptions = UserStorageAllFeatureEntriesOptions; + /** * User Storage Service - Get Storage Entry. * @@ -198,7 +200,7 @@ export async function upsertUserStorage( */ export async function batchUpsertUserStorage( data: [UserStoragePathWithKeyOnly, string][], - opts: UserStorageAllFeatureEntriesOptions, + opts: UserStorageBatchUpsertOptions, ): Promise { const { bearerToken, path, storageKey, nativeScryptCrypto } = opts;