Implement sending large thick thread notifs

Summary: This differential uploads large e2ee notifs to blob service Test Plan: Modify app to log all blob that it sends (JS) and decrypted blobs that it receives (native code). Send large and small notifs around. Ensure that only large notifs are logged in native code. Ensure that if there are multiple devices of the same platform as receivers only one distinct blob is uploaded to blob service. Reviewers: tomek, kamil, bartek Reviewed By: tomek, bartek Subscribers: ashoat Differential Revision: https://phab.comm.dev/D13495
CommE2E · Sep 27, 2024 · 39553e6 · 39553e6
1 parent 51312a1
commit 39553e6
Show file tree

Hide file tree

Showing 15 changed files with 262 additions and 76 deletions.
diff --git a/keyserver/src/creators/farcaster-channel-tag-creator.js b/keyserver/src/creators/farcaster-channel-tag-creator.js
@@ -8,6 +8,7 @@ import type {
   CreateOrUpdateFarcasterChannelTagResponse,
 } from 'lib/types/community-types.js';
 import { threadPermissions } from 'lib/types/thread-permission-types.js';
+import type { BlobOperationResult } from 'lib/utils/blob-service.js';
 import { ServerError } from 'lib/utils/errors.js';
 
 import {
@@ -18,11 +19,10 @@ import {
 import { fetchCommunityInfos } from '../fetchers/community-fetchers.js';
 import { checkThreadPermission } from '../fetchers/thread-permission-fetchers.js';
 import {
-  uploadBlob,
+  uploadBlobKeyserverWrapper,
   assignHolder,
   download,
   deleteBlob,
-  type BlobOperationResult,
   type BlobDownloadResult,
 } from '../services/blob.js';
 import { Viewer } from '../session/viewer.js';
@@ -162,7 +162,7 @@ async function uploadFarcasterChannelTagBlob(
   const hash = farcasterChannelTagBlobHash(farcasterChannelID);
   const blob = new Blob([payloadString]);
 
-  const uploadResult = await uploadBlob(blob, hash);
+  const uploadResult = await uploadBlobKeyserverWrapper(blob, hash);
 
   if (!uploadResult.success) {
     return uploadResult;

diff --git a/keyserver/src/creators/invite-link-creator.js b/keyserver/src/creators/invite-link-creator.js
@@ -11,6 +11,7 @@ import type {
   InviteLink,
 } from 'lib/types/link-types.js';
 import { threadPermissions } from 'lib/types/thread-permission-types.js';
+import type { BlobOperationResult } from 'lib/utils/blob-service.js';
 import { ServerError } from 'lib/utils/errors.js';
 import { reservedUsernamesSet } from 'lib/utils/reserved-users.js';
 
@@ -27,9 +28,8 @@ import {
   download,
   type BlobDownloadResult,
   assignHolder,
-  uploadBlob,
+  uploadBlobKeyserverWrapper,
   deleteBlob,
-  type BlobOperationResult,
 } from '../services/blob.js';
 import { Viewer } from '../session/viewer.js';
 import { thisKeyserverID } from '../user/identity.js';
@@ -272,7 +272,7 @@ async function uploadInviteLinkBlob(
   const key = inviteLinkBlobHash(linkSecret);
   const blob = new Blob([payloadString]);
 
-  const uploadResult = await uploadBlob(blob, key);
+  const uploadResult = await uploadBlobKeyserverWrapper(blob, key);
   if (!uploadResult.success) {
     return uploadResult;
   }

diff --git a/keyserver/src/push/encrypted-notif-utils-api.js b/keyserver/src/push/encrypted-notif-utils-api.js
@@ -69,6 +69,8 @@ const encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI = {
     const unencryptedDataBytes = new TextEncoder().encode(unencryptedData);
     return await encrypt(encryptionKeyBytes, unencryptedDataBytes);
   },
+  normalizeUint8ArrayForBlobUpload: (uint8Array: Uint8Array) =>
+    new Blob([uint8Array]),
 };
 
 export default encryptedNotifUtilsAPI;
diff --git a/keyserver/src/services/blob.js b/keyserver/src/services/blob.js
@@ -5,6 +5,10 @@ import {
   getBlobFetchableURL,
   makeBlobServiceEndpointURL,
 } from 'lib/utils/blob-service.js';
+import {
+  uploadBlob,
+  type BlobOperationResult,
+} from 'lib/utils/blob-service.js';
 import { createHTTPAuthorizationHeader } from 'lib/utils/services-utils.js';
 
 import { verifyUserLoggedIn } from '../user/login.js';
@@ -35,49 +39,6 @@ type BlobDescriptor = {
   +holder: string,
 };
 
-export type BlobOperationResult =
-  | {
-      +success: true,
-    }
-  | {
-      +success: false,
-      +reason: 'HASH_IN_USE' | 'OTHER',
-      +status: number,
-      +statusText: string,
-    };
-
-async function uploadBlob(
-  blob: Blob,
-  hash: string,
-): Promise<BlobOperationResult> {
-  const formData = new FormData();
-  formData.append('blob_hash', hash);
-  formData.append('blob_data', blob);
-
-  const headers = await createRequestHeaders(false);
-  const uploadBlobResponse = await fetch(
-    makeBlobServiceEndpointURL(blobService.httpEndpoints.UPLOAD_BLOB),
-    {
-      method: blobService.httpEndpoints.UPLOAD_BLOB.method,
-      body: formData,
-      headers,
-    },
-  );
-
-  if (!uploadBlobResponse.ok) {
-    const { status, statusText } = uploadBlobResponse;
-    const reason = status === 409 ? 'HASH_IN_USE' : 'OTHER';
-    return {
-      success: false,
-      reason,
-      status,
-      statusText,
-    };
-  }
-
-  return { success: true };
-}
-
 async function assignHolder(
   params: BlobDescriptor,
 ): Promise<BlobOperationResult> {
@@ -103,6 +64,14 @@ async function assignHolder(
   return { success: true };
 }
 
+async function uploadBlobKeyserverWrapper(
+  blob: Blob,
+  hash: string,
+): Promise<BlobOperationResult> {
+  const authHeaders = await createRequestHeaders(false);
+  return uploadBlob(blob, hash, authHeaders);
+}
+
 async function upload(
   blob: Blob,
   params: BlobDescriptor,
@@ -117,10 +86,9 @@ async function upload(
     },
 > {
   const { hash, holder } = params;
-
   const [holderResult, uploadResult] = await Promise.all([
     assignHolder({ hash, holder }),
-    uploadBlob(blob, hash),
+    uploadBlobKeyserverWrapper(blob, hash),
   ]);
   if (holderResult.success && uploadResult.success) {
     return { success: true };
@@ -171,4 +139,11 @@ async function deleteBlob(params: BlobDescriptor, instant?: boolean) {
   });
 }
 
-export { upload, uploadBlob, assignHolder, download, deleteBlob };
+export {
+  upload,
+  uploadBlob,
+  assignHolder,
+  download,
+  deleteBlob,
+  uploadBlobKeyserverWrapper,
+};
diff --git a/lib/facts/blob-service.js b/lib/facts/blob-service.js
@@ -2,7 +2,7 @@
 
 import { isDev } from '../utils/dev-utils.js';
 
-type BlobServicePath = '/blob/:blobHash' | '/blob';
+type BlobServicePath = '/blob/:blobHash' | '/blob' | '/holders';
 
 export type BlobServiceHTTPEndpoint = {
   +path: BlobServicePath,
@@ -23,6 +23,10 @@ const httpEndpoints = Object.freeze({
     path: '/blob',
     method: 'POST',
   },
+  ASSIGN_MULTIPLE_HOLDERS: {
+    path: '/holders',
+    method: 'POST',
+  },
   UPLOAD_BLOB: {
     path: '/blob',
     method: 'PUT',

diff --git a/lib/push/android-notif-creators.js b/lib/push/android-notif-creators.js
@@ -7,7 +7,9 @@ import {
   prepareEncryptedAndroidVisualNotifications,
   prepareEncryptedAndroidSilentNotifications,
   prepareLargeNotifData,
+  type LargeNotifEncryptionResult,
   type LargeNotifData,
+  generateBlobHolders,
 } from './crypto.js';
 import { hasMinCodeVersion } from '../shared/version-utils.js';
 import type { PlatformDetails } from '../types/device-types.js';
@@ -70,7 +72,7 @@ async function createAndroidVisualNotification(
   inputData: AndroidNotifInputData,
   devices: $ReadOnlyArray<NotificationTargetDevice>,
   largeNotifToEncryptionResultPromises?: {
-    [string]: Promise<LargeNotifData>,
+    [string]: Promise<LargeNotifEncryptionResult>,
   },
 ): Promise<{
   +targetedNotifications: $ReadOnlyArray<TargetedAndroidNotification>,
@@ -210,6 +212,7 @@ async function createAndroidVisualNotification(
   const copyWithMessageInfosDataBlob = JSON.stringify(
     copyWithMessageInfos.data,
   );
+
   if (
     canQueryBlobService &&
     largeNotifToEncryptionResultPromises &&
@@ -219,22 +222,21 @@ async function createAndroidVisualNotification(
       await largeNotifToEncryptionResultPromises[copyWithMessageInfosDataBlob];
     blobHash = largeNotifData.blobHash;
     encryptionKey = largeNotifData.encryptionKey;
-    blobHolders = largeNotifData.blobHolders;
+    blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length);
     encryptedCopyWithMessageInfos =
       largeNotifData.encryptedCopyWithMessageInfos;
   } else if (canQueryBlobService && largeNotifToEncryptionResultPromises) {
     largeNotifToEncryptionResultPromises[copyWithMessageInfosDataBlob] =
       prepareLargeNotifData(
         copyWithMessageInfosDataBlob,
-        devicesWithExcessiveSizeNoHolders.length,
         encryptedNotifUtilsAPI,
       );
 
     const largeNotifData =
       await largeNotifToEncryptionResultPromises[copyWithMessageInfosDataBlob];
     blobHash = largeNotifData.blobHash;
     encryptionKey = largeNotifData.encryptionKey;
-    blobHolders = largeNotifData.blobHolders;
+    blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length);
     encryptedCopyWithMessageInfos =
       largeNotifData.encryptedCopyWithMessageInfos;
   } else if (canQueryBlobService) {

diff --git a/lib/push/apns-notif-creators.js b/lib/push/apns-notif-creators.js
@@ -12,6 +12,8 @@ import {
   prepareEncryptedAPNsSilentNotifications,
   prepareLargeNotifData,
   type LargeNotifData,
+  type LargeNotifEncryptionResult,
+  generateBlobHolders,
 } from './crypto.js';
 import { getAPNsNotificationTopic } from '../shared/notif-utils.js';
 import { hasMinCodeVersion } from '../shared/version-utils.js';
@@ -45,7 +47,7 @@ async function createAPNsVisualNotification(
   inputData: APNsNotifInputData,
   devices: $ReadOnlyArray<NotificationTargetDevice>,
   largeNotifToEncryptionResultPromises?: {
-    [string]: Promise<LargeNotifData>,
+    [string]: Promise<LargeNotifEncryptionResult>,
   },
 ): Promise<{
   +targetedNotifications: $ReadOnlyArray<TargetedAPNsNotification>,
@@ -260,22 +262,22 @@ async function createAPNsVisualNotification(
       await largeNotifToEncryptionResultPromises[copyWithMessageInfosBlob];
     blobHash = largeNotifData.blobHash;
     encryptionKey = largeNotifData.encryptionKey;
-    blobHolders = largeNotifData.blobHolders;
+    blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length);
     encryptedCopyWithMessageInfos =
       largeNotifData.encryptedCopyWithMessageInfos;
   } else if (canQueryBlobService && largeNotifToEncryptionResultPromises) {
     largeNotifToEncryptionResultPromises[copyWithMessageInfosBlob] =
       prepareLargeNotifData(
         copyWithMessageInfosBlob,
-        devicesWithExcessiveSizeNoHolders.length,
+
         encryptedNotifUtilsAPI,
       );
 
     const largeNotifData =
       await largeNotifToEncryptionResultPromises[copyWithMessageInfosBlob];
     blobHash = largeNotifData.blobHash;
     encryptionKey = largeNotifData.encryptionKey;
-    blobHolders = largeNotifData.blobHolders;
+    blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length);
     encryptedCopyWithMessageInfos =
       largeNotifData.encryptedCopyWithMessageInfos;
   } else if (canQueryBlobService) {

diff --git a/lib/push/crypto.js b/lib/push/crypto.js
@@ -20,6 +20,7 @@ import type {
   APNsNotificationRescind,
   APNsBadgeOnlyNotification,
 } from '../types/notif-types.js';
+import { toBase64URL } from '../utils/base64.js';
 
 async function encryptAndroidNotificationPayload<T>(
   encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI,
@@ -611,21 +612,26 @@ function prepareEncryptedWNSNotifications(
   return Promise.all(notificationPromises);
 }
 
-export type LargeNotifData = {
-  +blobHolders: $ReadOnlyArray<string>,
+export type LargeNotifEncryptionResult = {
   +blobHash: string,
   +encryptionKey: string,
   +encryptedCopyWithMessageInfos: Uint8Array,
 };
 
+export type LargeNotifData = $ReadOnly<{
+  ...LargeNotifEncryptionResult,
+  +blobHolders: $ReadOnlyArray<string>,
+}>;
+
+function generateBlobHolders(numberOfDevices: number): $ReadOnlyArray<string> {
+  return Array.from({ length: numberOfDevices }, () => uuid.v4());
+}
+
 async function prepareLargeNotifData(
   copyWithMessageInfos: string,
-  numberOfDevices: number,
   encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI,
-): Promise<LargeNotifData> {
+): Promise<LargeNotifEncryptionResult> {
   const encryptionKey = await encryptedNotifUtilsAPI.generateAESKey();
-
-  const blobHolders = Array.from({ length: numberOfDevices }, () => uuid.v4());
   const encryptedCopyWithMessageInfos =
     await encryptedNotifUtilsAPI.encryptWithAESKey(
       encryptionKey,
@@ -634,9 +640,9 @@ async function prepareLargeNotifData(
   const blobHash = await encryptedNotifUtilsAPI.getBlobHash(
     encryptedCopyWithMessageInfos,
   );
+  const blobHashBase64url = toBase64URL(blobHash);
   return {
-    blobHolders,
-    blobHash,
+    blobHash: blobHashBase64url,
     encryptedCopyWithMessageInfos,
     encryptionKey,
   };
@@ -650,4 +656,5 @@ export {
   prepareEncryptedWebNotifications,
   prepareEncryptedWNSNotifications,
   prepareLargeNotifData,
+  generateBlobHolders,
 };