[PAY-2597] Test migration of a single endpoint from apiclient -> SDK (#…

…8091)

packages/common/package.json

@@ -48,6 +48,7 @@
     "async-retry": "1.3.3",
     "bn.js": "5.1.0",
     "dayjs": "1.10.7",
+    "deep-object-diff": "1.1.9",
     "formik": "2.4.1",
     "fxa-common-password-list": "0.0.2",
     "hashids": "2.2.1",

packages/common/src/api/topArtists.ts

@@ -1,6 +1,7 @@
 import { uniq } from 'lodash'
 
 import { createApi } from '~/audius-query'
+import { userMetadataListFromSDK } from '~/models'
 import { ID } from '~/models/Identifiers'
 import { Kind } from '~/models/Kind'
 
@@ -18,12 +19,26 @@ const topArtistsApi = createApi({
     getTopArtistsInGenre: {
       async fetch(args: GetTopArtistsForGenreArgs, context) {
         const { genre, limit, offset } = args
-        const { apiClient } = context
+        const { apiClient, audiusSdk, checkSDKMigration } = context
+        const sdk = await audiusSdk()
 
-        return await apiClient.getTopArtistGenres({
-          genres: [genre],
-          limit,
-          offset
+        return await checkSDKMigration({
+          legacy: apiClient.getTopArtistGenres({
+            genres: [genre],
+            limit,
+            offset
+          }),
+          migrated: async () =>
+            userMetadataListFromSDK(
+              (
+                await sdk.full.users.getTopUsersInGenre({
+                  genre: [genre],
+                  limit,
+                  offset
+                })
+              ).data
+            ),
+          endpointName: 'getTopArtistsInGenre'
         })
       },
       options: { idArgKey: 'genre', kind: Kind.USERS, schemaKey: 'users' }

packages/common/src/audius-query/AudiusQueryContext.ts

@@ -5,6 +5,7 @@ import type { Dispatch } from 'redux'
 
 import type { AudiusAPIClient } from '~/services/audius-api-client'
 import { AudiusBackend, Env, RemoteConfigInstance } from '~/services/index'
+import { SDKMigrationChecker } from '~/utils/sdkMigrationUtils'
 
 import { ReportToSentryArgs } from '../models'
 
@@ -14,6 +15,7 @@ export type AudiusQueryContextType = {
   audiusBackend: AudiusBackend
   dispatch: Dispatch
   reportToSentry: (args: ReportToSentryArgs) => void
+  checkSDKMigration: SDKMigrationChecker
   env: Env
   fetch: typeof fetch
   remoteConfigInstance: RemoteConfigInstance

packages/common/src/models/ImageSizes.ts

@@ -1,3 +1,5 @@
+import { full } from '@audius/sdk'
+
 export enum DefaultSizes {
   // Used as a catch-all fallback when no other size data is available.
   OVERRIDE = 'OVERRIDE'
@@ -31,3 +33,25 @@ export type ProfilePictureSizesCids =
 export type ProfilePictureSizes = ImageSizesObject<SquareSizes>
 export type CoverPhotoSizesCids = ImageSizesObjectWithoutOverride<WidthSizes>
 export type CoverPhotoSizes = ImageSizesObject<WidthSizes>
+
+export const coverPhotoSizesCIDsFromSDK = (
+  input: full.CoverPhoto
+): CoverPhotoSizesCids => {
+  return [WidthSizes.SIZE_640, WidthSizes.SIZE_2000].reduce((out, size) => {
+    out[size] = input[size] ?? null
+    return out
+  }, {})
+}
+
+export const profilePictureSizesCIDsFromSDK = (
+  input: full.ProfilePicture
+): ProfilePictureSizesCids => {
+  return [
+    SquareSizes.SIZE_1000_BY_1000,
+    SquareSizes.SIZE_150_BY_150,
+    SquareSizes.SIZE_480_BY_480
+  ].reduce((out, size) => {
+    out[size] = input[size] ?? null
+    return out
+  }, {})
+}

packages/common/src/models/PlaylistLibrary.ts

@@ -1,3 +1,5 @@
+import { full } from '@audius/sdk'
+
 import { SmartCollectionVariant } from '~/models/SmartCollectionVariant'
 
 import { ID } from './Identifiers'
@@ -43,3 +45,12 @@ export type PlaylistLibraryItem =
 export type PlaylistLibrary = {
   contents: (PlaylistLibraryFolder | PlaylistLibraryIdentifier)[]
 }
+
+export const playlistLibraryFromSDK = (
+  input?: full.PlaylistLibrary
+): PlaylistLibrary | undefined => {
+  if (!input) return undefined
+  return {
+    contents: input.contents as PlaylistLibraryItem[]
+  }
+}

packages/common/src/models/User.ts

@@ -1,15 +1,25 @@
+import { full } from '@audius/sdk'
+import { omit } from 'lodash'
+import snakecaseKeys from 'snakecase-keys'
+
 import { Collectible, CollectiblesMetadata } from '~/models/Collectible'
 import { Color } from '~/models/Color'
 import { CID, ID } from '~/models/Identifiers'
 import {
   CoverPhotoSizes,
   CoverPhotoSizesCids,
   ProfilePictureSizes,
-  ProfilePictureSizesCids
+  ProfilePictureSizesCids,
+  coverPhotoSizesCIDsFromSDK,
+  profilePictureSizesCIDsFromSDK
 } from '~/models/ImageSizes'
-import { PlaylistLibrary } from '~/models/PlaylistLibrary'
+import {
+  PlaylistLibrary,
+  playlistLibraryFromSDK
+} from '~/models/PlaylistLibrary'
 import { SolanaWalletAddress, StringWei, WalletAddress } from '~/models/Wallet'
-import { Nullable } from '~/utils/typeUtils'
+import { decodeHashId } from '~/utils/hashIds'
+import { Nullable, removeNullable } from '~/utils/typeUtils'
 
 import { Timestamped } from './Timestamped'
 import { UserEvent } from './UserEvent'
@@ -20,37 +30,37 @@ export type UserMetadata = {
   artist_pick_track_id: Nullable<number>
   bio: Nullable<string>
   blocknumber: number
+  collectibleList?: Collectible[]
+  collectibles?: CollectiblesMetadata
+  collectiblesOrderUnset?: boolean
+  cover_photo_cids?: Nullable<CoverPhotoSizesCids>
+  cover_photo_sizes: Nullable<CID>
   cover_photo: Nullable<CID>
   creator_node_endpoint: Nullable<string>
   current_user_followee_follow_count: number
   does_current_user_follow: boolean
   does_current_user_subscribe?: boolean
+  erc_wallet: WalletAddress
   followee_count: number
   follower_count: number
-  supporter_count: number
-  supporting_count: number
-  handle: string
   handle_lc: string
+  handle: string
+  has_collectibles: boolean
   is_deactivated: boolean
   is_verified: boolean
   location: Nullable<string>
+  metadata_multihash: Nullable<CID>
   name: string
   playlist_count: number
+  profile_picture_cids?: Nullable<ProfilePictureSizesCids>
+  profile_picture_sizes: Nullable<CID>
   profile_picture: Nullable<CID>
   repost_count: number
-  track_count: number
-  cover_photo_sizes: Nullable<CID>
-  cover_photo_cids?: Nullable<CoverPhotoSizesCids>
-  profile_picture_sizes: Nullable<CID>
-  profile_picture_cids?: Nullable<ProfilePictureSizesCids>
-  metadata_multihash: Nullable<CID>
-  erc_wallet: WalletAddress
-  spl_wallet: Nullable<SolanaWalletAddress>
-  has_collectibles: boolean
-  collectibles?: CollectiblesMetadata
-  collectiblesOrderUnset?: boolean
-  collectibleList?: Collectible[]
   solanaCollectibleList?: Collectible[]
+  spl_wallet: Nullable<SolanaWalletAddress>
+  supporter_count: number
+  supporting_count: number
+  track_count: number
 
   // Only present on the "current" account
   track_save_count?: number
@@ -69,7 +79,7 @@ export type UserMetadata = {
   associated_wallets?: Nullable<string[]>
   associated_sol_wallets?: Nullable<string[]>
   associated_wallets_balance?: Nullable<StringWei>
-  playlist_library?: PlaylistLibrary
+  playlist_library?: Nullable<PlaylistLibrary>
   userBank?: SolanaWalletAddress
   local?: boolean
   events?: UserEvent
@@ -100,3 +110,57 @@ export type UserMultihash = Pick<
   User,
   'metadata_multihash' | 'creator_node_endpoint'
 >
+
+/** Converts a SDK `full.UserFull` response to a UserMetadata. Note: Will _not_ include the "current user" fields as those aren't returned by the Users API */
+export const userMetadataFromSDK = (
+  input: full.UserFull
+): UserMetadata | undefined => {
+  const user = snakecaseKeys(input)
+  const decodedUserId = decodeHashId(user.id)
+  if (!decodedUserId) {
+    return undefined
+  }
+
+  const newUser: UserMetadata = {
+    // Fields from API that are omitted in this model
+    ...omit(user, ['id', 'cover_photo_legacy', 'profile_picture_legacy']),
+
+    // Conversions
+    artist_pick_track_id: user.artist_pick_track_id
+      ? decodeHashId(user.artist_pick_track_id)
+      : null,
+
+    // Nested Types
+    playlist_library: playlistLibraryFromSDK(user.playlist_library) ?? null,
+    cover_photo_cids: user.cover_photo_cids
+      ? coverPhotoSizesCIDsFromSDK(user.cover_photo_cids)
+      : null,
+    profile_picture_cids: user.profile_picture_cids
+      ? profilePictureSizesCIDsFromSDK(user.profile_picture_cids)
+      : null,
+
+    // Re-types
+    balance: user.balance as StringWei,
+    associated_wallets_balance: user.associated_wallets_balance as StringWei,
+    total_balance: user.total_balance as StringWei,
+    user_id: decodedUserId,
+    spl_wallet: user.spl_wallet as SolanaWalletAddress,
+
+    // Legacy Overrides
+    cover_photo: user.cover_photo_legacy ?? null,
+    profile_picture: user.profile_picture_legacy ?? null,
+
+    // Required Nullable fields
+    bio: user.bio ?? null,
+    cover_photo_sizes: user.cover_photo_sizes ?? null,
+    creator_node_endpoint: user.creator_node_endpoint ?? null,
+    location: user.location ?? null,
+    metadata_multihash: user.metadata_multihash ?? null,
+    profile_picture_sizes: user.profile_picture_sizes ?? null
+  }
+
+  return newUser
+}
+
+export const userMetadataListFromSDK = (input?: full.UserFull[]) =>
+  input ? input.map((d) => userMetadataFromSDK(d)).filter(removeNullable) : []

packages/common/src/services/remote-config/feature-flags.ts

@@ -58,6 +58,7 @@ export enum FeatureFlags {
   TIKTOK_NATIVE_AUTH = 'tiktok_native_auth',
   PREMIUM_ALBUMS_ENABLED = 'premium_albums_enabled',
   REWARDS_COOLDOWN = 'rewards_cooldown',
+  SDK_MIGRATION_SHADOWING = 'sdk_migration_shadowing',
   USE_SDK_TIPS = 'use_sdk_tips',
   USE_SDK_REWARDS = 'use_sdk_rewards',
   DISCOVERY_TIP_REACTIONS = 'discovery_tip_reactions'
@@ -134,6 +135,7 @@ export const flagDefaults: FlagDefaults = {
   [FeatureFlags.TIKTOK_NATIVE_AUTH]: true,
   [FeatureFlags.PREMIUM_ALBUMS_ENABLED]: false,
   [FeatureFlags.REWARDS_COOLDOWN]: false,
+  [FeatureFlags.SDK_MIGRATION_SHADOWING]: false,
   [FeatureFlags.USE_SDK_TIPS]: false,
   [FeatureFlags.USE_SDK_REWARDS]: false,
   [FeatureFlags.DISCOVERY_TIP_REACTIONS]: false

packages/common/src/store/effects.ts

@@ -1,9 +1,87 @@
-import { GetContextEffect } from 'redux-saga/effects'
-import { getContext as getContextBase, SagaGenerator } from 'typed-redux-saga'
+import { AllEffect, CallEffect, GetContextEffect } from 'redux-saga/effects'
+import {
+  all,
+  call,
+  getContext as getContextBase,
+  SagaGenerator
+} from 'typed-redux-saga'
+
+import { ErrorLevel } from '~/models/ErrorReporting'
+import { FeatureFlags } from '~/services'
+import {
+  compareSDKResponse,
+  SDKMigrationFailedError
+} from '~/utils/sdkMigrationUtils'
 
 import { CommonStoreContext } from './storeContext'
 
 export const getContext = <Prop extends keyof CommonStoreContext>(
   prop: Prop
 ): SagaGenerator<CommonStoreContext[Prop], GetContextEffect> =>
   getContextBase(prop)
+
+/** Helper generator that returns a fully-awaited AudiusSDK instance */
+export function* getSDK() {
+  const audiusSdk = yield* getContext('audiusSdk')
+  return yield* call(audiusSdk)
+}
+
+/** This effect is used to shadow a migration without affecting the return value.
+ * It will run two effects in parallel to fetch the legacy and migrated responses,
+ * compare the results, log the diff, and then return the legacy value. Errors thrown
+ * by the effect for the migrated response will be caught to avoid bugs in the migrated
+ * code from causing errors.
+ */
+export function* checkSDKMigration<T extends object>({
+  legacy: legacyCall,
+  migrated: migratedCall,
+  endpointName
+}: {
+  legacy: SagaGenerator<T, CallEffect<T>>
+  migrated: SagaGenerator<T, CallEffect<T>>
+  endpointName: string
+}) {
+  const getFeatureEnabled = yield* getContext('getFeatureEnabled')
+  const reportToSentry = yield* getContext('reportToSentry')
+
+  if (!getFeatureEnabled(FeatureFlags.SDK_MIGRATION_SHADOWING)) {
+    return yield* legacyCall
+  }
+
+  const [legacy, migrated] = yield* all([
+    legacyCall,
+    call(function* settle() {
+      try {
+        return yield* migratedCall
+      } catch (e) {
+        return e instanceof Error ? e : new Error(`${e}`)
+      }
+    })
+  ]) as SagaGenerator<T[], AllEffect<CallEffect<T>>>
+
+  try {
+    compareSDKResponse({ legacy, migrated }, endpointName)
+  } catch (e) {
+    const error =
+      e instanceof SDKMigrationFailedError
+        ? e
+        : new SDKMigrationFailedError({
+            endpointName,
+            innerMessage: `Unknown error: ${e}`,
+            legacyValue: legacy,
+            migratedValue: migrated
+          })
+    console.warn('SDK Migration failed', error)
+    yield* call(reportToSentry, {
+      error,
+      level: ErrorLevel.Warning,
+      additionalInfo: {
+        diff: JSON.stringify(error.diff, null, 2),
+        legacyValue: JSON.stringify(error.legacyValue, null, 2),
+        migratedValue: JSON.stringify(error.migratedValue, null, 2)
+      },
+      tags: { endpointName: error.endpointName }
+    })
+  }
+  return legacy
+}