Storage v2: add image upload+polling, use CIDs, remove metadata

creator-node/src/utils/fileHasher.js

@@ -179,19 +179,7 @@ export const fileHasher = {
    */
   async generateNonImageCid(content, logger = console) {
     const buffer = await fileHasher.convertToBuffer(content, logger)
-
-    const startHashing = process.hrtime.bigint()
-    const cid = await fileHasher.hashNonImages(buffer)
-
-    const hashDurationMs = fileHasher.convertNanosToMillis(
-      process.hrtime.bigint() - startHashing
-    )
-
-    logger.debug(
-      `[fileHasher - generateNonImageCid()] CID=${cid} hashDurationMs=${hashDurationMs}ms`
-    )
-
-    return cid
+    return fileHasher.hashNonImages(buffer)
   },
 
   /**
@@ -200,17 +188,7 @@ export const fileHasher = {
    * @param {Object?} logger
    * @returns {HashedImage[]} only hash responses with the structure [{path: <string>, cid: <string>, size: <number>}]
    */
-  async generateImageCids(content, logger = console) {
-    const startHashing = process.hrtime.bigint()
-    const hashedImages = await fileHasher.hashImages(content)
-    const hashDurationMs = fileHasher.convertNanosToMillis(
-      process.hrtime.bigint() - startHashing
-    )
-
-    const hashedImagesStr = JSON.stringify(hashedImages)
-    logger.debug(
-      `[fileHasher - generateImageCids()] hashedImages=${hashedImagesStr} hashImagesDurationMs=${hashDurationMs}ms`
-    )
-    return hashedImages
+  async generateImageCids(content, _ = console) {
+    return fileHasher.hashImages(content)
   }
 }

creator-node/src/utils/fsUtils.ts

@@ -91,9 +91,6 @@ export async function validateStateForImageDirCIDAndReturnFileUUID(
   if (!imageDirCID) {
     return null
   }
-  req.logger.debug(
-    `Beginning validateStateForImageDirCIDAndReturnFileUUID for imageDirCID ${imageDirCID}`
-  )
 
   // Ensure db row exists for dirCID
   const dirFile = await models.File.findOne({
@@ -139,9 +136,6 @@ export async function validateStateForImageDirCIDAndReturnFileUUID(
     })
   )
 
-  req.logger.debug(
-    `Completed validateStateForImageDirCIDAndReturnFileUUID for imageDirCID ${imageDirCID}`
-  )
   return dirFile.fileUUID
 }
 
@@ -262,7 +256,6 @@ export function computeFilePathInDir(dirName: string, fileName: string) {
 
   const parentDirPath = computeFilePath(dirName)
   const absolutePath = path.join(parentDirPath, fileName)
-  genericLogger.debug(`File path computed, absolutePath=${absolutePath}`)
   return absolutePath
 }
 
@@ -279,7 +272,6 @@ export async function computeFilePathInDirAndEnsureItExists(
   const parentDirPath = computeFilePath(dirName)
   await ensureDirPathExists(parentDirPath)
   const absolutePath = path.join(parentDirPath, fileName)
-  genericLogger.debug(`File path computed, absolutePath=${absolutePath}`)
   return absolutePath
 }
 

discovery-provider/src/queries/get_tracks.py

@@ -163,9 +163,10 @@ def _get_tracks(session, args):
                 sort_fn(
                     coalesce(
                         func.to_timestamp_safe(
-                            TrackWithAggregates.release_date, "Dy Mon DD YYYY HH24:MI:SS GMTTZHTZM"
+                            TrackWithAggregates.release_date,
+                            "Dy Mon DD YYYY HH24:MI:SS GMTTZHTZM",
                         ),
-                        TrackWithAggregates.created_at
+                        TrackWithAggregates.created_at,
                     )
                 )
             )
@@ -204,7 +205,8 @@ def _get_tracks(session, args):
                 coalesce(
                     # This func is defined in alembic migrations
                     func.to_timestamp_safe(
-                        TrackWithAggregates.release_date, "Dy Mon DD YYYY HH24:MI:SS GMTTZHTZM"
+                        TrackWithAggregates.release_date,
+                        "Dy Mon DD YYYY HH24:MI:SS GMTTZHTZM",
                     ),
                     TrackWithAggregates.created_at,
                 ).desc()

libs/src/api/Track.ts

@@ -394,84 +394,48 @@ export class Track extends Base {
     metadata: TrackMetadata,
     onProgress: () => void
   ) {
-    this.REQUIRES(Services.CREATOR_NODE) // TODO: Change to storage node
+    // Validate inputs
+    this.REQUIRES(Services.CREATOR_NODE)
     this.FILE_IS_VALID(trackFile)
-
-    try {
-      if (coverArtFile) this.FILE_IS_VALID(coverArtFile)
-
-      this.IS_OBJECT(metadata)
-
-      const ownerId = this.userStateManager.getCurrentUserId()
-      if (!ownerId) {
-        return {
-          error: 'No users loaded for this wallet'
-        }
+    if (coverArtFile) this.FILE_IS_VALID(coverArtFile)
+    this.IS_OBJECT(metadata)
+    const ownerId = this.userStateManager.getCurrentUserId()
+    if (!ownerId) {
+      return {
+        error: 'No users loaded for this wallet'
       }
+    }
+    metadata.owner_id = ownerId
+    this._validateTrackMetadata(metadata)
 
-      metadata.owner_id = ownerId
-      this._validateTrackMetadata(metadata)
-
-      // Upload metadata
-      const {
-        metadataMultihash,
-        // metadataFileUUID,
-        // transcodedTrackUUID,
-        transcodedTrackCID
-      } = await retry(
-        async () => {
-          return await this.creatorNode.uploadTrackContentV2(
-            trackFile,
-            coverArtFile,
-            metadata,
-            onProgress
-          )
-          // return this.creatorNode.uploadTrackContent(
-          //   trackFile,
-          //   coverArtFile,
-          //   metadata,
-          //   onProgress
-          // )
-        },
-        {
-          // Retry function 3x
-          // 1st retry delay = 500ms, 2nd = 1500ms, 3rd...nth retry = 4000 ms (capped)
-          minTimeout: 500,
-          maxTimeout: 4000,
-          factor: 3,
-          retries: 3,
-          onRetry: (err) => {
-            if (err) {
-              console.log('uploadTrackContentV2 retry error: ', err)
-            }
-          }
-        }
-      )
+    // Upload track audio and cover art to storage node
+    const updatedMetadata = await this.creatorNode.uploadTrackAudioAndCoverArtV2(
+      trackFile,
+      coverArtFile,
+      metadata,
+      onProgress
+    )
 
-      // Write metadata to chain
-      // TODO: Make discovery index by reading its own db instead of hitting CN /ipfs
-      const trackId = await this._generateTrackId()
-      const response = await this.contracts.EntityManagerClient!.manageEntity(
-        ownerId,
-        EntityManagerClient.EntityType.TRACK,
-        trackId,
-        EntityManagerClient.Action.CREATE,
-        metadataMultihash
-      )
-      const txReceipt = response.txReceipt
+    // Write metadata to chain
+    const trackId = await this._generateTrackId()
+    // TODO: Uncomment once we can index full metadata (not metadata hash)
+    // TODO: DON'T UNCOMMENTED ON STAGE OR PROD UNTIL THEN, OR ELSE INDEXING WILL STALL!!
+    // const response = await this.contracts.EntityManagerClient!.manageEntity(
+    //   ownerId,
+    //   EntityManagerClient.EntityType.TRACK,
+    //   trackId,
+    //   EntityManagerClient.Action.CREATE,
+    //   JSON.stringify(updatedMetadata) // TODO: @michelle: would this work? It doesn't need to be a different EntityType that accepts metadata instead of metadata hash?
+    // )
+    // const txReceipt = response.txReceipt
+    const txReceipt = { blockHash: 'UNCOMMENT ABOVE', blockNumber: 1 }
 
-      return {
-        blockHash: txReceipt.blockHash,
-        blockNumber: txReceipt.blockNumber,
-        trackId,
-        transcodedTrackCID,
-        error: false
-      }
-    } catch (e) {
-      return {
-        error: (e as Error).message,
-        stack: (e as Error).stack
-      }
+    return {
+      blockHash: txReceipt.blockHash,
+      blockNumber: txReceipt.blockNumber,
+      trackId,
+      transcodedTrackCID: updatedMetadata.track_cid,
+      error: false
     }
   }