Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

⚠️ [PAY-2421] Upload stems in parallel #7644

Merged
merged 14 commits into from
Feb 21, 2024
23 changes: 14 additions & 9 deletions packages/common/src/services/audius-backend/AudiusBackend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1018,21 +1018,26 @@ export const audiusBackend = ({
trackFile: File,
coverArtFile: File,
metadata: TrackMetadata,
onProgress: (loaded: number, total: number) => void
onProgress: (loaded: number, total: number) => void,
trackId?: number
) {
try {
const { trackId, updatedMetadata, txReceipt } =
await audiusLibs.Track.uploadTrackV2AndWriteToChain(
trackFile,
coverArtFile,
metadata,
onProgress
)
const {
trackId: updatedTrackId,
updatedMetadata,
txReceipt
} = await audiusLibs.Track.uploadTrackV2AndWriteToChain(
trackFile,
coverArtFile,
metadata,
onProgress,
trackId
)
// Return with properties that confirmer expects
return {
blockHash: txReceipt.blockHash,
blockNumber: txReceipt.blockNumber,
trackId,
trackId: updatedTrackId,
transcodedTrackCID: updatedMetadata.track_cid,
error: false
}
Expand Down
6 changes: 3 additions & 3 deletions packages/common/src/store/upload/actions.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { TrackMetadata } from '../../models'
import { StemUploadWithFile } from '../../models'

import {
ExtendedCollectionMetadata,
Expand Down Expand Up @@ -48,7 +48,7 @@ export const uploadTracks = (
tracks: UploadTrack[],
metadata?: ExtendedCollectionMetadata,
uploadType?: UploadType,
stems?: TrackMetadata[]
stems?: StemUploadWithFile[]
) => {
return { type: UPLOAD_TRACKS, tracks, metadata, uploadType, stems }
}
Expand All @@ -61,7 +61,7 @@ export const uploadTracksRequested = (
tracks: UploadTrack[],
metadata?: ExtendedCollectionMetadata,
uploadType?: UploadType,
stems?: TrackMetadata[]
stems?: StemUploadWithFile[][]
) => {
return {
type: UPLOAD_TRACKS_REQUESTED,
Expand Down
18 changes: 14 additions & 4 deletions packages/common/src/store/upload/reducer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ import {
updateProgress,
uploadSingleTrackFailed
} from './actions'
import { ProgressStatus, UploadState } from './types'
import { ProgressStatus, UploadState, UploadTrack } from './types'
import { StemUploadWithFile } from '~/models'

const initialState: UploadState = {
openMultiTrackNotification: true,
Expand Down Expand Up @@ -52,6 +53,13 @@ const initialUploadState = {
transcode: 0
}
}
const getInitialProgress = (upload: UploadTrack | StemUploadWithFile) => {
const res = cloneDeep(initialUploadState)
res.art.total =
'artwork' in upload.metadata ? upload.metadata.artwork?.file?.size ?? 0 : 0
res.audio.total = upload.file?.size ?? 0
return res
}

const actionsMap = {
[TOGGLE_MULTI_TRACK_NOTIFICATION](
Expand All @@ -70,12 +78,13 @@ const actionsMap = {
const newState = { ...state }
newState.uploading = true
newState.tracks = action.tracks
newState.uploadProgress = action.tracks.map(() =>
cloneDeep(initialUploadState)
)
newState.uploadProgress = action.tracks
.map(getInitialProgress)
.concat(action.stems?.map((t) => t.map(getInitialProgress)).flat(1) ?? [])
newState.metadata = action.metadata ?? null
newState.uploadType = action.uploadType ?? null
newState.stems = action.stems ?? newState.stems
newState.error = false
return newState
},
[UPLOAD_TRACKS_SUCCEEDED](
Expand Down Expand Up @@ -107,6 +116,7 @@ const actionsMap = {
newState.tracks = null
newState.metadata = null
newState.stems = []
newState.error = true
return newState
},
[UPDATE_PROGRESS](
Expand Down
10 changes: 7 additions & 3 deletions packages/common/src/store/upload/selectors.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { round, clamp } from 'lodash'
import { floor, clamp } from 'lodash'

import { CommonState } from '../commonStore'
import { ProgressStatus } from './types'

export const getStems = (state: CommonState) => state.upload.stems
export const getUploadProgress = (state: CommonState) =>
Expand All @@ -22,7 +23,10 @@ const getKeyUploadProgress = (state: CommonState, key: 'art' | 'audio') => {
const uploadProgress = getUploadProgress(state)
if (uploadProgress == null) return 0

const filteredProgress = uploadProgress.filter((progress) => key in progress)
const filteredProgress = uploadProgress.filter(
(progress) =>
key in progress && progress[key].status !== ProgressStatus.ERROR
)
if (filteredProgress.length === 0) return 0

const loaded = filteredProgress.reduce(
Expand Down Expand Up @@ -53,7 +57,7 @@ const getKeyUploadProgress = (state: CommonState, key: 'art' | 'audio') => {
export const getCombinedUploadPercentage = (state: CommonState) => {
const artProgress = getKeyUploadProgress(state, 'art')
const audioProgress = getKeyUploadProgress(state, 'audio')
const percent = round(
const percent = floor(
100 * (ART_WEIGHT * artProgress + AUDIO_WEIGHT * audioProgress)
)
return clamp(percent, 0, 100)
Expand Down
4 changes: 2 additions & 2 deletions packages/common/src/store/upload/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { CollectionMetadata, TrackMetadata } from '../../models'
import { CollectionMetadata, StemUpload, TrackMetadata } from '../../models'
import { Nullable } from '../../utils/typeUtils'

export type NativeFile = {
Expand Down Expand Up @@ -68,6 +68,6 @@ export interface UploadState {
error: boolean
shouldReset: boolean
completionId: Nullable<number>
stems: TrackMetadata[]
stems: StemUpload[][]
failedTrackIndices: number[]
}
32 changes: 17 additions & 15 deletions packages/libs/src/api/Track.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,19 +371,21 @@ export class Track extends Base {
trackFile: File,
coverArtFile: File,
metadata: TrackMetadata,
onProgress: () => void
onProgress: () => void,
trackId?: number
) {
const updatedMetadata = await this.uploadTrackV2(
trackFile,
coverArtFile,
metadata,
onProgress
)
const { trackId, metadataCid, txReceipt } = await this.writeTrackToChain(
updatedMetadata,
Action.CREATE
)
return { trackId, metadataCid, updatedMetadata, txReceipt }
const {
trackId: updatedTrackId,
metadataCid,
txReceipt
} = await this.writeTrackToChain(updatedMetadata, Action.CREATE, trackId)
return { trackId: updatedTrackId, metadataCid, updatedMetadata, txReceipt }
}

/**
Expand Down Expand Up @@ -474,7 +476,7 @@ export class Track extends Base {
throw new Error('No users loaded for this wallet')
}

if (!trackId) trackId = await this._generateTrackId()
if (!trackId) trackId = await this.generateTrackId()
const metadataCid = await Utils.fileHasher.generateMetadataCidV1(
trackMetadata
)
Expand Down Expand Up @@ -575,19 +577,19 @@ export class Track extends Base {
)
}

async generateTrackId(): Promise<number> {
const encodedId = await this.discoveryProvider.getUnclaimedId('tracks')
if (!encodedId) {
throw new Error('No unclaimed track IDs')
}
return decodeHashId(encodedId)!
}

/* ------- PRIVATE ------- */

// Throws an error upon validation failure
_validateTrackMetadata(metadata: TrackMetadata) {
this.OBJECT_HAS_PROPS(metadata, TRACK_PROPS, TRACK_REQUIRED_PROPS)
this.creatorNode.validateTrackSchema(metadata)
}

async _generateTrackId(): Promise<number> {
const encodedId = await this.discoveryProvider.getUnclaimedId('tracks')
if (!encodedId) {
throw new Error('No unclaimed track IDs')
}
return decodeHashId(encodedId)!
}
}
Loading