diff --git a/packages/hms-video-store/src/error/ErrorCodes.ts b/packages/hms-video-store/src/error/ErrorCodes.ts index 24e00e6419..dccd5b6286 100644 --- a/packages/hms-video-store/src/error/ErrorCodes.ts +++ b/packages/hms-video-store/src/error/ErrorCodes.ts @@ -76,6 +76,9 @@ export const ErrorCodes = { // Selected device not detected on change SELECTED_DEVICE_MISSING: 3014, + + // Track is publishing with no data, can happen when a whatsapp call is ongoing before 100ms call in mweb + NO_DATA: 3015, }, WebrtcErrors: { diff --git a/packages/hms-video-store/src/error/ErrorFactory.ts b/packages/hms-video-store/src/error/ErrorFactory.ts index e0655a225f..7345d26749 100644 --- a/packages/hms-video-store/src/error/ErrorFactory.ts +++ b/packages/hms-video-store/src/error/ErrorFactory.ts @@ -251,6 +251,16 @@ export const ErrorFactory = { false, ); }, + + NoDataInTrack(description: string) { + return new HMSException( + ErrorCodes.TracksErrors.NO_DATA, + 'Track does not have any data', + HMSAction.TRACK, + description, + 'This could possibily due to another application taking priority over the access to camera or microphone or due to an incoming call', + ); + }, }, WebrtcErrors: { diff --git a/packages/hms-video-store/src/interfaces/config.ts b/packages/hms-video-store/src/interfaces/config.ts index 07d75e21a4..23b88c4b58 100644 --- a/packages/hms-video-store/src/interfaces/config.ts +++ b/packages/hms-video-store/src/interfaces/config.ts @@ -42,10 +42,6 @@ export interface HMSConfig { audioSinkElementId?: string; autoVideoSubscribe?: boolean; initEndpoint?: string; - /** - * Request Camera/Mic permissions irrespective of role to avoid delay in getting device list - */ - alwaysRequestPermissions?: boolean; /** * Enable to get a network quality score while in preview. The score ranges from -1 to 5. * -1 when we are not able to connect to 100ms servers within an expected time limit diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts index b97efa49cd..d194ccca37 100644 --- a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts @@ -56,6 +56,7 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { ) { super(stream, track, source); stream.tracks.push(this); + this.addTrackEventListeners(track); this.settings = settings; // Replace the 'default' or invalid deviceId with the actual deviceId @@ -99,13 +100,9 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { this.manuallySelectedDeviceId = undefined; } - private isTrackNotPublishing = () => { - return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted; - }; - private handleVisibilityChange = async () => { // track state is fine do nothing - if (!this.isTrackNotPublishing()) { + if (!this.shouldReacquireTrack()) { HMSLogger.d(this.TAG, `visibiltiy: ${document.visibilityState}`, `${this}`); return; } @@ -151,16 +148,19 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { * no audio when the above getAudioTrack throws an error. ex: DeviceInUse error */ prevTrack?.stop(); + this.removeTrackEventListeners(prevTrack); this.tracksCreated.forEach(track => track.stop()); this.tracksCreated.clear(); try { const newTrack = await getAudioTrack(settings); + this.addTrackEventListeners(newTrack); this.tracksCreated.add(newTrack); HMSLogger.d(this.TAG, 'replaceTrack, Previous track stopped', prevTrack, 'newTrack', newTrack); await this.updateTrack(newTrack); } catch (e) { // Generate a new track from previous settings so there will be audio because previous track is stopped const newTrack = await getAudioTrack(this.settings); + this.addTrackEventListeners(newTrack); this.tracksCreated.add(newTrack); await this.updateTrack(newTrack); if (this.isPublished) { @@ -184,8 +184,8 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { return; } - // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled. - if (value && (isEmptyTrack(this.nativeTrack) || this.nativeTrack.muted)) { + // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled or ended track or when silence is detected. + if (value && this.shouldReacquireTrack()) { await this.replaceTrackWith(this.settings); } await super.setEnabled(value); @@ -303,6 +303,40 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { return this.processedTrack || this.nativeTrack; } + private addTrackEventListeners(track: MediaStreamTrack) { + track.addEventListener('mute', this.handleTrackMute); + track.addEventListener('unmute', this.handleTrackUnmute); + } + + private removeTrackEventListeners(track: MediaStreamTrack) { + track.removeEventListener('mute', this.handleTrackMute); + track.removeEventListener('unmute', this.handleTrackUnmute); + } + + private handleTrackMute = () => { + HMSLogger.d(this.TAG, 'muted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: true, + reason, + }), + ); + }; + + /** @internal */ + handleTrackUnmute = () => { + HMSLogger.d(this.TAG, 'unmuted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: false, + reason, + }), + ); + this.setEnabled(this.enabled); + }; + private replaceSenderTrack = async () => { if (!this.transceiver || this.transceiver.direction !== 'sendonly') { HMSLogger.d(this.TAG, `transceiver for ${this.trackId} not available or not connected yet`); @@ -311,6 +345,12 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { await this.transceiver.sender.replaceTrack(this.processedTrack || this.nativeTrack); }; + private shouldReacquireTrack = () => { + return ( + isEmptyTrack(this.nativeTrack) || this.isTrackNotPublishing() || this.audioLevelMonitor?.isSilentThisInstant() + ); + }; + private buildNewSettings(settings: Partial) { const { volume, codec, maxBitrate, deviceId, advanced, audioMode } = { ...this.settings, ...settings }; const newSettings = new HMSAudioTrackSettings(volume, codec, maxBitrate, deviceId, advanced, audioMode); diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts index f05da3f17e..46d4eeea77 100644 --- a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts @@ -131,6 +131,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { * use this function to set the enabled state of a track. If true the track will be unmuted and muted otherwise. * @param value */ + // eslint-disable-next-line complexity async setEnabled(value: boolean): Promise { if (value === this.enabled) { return; @@ -546,6 +547,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { super.handleTrackUnmute(); this.eventBus.localVideoEnabled.publish({ enabled: this.enabled, track: this }); this.eventBus.localVideoUnmutedNatively.publish(); + this.setEnabled(this.enabled); }; /** diff --git a/packages/hms-video-store/src/media/tracks/HMSTrack.ts b/packages/hms-video-store/src/media/tracks/HMSTrack.ts index 04f9ff511f..208d021ba4 100644 --- a/packages/hms-video-store/src/media/tracks/HMSTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSTrack.ts @@ -85,6 +85,11 @@ export abstract class HMSTrack { protected setFirstTrackId(trackId: string) { this.firstTrackId = trackId; } + + isTrackNotPublishing = () => { + return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted; + }; + /** * @internal * It will send event to analytics when interruption start/stop diff --git a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts index 3f6e60b2e7..03e6d7e0b7 100644 --- a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts @@ -93,8 +93,8 @@ export class HMSVideoTrack extends HMSTrack { private reTriggerPlay = ({ videoElement }: { videoElement: HTMLVideoElement }) => { setTimeout(() => { - videoElement.play().catch(() => { - HMSLogger.w('[HMSVideoTrack]', 'failed to play'); + videoElement.play().catch((e: Error) => { + HMSLogger.w('[HMSVideoTrack]', 'failed to play', e.message); }); }, 0); }; diff --git a/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts b/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts index b962dabd20..e1788ae25e 100644 --- a/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts +++ b/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts @@ -109,6 +109,7 @@ const mockMediaStream = { kind: 'video', getSettings: jest.fn(() => ({ deviceId: 'video-device-id' })), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), }, ]), getAudioTracks: jest.fn(() => [ @@ -117,6 +118,7 @@ const mockMediaStream = { kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-device-id' })), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), }, ]), addTrack: jest.fn(() => {}), @@ -206,7 +208,13 @@ const mockAudioContext = { return { stream: { getAudioTracks: jest.fn(() => [ - { id: 'audio-id', kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })) }, + { + id: 'audio-id', + kind: 'audio', + getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })), + addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), + }, ]), }, }; @@ -426,6 +434,7 @@ describe('LocalTrackManager', () => { kind: 'video', getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), } as unknown as MediaStreamTrack, HMSPeerType.REGULAR, testEventBus, @@ -459,6 +468,7 @@ describe('LocalTrackManager', () => { kind: 'video', getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), } as unknown as MediaStreamTrack, HMSPeerType.REGULAR, testEventBus, diff --git a/packages/hms-video-store/src/sdk/index.ts b/packages/hms-video-store/src/sdk/index.ts index e0a015a817..dd0f751cbe 100644 --- a/packages/hms-video-store/src/sdk/index.ts +++ b/packages/hms-video-store/src/sdk/index.ts @@ -435,13 +435,6 @@ export class HMSSdk implements HMSInterface { this.analyticsTimer.start(TimedEvent.PREVIEW); this.setUpPreview(config, listener); - // Request permissions and populate devices before waiting for policy - if (config.alwaysRequestPermissions) { - this.localTrackManager.requestPermissions().then(async () => { - await this.initDeviceManagers(); - }); - } - let initSuccessful = false; let networkTestFinished = false; const timerId = setTimeout(() => { @@ -457,7 +450,21 @@ export class HMSSdk implements HMSInterface { this.localPeer.asRole = newRole || this.localPeer.role; } const tracks = await this.localTrackManager.getTracksToPublish(config.settings); - tracks.forEach(track => this.setLocalPeerTrack(track)); + tracks.forEach(track => { + this.setLocalPeerTrack(track); + if (track.isTrackNotPublishing()) { + const error = ErrorFactory.TracksErrors.NoDataInTrack( + `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`, + ); + this.sendAnalyticsEvent( + AnalyticsEventFactory.publish({ + devices: this.deviceManager.getDevices(), + error: error, + }), + ); + this.listener?.onError(error); + } + }); this.localPeer?.audioTrack && this.initPreviewTrackAudioLevelMonitor(); await this.initDeviceManagers(); this.sdkState.isPreviewInProgress = false; @@ -1332,6 +1339,18 @@ export class HMSSdk implements HMSInterface { private async setAndPublishTracks(tracks: HMSLocalTrack[]) { for (const track of tracks) { await this.transport.publish([track]); + if (track.isTrackNotPublishing()) { + const error = ErrorFactory.TracksErrors.NoDataInTrack( + `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`, + ); + this.sendAnalyticsEvent( + AnalyticsEventFactory.publish({ + devices: this.deviceManager.getDevices(), + error: error, + }), + ); + this.listener?.onError(error); + } this.setLocalPeerTrack(track); this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, this.localPeer!); } diff --git a/packages/hms-video-store/src/utils/track-audio-level-monitor.ts b/packages/hms-video-store/src/utils/track-audio-level-monitor.ts index fa46e1cd20..c191424a87 100644 --- a/packages/hms-video-store/src/utils/track-audio-level-monitor.ts +++ b/packages/hms-video-store/src/utils/track-audio-level-monitor.ts @@ -133,7 +133,7 @@ export class TrackAudioLevelMonitor { return percent; } - private isSilentThisInstant() { + isSilentThisInstant() { if (!this.analyserNode || !this.dataArray) { HMSLogger.d(this.TAG, 'AudioContext not initialized'); return;