fix: add an event when tracks are published with no data (#3298)

Co-authored-by: Yogesh Singh <yogesh@100ms.live>

packages/hms-video-store/src/error/ErrorCodes.ts

@@ -76,6 +76,9 @@ export const ErrorCodes = {
 
     // Selected device not detected on change
     SELECTED_DEVICE_MISSING: 3014,
+
+    // Track is publishing with no data, can happen when a whatsapp call is ongoing before 100ms call in mweb
+    NO_DATA: 3015,
   },
 
   WebrtcErrors: {

packages/hms-video-store/src/error/ErrorFactory.ts

@@ -251,6 +251,16 @@ export const ErrorFactory = {
         false,
       );
     },
+
+    NoDataInTrack(description: string) {
+      return new HMSException(
+        ErrorCodes.TracksErrors.NO_DATA,
+        'Track does not have any data',
+        HMSAction.TRACK,
+        description,
+        'This could possibily due to another application taking priority over the access to camera or microphone or due to an incoming call',
+      );
+    },
   },
 
   WebrtcErrors: {

packages/hms-video-store/src/interfaces/config.ts

@@ -42,10 +42,6 @@ export interface HMSConfig {
   audioSinkElementId?: string;
   autoVideoSubscribe?: boolean;
   initEndpoint?: string;
-  /**
-   * Request Camera/Mic permissions irrespective of role to avoid delay in getting device list
-   */
-  alwaysRequestPermissions?: boolean;
   /**
    * Enable to get a network quality score while in preview. The score ranges from -1 to 5.
    * -1 when we are not able to connect to 100ms servers within an expected time limit

packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts

@@ -56,6 +56,7 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
   ) {
     super(stream, track, source);
     stream.tracks.push(this);
+    this.addTrackEventListeners(track);
 
     this.settings = settings;
     // Replace the 'default' or invalid deviceId with the actual deviceId
@@ -99,13 +100,9 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     this.manuallySelectedDeviceId = undefined;
   }
 
-  private isTrackNotPublishing = () => {
-    return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted;
-  };
-
   private handleVisibilityChange = async () => {
     // track state is fine do nothing
-    if (!this.isTrackNotPublishing()) {
+    if (!this.shouldReacquireTrack()) {
       HMSLogger.d(this.TAG, `visibiltiy: ${document.visibilityState}`, `${this}`);
       return;
     }
@@ -151,16 +148,19 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
      * no audio when the above getAudioTrack throws an error. ex: DeviceInUse error
      */
     prevTrack?.stop();
+    this.removeTrackEventListeners(prevTrack);
     this.tracksCreated.forEach(track => track.stop());
     this.tracksCreated.clear();
     try {
       const newTrack = await getAudioTrack(settings);
+      this.addTrackEventListeners(newTrack);
       this.tracksCreated.add(newTrack);
       HMSLogger.d(this.TAG, 'replaceTrack, Previous track stopped', prevTrack, 'newTrack', newTrack);
       await this.updateTrack(newTrack);
     } catch (e) {
       // Generate a new track from previous settings so there will be audio because previous track is stopped
       const newTrack = await getAudioTrack(this.settings);
+      this.addTrackEventListeners(newTrack);
       this.tracksCreated.add(newTrack);
       await this.updateTrack(newTrack);
       if (this.isPublished) {
@@ -184,8 +184,8 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
       return;
     }
 
-    // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled.
-    if (value && (isEmptyTrack(this.nativeTrack) || this.nativeTrack.muted)) {
+    // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled or ended track or when silence is detected.
+    if (value && this.shouldReacquireTrack()) {
       await this.replaceTrackWith(this.settings);
     }
     await super.setEnabled(value);
@@ -303,6 +303,40 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     return this.processedTrack || this.nativeTrack;
   }
 
+  private addTrackEventListeners(track: MediaStreamTrack) {
+    track.addEventListener('mute', this.handleTrackMute);
+    track.addEventListener('unmute', this.handleTrackUnmute);
+  }
+
+  private removeTrackEventListeners(track: MediaStreamTrack) {
+    track.removeEventListener('mute', this.handleTrackMute);
+    track.removeEventListener('unmute', this.handleTrackUnmute);
+  }
+
+  private handleTrackMute = () => {
+    HMSLogger.d(this.TAG, 'muted natively');
+    const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call';
+    this.eventBus.analytics.publish(
+      this.sendInterruptionEvent({
+        started: true,
+        reason,
+      }),
+    );
+  };
+
+  /** @internal */
+  handleTrackUnmute = () => {
+    HMSLogger.d(this.TAG, 'unmuted natively');
+    const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call';
+    this.eventBus.analytics.publish(
+      this.sendInterruptionEvent({
+        started: false,
+        reason,
+      }),
+    );
+    this.setEnabled(this.enabled);
+  };
+
   private replaceSenderTrack = async () => {
     if (!this.transceiver || this.transceiver.direction !== 'sendonly') {
       HMSLogger.d(this.TAG, `transceiver for ${this.trackId} not available or not connected yet`);
@@ -311,6 +345,12 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     await this.transceiver.sender.replaceTrack(this.processedTrack || this.nativeTrack);
   };
 
+  private shouldReacquireTrack = () => {
+    return (
+      isEmptyTrack(this.nativeTrack) || this.isTrackNotPublishing() || this.audioLevelMonitor?.isSilentThisInstant()
+    );
+  };
+
   private buildNewSettings(settings: Partial<HMSAudioTrackSettings>) {
     const { volume, codec, maxBitrate, deviceId, advanced, audioMode } = { ...this.settings, ...settings };
     const newSettings = new HMSAudioTrackSettings(volume, codec, maxBitrate, deviceId, advanced, audioMode);

packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts

@@ -131,6 +131,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
    * use this function to set the enabled state of a track. If true the track will be unmuted and muted otherwise.
    * @param value
    */
+  // eslint-disable-next-line complexity
   async setEnabled(value: boolean): Promise<void> {
     if (value === this.enabled) {
       return;
@@ -544,6 +545,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
     super.handleTrackUnmute();
     this.eventBus.localVideoEnabled.publish({ enabled: this.enabled, track: this });
     this.eventBus.localVideoUnmutedNatively.publish();
+    this.setEnabled(this.enabled);
   };
 
   /**

packages/hms-video-store/src/media/tracks/HMSTrack.ts

@@ -85,6 +85,11 @@ export abstract class HMSTrack {
   protected setFirstTrackId(trackId: string) {
     this.firstTrackId = trackId;
   }
+
+  isTrackNotPublishing = () => {
+    return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted;
+  };
+
   /**
    * @internal
    * It will send event to analytics when interruption start/stop

packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts

@@ -93,8 +93,8 @@ export class HMSVideoTrack extends HMSTrack {
 
   private reTriggerPlay = ({ videoElement }: { videoElement: HTMLVideoElement }) => {
     setTimeout(() => {
-      videoElement.play().catch(() => {
-        HMSLogger.w('[HMSVideoTrack]', 'failed to play');
+      videoElement.play().catch((e: Error) => {
+        HMSLogger.w('[HMSVideoTrack]', 'failed to play', e.message);
       });
     }, 0);
   };

packages/hms-video-store/src/sdk/LocalTrackManager.test.ts

@@ -109,6 +109,7 @@ const mockMediaStream = {
       kind: 'video',
       getSettings: jest.fn(() => ({ deviceId: 'video-device-id' })),
       addEventListener: jest.fn(() => {}),
+      removeEventListener: jest.fn(() => {}),
     },
   ]),
   getAudioTracks: jest.fn(() => [
@@ -117,6 +118,7 @@ const mockMediaStream = {
       kind: 'audio',
       getSettings: jest.fn(() => ({ deviceId: 'audio-device-id' })),
       addEventListener: jest.fn(() => {}),
+      removeEventListener: jest.fn(() => {}),
     },
   ]),
   addTrack: jest.fn(() => {}),
@@ -206,7 +208,13 @@ const mockAudioContext = {
     return {
       stream: {
         getAudioTracks: jest.fn(() => [
-          { id: 'audio-id', kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })) },
+          {
+            id: 'audio-id',
+            kind: 'audio',
+            getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })),
+            addEventListener: jest.fn(() => {}),
+            removeEventListener: jest.fn(() => {}),
+          },
         ]),
       },
     };
@@ -426,6 +434,7 @@ describe('LocalTrackManager', () => {
           kind: 'video',
           getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }),
           addEventListener: jest.fn(() => {}),
+          removeEventListener: jest.fn(() => {}),
         } as unknown as MediaStreamTrack,
         HMSPeerType.REGULAR,
         testEventBus,
@@ -459,6 +468,7 @@ describe('LocalTrackManager', () => {
           kind: 'video',
           getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }),
           addEventListener: jest.fn(() => {}),
+          removeEventListener: jest.fn(() => {}),
         } as unknown as MediaStreamTrack,
         HMSPeerType.REGULAR,
         testEventBus,

packages/hms-video-store/src/sdk/index.ts

@@ -435,13 +435,6 @@ export class HMSSdk implements HMSInterface {
     this.analyticsTimer.start(TimedEvent.PREVIEW);
     this.setUpPreview(config, listener);
 
-    // Request permissions and populate devices before waiting for policy
-    if (config.alwaysRequestPermissions) {
-      this.localTrackManager.requestPermissions().then(async () => {
-        await this.initDeviceManagers();
-      });
-    }
-
     let initSuccessful = false;
     let networkTestFinished = false;
     const timerId = setTimeout(() => {
@@ -457,7 +450,21 @@ export class HMSSdk implements HMSInterface {
           this.localPeer.asRole = newRole || this.localPeer.role;
         }
         const tracks = await this.localTrackManager.getTracksToPublish(config.settings);
-        tracks.forEach(track => this.setLocalPeerTrack(track));
+        tracks.forEach(track => {
+          this.setLocalPeerTrack(track);
+          if (track.isTrackNotPublishing()) {
+            const error = ErrorFactory.TracksErrors.NoDataInTrack(
+              `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`,
+            );
+            this.sendAnalyticsEvent(
+              AnalyticsEventFactory.publish({
+                devices: this.deviceManager.getDevices(),
+                error: error,
+              }),
+            );
+            this.listener?.onError(error);
+          }
+        });
         this.localPeer?.audioTrack && this.initPreviewTrackAudioLevelMonitor();
         await this.initDeviceManagers();
         this.sdkState.isPreviewInProgress = false;
@@ -1332,6 +1339,18 @@ export class HMSSdk implements HMSInterface {
   private async setAndPublishTracks(tracks: HMSLocalTrack[]) {
     for (const track of tracks) {
       await this.transport.publish([track]);
+      if (track.isTrackNotPublishing()) {
+        const error = ErrorFactory.TracksErrors.NoDataInTrack(
+          `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`,
+        );
+        this.sendAnalyticsEvent(
+          AnalyticsEventFactory.publish({
+            devices: this.deviceManager.getDevices(),
+            error: error,
+          }),
+        );
+        this.listener?.onError(error);
+      }
       this.setLocalPeerTrack(track);
       this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, this.localPeer!);
     }

packages/hms-video-store/src/utils/track-audio-level-monitor.ts

@@ -133,7 +133,7 @@ export class TrackAudioLevelMonitor {
     return percent;
   }
 
-  private isSilentThisInstant() {
+  isSilentThisInstant() {
     if (!this.analyserNode || !this.dataArray) {
       HMSLogger.d(this.TAG, 'AudioContext not initialized');
       return;