[media] Support configurable audio write ahead (#1882)

Enables the web app to configure the audio write ahead duration through
an extension, and enables the extension for linux-x64x11.

Se the original PR #121 for
details.

b/267678497
b/307832651

---------

Co-authored-by: xiaomings <xiaomings@google.com>

cobalt/demos/content/media-element-demo/src/components/player.ts

@@ -25,6 +25,18 @@ export class Player extends Component<Props> {
   /** The <video> element. */
   private videoEl!: HTMLVideoElement;
 
+  /** The video SourceBuffer */
+  private videoSourceBuffer!: SourceBuffer;
+
+  /** The audio SourceBuffer */
+  private audioSourceBuffer!: SourceBuffer;
+
+  /** max(videoSourceBuffer.writeHead - videoEl.currentTime) */
+  private maxVideoWriteHeadDistance!: number;
+
+  /** max(audioSourceBuffer.writeHead - videoEl.currentTime) */
+  private maxAudioWriteHeadDistance!: number;
+
   /** The element displaying video download buffer info. */
   private videoDownloadBufferInfo!: Element;
 
@@ -53,6 +65,8 @@ export class Player extends Component<Props> {
     super(props);
     this.videos = convertToMediaArray(props.video);
     this.audios = convertToMediaArray(props.audio);
+    this.maxVideoWriteHeadDistance = 0;
+    this.maxAudioWriteHeadDistance = 0;
   }
 
   /** @override */
@@ -93,12 +107,37 @@ export class Player extends Component<Props> {
   }
 
   private renderVideoInfo() {
+    var h5vccAudioConnectors = '';
+    try {
+      h5vccAudioConnectors = this.videoEl.h5vccAudioConnectors;
+    } catch (error) {}
     renderComponent(
         VideoInfo, {
           duration: this.videoEl.duration,
           currentTime: this.videoEl.currentTime,
+          audioConnectors: h5vccAudioConnectors,
         },
         this.videoInfo);
+    if (this.videoSourceBuffer) {
+      this.maxVideoWriteHeadDistance =
+          Math.max(this.maxVideoWriteHeadDistance,
+                   this.videoSourceBuffer.writeHead - this.videoEl.currentTime);
+      renderComponent(
+        SourceBufferInfo,
+          {name: 'Video', sourceBuffer: this.videoSourceBuffer,
+           maxWriteHeadDistance: this.maxVideoWriteHeadDistance},
+         this.videoSourceBufferInfo);
+      }
+    if (this.audioSourceBuffer) {
+      this.maxAudioWriteHeadDistance =
+          Math.max(this.maxAudioWriteHeadDistance,
+                   this.audioSourceBuffer.writeHead - this.videoEl.currentTime);
+      renderComponent(
+        SourceBufferInfo,
+        {name: 'Audio', sourceBuffer: this.audioSourceBuffer,
+         maxWriteHeadDistance: this.maxAudioWriteHeadDistance},
+        this.audioSourceBufferInfo);
+    }
   }
 
   private async play() {
@@ -149,7 +188,7 @@ export class Player extends Component<Props> {
 
   /**
    * Plays all videos as adaptive videos.
-   * TODO: dynmaically calculate the source buffer MIME.
+   * TODO: dynamically calculate the source buffer MIME.
    */
   private playAdaptiveVideo() {
     const ms = new MediaSource();
@@ -158,12 +197,7 @@ export class Player extends Component<Props> {
       if (this.videos.length > 0) {
         const videoSourceBuffer =
             ms.addSourceBuffer('video/mp4; codecs="avc1.640028"');
-        videoSourceBuffer.addEventListener('updateend', () => {
-          renderComponent(
-              SourceBufferInfo,
-              {name: 'Video', sourceBuffer: videoSourceBuffer},
-              this.videoSourceBufferInfo);
-        });
+        this.videoSourceBuffer = videoSourceBuffer;
         const downloadBuffer = new DownloadBuffer(this.videos);
         downloadBuffer.register((reportMap) => {
           renderComponent(
@@ -176,12 +210,7 @@ export class Player extends Component<Props> {
       if (this.audios.length > 0) {
         const audioSourceBuffer =
             ms.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"');
-        audioSourceBuffer.addEventListener('updateend', () => {
-          renderComponent(
-              SourceBufferInfo,
-              {name: 'Audio', sourceBuffer: audioSourceBuffer},
-              this.audioSourceBufferInfo);
-        });
+        this.audioSourceBuffer = audioSourceBuffer;
         const downloadBuffer = new DownloadBuffer(this.audios);
         downloadBuffer.register(
             (reportMap) => {renderComponent(

cobalt/demos/content/media-element-demo/src/components/source_buffer_info.ts

@@ -1,9 +1,12 @@
 interface Props {
   sourceBuffer: SourceBuffer;
   name: string;
+  maxWriteHeadDistance: number;
 }
 
 /** A component that displays the source buffer info. */
-export function SourceBufferInfo({sourceBuffer, name}: Props) {
-  return `<div>${name} buffered: ${sourceBuffer.buffered.end(0)} sec</div>`;
+export function SourceBufferInfo({sourceBuffer, name, maxWriteHeadDistance}: Props) {
+  return `<div>${name} buffered: ${sourceBuffer.buffered.end(0)} sec` +
+      `, writeHead: ${sourceBuffer.writeHead} sec` +
+      `, maxWriteHeadDistance: ${maxWriteHeadDistance} sec</div>`;
 }

cobalt/demos/content/media-element-demo/src/components/video_info.ts

@@ -1,9 +1,13 @@
 interface Props {
   duration: number;
   currentTime: number;
+  audioConnectors: string;
 }
 
 /** A component that displays video info. */
-export function VideoInfo({duration, currentTime}: Props) {
+export function VideoInfo({duration, currentTime, audioConnectors}: Props) {
+  if (audioConnectors) {
+    return `<div>${currentTime} / ${duration} / audioConnectors: ${audioConnectors}</div>`;
+  }
   return `<div>${currentTime} / ${duration}</div>`;
 }

cobalt/dom/html_media_element.cc

@@ -19,13 +19,15 @@
 #include <limits>
 #include <memory>
 #include <utility>
+#include <vector>
 
 #include "base/bind.h"
 #include "base/compiler_specific.h"
 #include "base/guid.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/message_loop/message_loop.h"
+#include "base/strings/string_util.h"
 #include "base/trace_event/trace_event.h"
 #include "cobalt/base/instance_counter.h"
 #include "cobalt/base/tokens.h"
@@ -36,6 +38,7 @@
 #include "cobalt/dom/media_settings.h"
 #include "cobalt/dom/media_source.h"
 #include "cobalt/dom/media_source_ready_state.h"
+#include "cobalt/extension/audio_write_ahead.h"
 #include "cobalt/loader/fetcher_factory.h"
 #include "cobalt/media/url_fetcher_data_source.h"
 #include "cobalt/media/web_media_player_factory.h"
@@ -45,6 +48,7 @@
 #include "cobalt/web/dom_exception.h"
 #include "cobalt/web/event.h"
 #include "cobalt/web/web_settings.h"
+#include "starboard/system.h"
 
 #include "cobalt/dom/eme/media_encrypted_event.h"
 #include "cobalt/dom/eme/media_encrypted_event_init.h"
@@ -641,6 +645,32 @@ void HTMLMediaElement::ScheduleEvent(const scoped_refptr<web::Event>& event) {
   event_queue_.Enqueue(event);
 }
 
+std::string HTMLMediaElement::h5vcc_audio_connectors(
+    script::ExceptionState* exception_state) const {
+  static const CobaltExtensionConfigurableAudioWriteAheadApi* extension_api =
+      static_cast<const CobaltExtensionConfigurableAudioWriteAheadApi*>(
+          SbSystemGetExtension(
+              kCobaltExtensionConfigurableAudioWriteAheadName));
+  if (extension_api) {
+    if (!player_) {
+      web::DOMException::Raise(web::DOMException::kInvalidStateErr,
+                               exception_state);
+      return std::string();
+    }
+
+    DCHECK_EQ(extension_api->name,
+              std::string(kCobaltExtensionConfigurableAudioWriteAheadName));
+    DCHECK_EQ(extension_api->version, 1u);
+
+    std::vector<std::string> configs = player_->GetAudioConnectors();
+    return base::JoinString(configs, ";");
+  }
+
+  web::DOMException::Raise(web::DOMException::kNotSupportedErr,
+                           exception_state);
+  return std::string();
+}
+
 void HTMLMediaElement::CreateMediaPlayer() {
   TRACE_EVENT0("cobalt::dom", "HTMLMediaElement::CreateMediaPlayer()");
   LOG(INFO) << "Create media player.";

cobalt/dom/html_media_element.h

@@ -146,6 +146,12 @@ class HTMLMediaElement : public HTMLElement,
   // function won't modify the target of the |event| passed in.
   void ScheduleEvent(const scoped_refptr<web::Event>& event);
 
+  // Returns semicolon separated names of audio connectors, like
+  // "hdmi;bluetooth".
+  // TODO(b/267678497): The current interface is tentative, to be refined.
+  std::string h5vcc_audio_connectors(
+      script::ExceptionState* exception_state) const;
+
   // Set max video capabilities.
   void SetMaxVideoCapabilities(const std::string& max_video_capabilities,
                                script::ExceptionState* exception_state);

cobalt/dom/html_media_element.idl

@@ -61,4 +61,10 @@ interface HTMLMediaElement : HTMLElement {
   attribute boolean controls;
   [RaisesException] attribute double volume;
   attribute boolean muted;
+
+  // non standard, semicolon separated names of audio connectors, like
+  // "hdmi;bluetooth".  It raises `NotSupportedError` on apps doesn't support
+  // this feature, or `InvalidStateError` if there isn't an active playback.
+  // TODO(b/267678497): The current interface is tentative, to be refined.
+  [RaisesException] readonly attribute DOMString h5vccAudioConnectors;
 };

cobalt/dom/source_buffer.cc

@@ -453,6 +453,17 @@ void SourceBuffer::set_track_defaults(
   track_defaults_ = track_defaults;
 }
 
+double SourceBuffer::write_head(script::ExceptionState* exception_state) const {
+  if (media_source_ == NULL) {
+    web::DOMException::Raise(web::DOMException::kInvalidStateErr,
+                             exception_state);
+    return 0.0;
+  }
+
+  DCHECK(chunk_demuxer_);
+  return chunk_demuxer_->GetWriteHead(id_).InSecondsF();
+}
+
 void SourceBuffer::OnRemovedFromMediaSource() {
   if (media_source_ == NULL) {
     return;

cobalt/dom/source_buffer.h

@@ -134,6 +134,9 @@ class SourceBuffer : public web::EventTarget {
 
   // Custom, not in any spec.
   //
+  // Return the highest presentation timestamp written to SbPlayer.
+  double write_head(script::ExceptionState* exception_state) const;
+
   void OnRemovedFromMediaSource();
   double GetHighestPresentationTimestamp() const;
 

cobalt/dom/source_buffer.idl

@@ -38,4 +38,10 @@ interface SourceBuffer : EventTarget {
   // [RaisesException] void abort();
   [RaisesException] void remove(double start, unrestricted double end);
   [RaisesException] attribute TrackDefaultList trackDefaults;
+
+  // Non standard interface (b/267678497).
+  // Returns the highest presentation timestamp written to SbPlayer, raises
+  // `InvalidStateError` if the SourceBuffer object has been removed from the
+  // MediaSource object.
+  [RaisesException] readonly attribute double writeHead;
 };

cobalt/extension/audio_write_ahead.h

@@ -0,0 +1,75 @@
+// Copyright 2023 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef COBALT_EXTENSION_AUDIO_WRITE_AHEAD_H_
+#define COBALT_EXTENSION_AUDIO_WRITE_AHEAD_H_
+
+#include "starboard/media.h"
+#include "starboard/player.h"
+#include "starboard/time.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define kCobaltExtensionConfigurableAudioWriteAheadName \
+  "dev.cobalt.extension.ConfigurableAudioWriteAhead"
+
+#define kCobaltExtensionPlayerWriteDurationLocal (kSbTimeSecond / 2)
+#define kCobaltExtensionPlayerWriteDurationRemote (kSbTimeSecond * 10)
+
+typedef enum CobaltExtensionMediaAudioConnector {
+  kCobaltExtensionMediaAudioConnectorUnknown,
+
+  kCobaltExtensionMediaAudioConnectorAnalog,
+  kCobaltExtensionMediaAudioConnectorBluetooth,
+  kCobaltExtensionMediaAudioConnectorBuiltIn,
+  kCobaltExtensionMediaAudioConnectorHdmi,
+  kCobaltExtensionMediaAudioConnectorRemoteWired,
+  kCobaltExtensionMediaAudioConnectorRemoteWireless,
+  kCobaltExtensionMediaAudioConnectorRemoteOther,
+  kCobaltExtensionMediaAudioConnectorSpdif,
+  kCobaltExtensionMediaAudioConnectorUsb,
+} CobaltExtensionMediaAudioConnector;
+
+typedef struct CobaltExtensionMediaAudioConfiguration {
+  int index;
+  CobaltExtensionMediaAudioConnector connector;
+  SbTime latency;
+  SbMediaAudioCodingType coding_type;
+  int number_of_channels;
+} CobaltExtensionMediaAudioConfiguration;
+
+typedef struct CobaltExtensionConfigurableAudioWriteAheadApi {
+  // Name should be the string
+  // |kCobaltExtensionConfigurableAudioWriteAheadName|. This helps to validate
+  // that the extension API is correct.
+  const char* name;
+
+  // This specifies the version of the API that is implemented.
+  uint32_t version;
+
+  // The fields below this point were added in version 1 or later.
+
+  bool (*PlayerGetAudioConfiguration)(
+      SbPlayer player, int index,
+      CobaltExtensionMediaAudioConfiguration* out_audio_configuration);
+
+} CobaltExtensionConfigurableAudioWriteAheadApi;
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // COBALT_EXTENSION_AUDIO_WRITE_AHEAD_H_

cobalt/media/base/pipeline.h

@@ -98,6 +98,7 @@ class MEDIA_EXPORT Pipeline : public base::RefCountedThreadSafe<Pipeline> {
       const GetDecodeTargetGraphicsContextProviderFunc&
           get_decode_target_graphics_context_provider_func,
       bool allow_resume_after_suspend, bool allow_batched_sample_write,
+      SbTime audio_write_duration_local, SbTime audio_write_duration_remote,
       MediaLog* media_log, DecodeTargetProvider* decode_target_provider);
 
   virtual ~Pipeline() {}
@@ -217,6 +218,9 @@ class MEDIA_EXPORT Pipeline : public base::RefCountedThreadSafe<Pipeline> {
   // be 0.
   virtual void GetNaturalVideoSize(gfx::Size* out_size) const = 0;
 
+  // Gets the names of audio connectors used by the audio output.
+  virtual std::vector<std::string> GetAudioConnectors() const = 0;
+
   // Return true if loading progress has been made since the last time this
   // method was called.
   virtual bool DidLoadingProgress() const = 0;

cobalt/media/base/sbplayer_bridge.cc

@@ -370,6 +370,36 @@ void SbPlayerBridge::GetInfo(uint32* video_frames_decoded,
   GetInfo_Locked(video_frames_decoded, video_frames_dropped, media_time);
 }
 
+std::vector<CobaltExtensionMediaAudioConfiguration>
+SbPlayerBridge::GetAudioConfigurations() {
+  DCHECK(sbplayer_interface_->IsAudioWriteAheadExtensionEnabled());
+  base::AutoLock auto_lock(lock_);
+
+  if (!SbPlayerIsValid(player_)) {
+    return std::vector<CobaltExtensionMediaAudioConfiguration>();
+  }
+
+  std::vector<CobaltExtensionMediaAudioConfiguration> configurations;
+
+  // Set a limit to avoid infinite loop.
+  constexpr int kMaxAudioConfigurations = 32;
+
+  for (int i = 0; i < kMaxAudioConfigurations; ++i) {
+    CobaltExtensionMediaAudioConfiguration configuration;
+    if (!sbplayer_interface_->GetAudioConfiguration(player_, i,
+                                                    &configuration)) {
+      break;
+    }
+
+    configurations.push_back(configuration);
+  }
+
+  LOG_IF(WARNING, configurations.empty())
+      << "Failed to find any audio configurations.";
+
+  return configurations;
+}
+
 #if SB_HAS(PLAYER_WITH_URL)
 void SbPlayerBridge::GetUrlPlayerBufferedTimeRanges(
     base::TimeDelta* buffer_start_time, base::TimeDelta* buffer_length_time) {