Skip to content

Commit

Permalink
feat: Support codecs switching when possible via sourceBuffer.changeT…
Browse files Browse the repository at this point in the history 
…ype (#841)
  • Loading branch information
@brandonocasey
brandonocasey authored Jun 25, 2020
1 parent 0ca43bd commit 267cc34
Show file tree
Hide file tree
Showing 12 changed files with 1,513 additions and 339 deletions.
276 changes: 161 additions & 115 deletions src/master-playlist-controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -545,9 +545,23 @@ export class MasterPlaylistController extends videojs.EventTarget {
}, ABORT_EARLY_BLACKLIST_SECONDS);
});

this.mainSegmentLoader_.on('trackinfo', () => {
this.tryToCreateSourceBuffers_();
});
const updateCodecs = () => {
if (!this.sourceUpdater_.ready()) {
return this.tryToCreateSourceBuffers_();
}

const codecs = this.getCodecsOrExclude_();

// no codecs means that the playlist was excluded
if (!codecs) {
return;
}

this.sourceUpdater_.addOrChangeSourceBuffers(codecs);
};

this.mainSegmentLoader_.on('trackinfo', updateCodecs);
this.audioSegmentLoader_.on('trackinfo', updateCodecs);

this.mainSegmentLoader_.on('fmp4', () => {
if (!this.triggeredFmp4Usage) {
Expand All @@ -569,10 +583,6 @@ export class MasterPlaylistController extends videojs.EventTarget {
this.logger_('audioSegmentLoader ended');
this.onEndOfStream();
});

this.audioSegmentLoader_.on('trackinfo', () => {
this.tryToCreateSourceBuffers_();
});
}

mediaSecondsLoaded_() {
Expand Down Expand Up @@ -732,17 +742,7 @@ export class MasterPlaylistController extends videojs.EventTarget {
// Only attempt to create the source buffer if none already exist.
// handleSourceOpen is also called when we are "re-opening" a source buffer
// after `endOfStream` has been called (in response to a seek for instance)
try {
this.tryToCreateSourceBuffers_();
} catch (e) {
videojs.log.warn('Failed to create Source Buffers', e);
if (this.mediaSource.readyState !== 'open') {
this.trigger('error');
} else {
this.sourceUpdater_.endOfStream('decode');
}
return;
}
this.tryToCreateSourceBuffers_();

// if autoplay is enabled, begin playback. This is duplicative of
// code in video.js but is required because play() must be invoked
Expand Down Expand Up @@ -1278,124 +1278,164 @@ export class MasterPlaylistController extends videojs.EventTarget {
return this.masterPlaylistLoader_.media() || this.initialMedia_;
}

/**
* Create source buffers and exlude any incompatible renditions.
*
* @private
*/
tryToCreateSourceBuffers_() {
// media source is not ready yet
if (this.mediaSource.readyState !== 'open') {
return;
}
areMediaTypesKnown_() {
const usingAudioLoader = !!this.mediaTypes_.AUDIO.activePlaylistLoader;

// source buffers are already created
if (this.sourceUpdater_.ready()) {
return;
// one or both loaders has not loaded sufficently to get codecs
if (!this.mainSegmentLoader_.startingMedia_ || (usingAudioLoader && !this.audioSegmentLoader_.startingMedia_)) {
return false;
}

const mainStartingMedia = this.mainSegmentLoader_.startingMedia_;
const hasAltAudio = !!this.mediaTypes_.AUDIO.activePlaylistLoader;
return true;
}

getCodecsOrExclude_() {
const media = {
main: this.mainSegmentLoader_.startingMedia_ || {},
audio: this.audioSegmentLoader_.startingMedia_ || {}
};

// Because a URI is required for EXT-X-STREAM-INF tags (therefore, there must always
// be a playlist, even for audio only playlists with alt audio), a segment will always
// be downloaded for the main segment loader, and the track info parsed from it.
// Therefore we must always wait for the segment loader's track info.
if (!mainStartingMedia || (hasAltAudio && !this.audioSegmentLoader_.startingMedia_)) {
return;
}
const audioStartingMedia = this.audioSegmentLoader_ && this.audioSegmentLoader_.startingMedia_ || {};
const media = this.masterPlaylistLoader_.media();
const playlistCodecs = codecsForPlaylist(this.masterPlaylistLoader_.master, media);
// set "main" media equal to video
media.video = media.main;
const playlistCodecs = codecsForPlaylist(this.master(), this.media());
const codecs = {};
const usingAudioLoader = !!this.mediaTypes_.AUDIO.activePlaylistLoader;

// priority of codecs: playlist -> mux.js parsed codecs -> default
if (mainStartingMedia.isMuxed) {
codecs.video = playlistCodecs.video || mainStartingMedia.videoCodec || DEFAULT_VIDEO_CODEC;
codecs.video += ',' + (playlistCodecs.audio || mainStartingMedia.audioCodec || DEFAULT_AUDIO_CODEC);
if (hasAltAudio) {
codecs.audio = playlistCodecs.audio ||
audioStartingMedia.audioCodec ||
DEFAULT_AUDIO_CODEC;
}
} else {
if (mainStartingMedia.hasAudio || hasAltAudio) {
codecs.audio = playlistCodecs.audio ||
mainStartingMedia.audioCodec ||
audioStartingMedia.audioCodec ||
DEFAULT_AUDIO_CODEC;
}
if (media.main.hasVideo) {
codecs.video = playlistCodecs.video || media.main.videoCodec || DEFAULT_VIDEO_CODEC;
}

if (mainStartingMedia.hasVideo) {
codecs.video =
playlistCodecs.video ||
mainStartingMedia.videoCodec ||
DEFAULT_VIDEO_CODEC;
}
if (media.main.isMuxed) {
codecs.video += `,${playlistCodecs.audio || media.main.audioCodec || DEFAULT_AUDIO_CODEC}`;
}

if ((media.main.hasAudio && !media.main.isMuxed) || media.audio.hasAudio) {
codecs.audio = playlistCodecs.audio || media.main.audioCodec || media.audio.audioCodec || DEFAULT_AUDIO_CODEC;
// set audio isFmp4 so we use the correct "supports" function below
media.audio.isFmp4 = (media.main.hasAudio && !media.main.isMuxed) ? media.main.isFmp4 : media.audio.isFmp4;
}

// no codecs, no playback.
if (!codecs.audio && !codecs.video) {
this.blacklistCurrentPlaylist({
playlist: this.media(),
message: 'Could not determine codecs for playlist.',
blacklistDuration: Infinity
});
return;
}

// fmp4 relies on browser support, while ts relies on muxer support
const supportFunction = mainStartingMedia.isFmp4 ? browserSupportsCodec : muxerSupportsCodec;
const unsupportedCodecs = [];
const supportFunction = (isFmp4, codec) => (isFmp4 ? browserSupportsCodec(codec) : muxerSupportsCodec(codec));
const unsupportedCodecs = {};
let unsupportedAudio;

['video', 'audio'].forEach(function(type) {
if (codecs.hasOwnProperty(type) && !supportFunction(media[type].isFmp4, codecs[type])) {
const supporter = media[type].isFmp4 ? 'browser' : 'muxer';

['audio', 'video'].forEach(function(type) {
if (codecs.hasOwnProperty(type) && !supportFunction(codecs[type])) {
unsupportedCodecs.push(codecs[type]);
unsupportedCodecs[supporter] = unsupportedCodecs[supporter] || [];
unsupportedCodecs[supporter].push(codecs[type]);

if (type === 'audio') {
unsupportedAudio = supporter;
}
}
});

if (usingAudioLoader && unsupportedAudio && this.media().attributes.AUDIO) {
const audioGroup = this.media().attributes.AUDIO;

this.mediaTypes_.AUDIO.activePlaylistLoader.pause();
this.audioSegmentLoader_.pause();
this.audioSegmentLoader_.abort();
this.master().playlists.forEach(variant => {
const variantAudioGroup = variant.attributes && variant.attributes.AUDIO;

if (variantAudioGroup === audioGroup && variant !== this.media()) {
variant.excludeUntil = Infinity;
}
});
this.logger_(`excluding audio group ${audioGroup} as ${unsupportedAudio} does not support codec(s): "${codecs.audio}"`);
}

// if we have any unsupported codecs blacklist this playlist.
if (unsupportedCodecs.length) {
const supporter = mainStartingMedia.isFmp4 ? 'browser' : 'muxer';
if (Object.keys(unsupportedCodecs).length) {
const message = Object.keys(unsupportedCodecs).reduce((acc, supporter) => {

if (acc) {
acc += ', ';
}

// reset startingMedia_ when the intial playlist is blacklisted.
this.mainSegmentLoader_.startingMedia_ = void 0;
acc += `${supporter} does not support codec(s): "${unsupportedCodecs[supporter].join(',')}"`;

return acc;
}, '') + '.';

this.blacklistCurrentPlaylist({
playlist: media,
message: `${supporter} does not support codec(s): "${unsupportedCodecs.join(',')}".`,
internal: true
}, Infinity);
playlist: this.media(),
internal: true,
message,
blacklistDuration: Infinity
});
return;
}
// check if codec switching is happening
if (this.sourceUpdater_.ready() && !this.sourceUpdater_.canChangeType()) {
const switchMessages = [];

if (!codecs.video && !codecs.audio) {
const error = 'Failed to create SourceBuffers. No compatible SourceBuffer ' +
'configuration for the variant stream:' + media.resolvedUri;
['video', 'audio'].forEach((type) => {
const newCodec = (parseCodecs(this.sourceUpdater_.codecs[type] || '')[type] || {}).type;
const oldCodec = (parseCodecs(codecs[type] || '')[type] || {}).type;

videojs.log.warn(error);
this.error = error;
if (newCodec && oldCodec && newCodec.toLowerCase() !== oldCodec.toLowerCase()) {
switchMessages.push(`"${this.sourceUpdater_.codecs[type]}" -> "${codecs[type]}"`);
}
});

if (this.mediaSource.readyState !== 'open') {
this.trigger('error');
} else {
this.sourceUpdater_.endOfStream('decode');
if (switchMessages.length) {
this.blacklistCurrentPlaylist({
playlist: this.media(),
message: `Codec switching not supported: ${switchMessages.join(', ')}.`,
blacklistDuration: Infinity,
internal: true
});
return;
}
}

try {
this.sourceUpdater_.createSourceBuffers(codecs);
} catch (e) {
const error = 'Failed to create SourceBuffers: ' + e;
// TODO: when using the muxer shouldn't we just return
// the codecs that the muxer outputs?
return codecs;
}

/**
* Create source buffers and exlude any incompatible renditions.
*
* @private
*/
tryToCreateSourceBuffers_() {
// media source is not ready yet or sourceBuffers are already
// created.
if (this.mediaSource.readyState !== 'open' || this.sourceUpdater_.ready()) {
return;
}

if (!this.areMediaTypesKnown_()) {
return;
}

const codecs = this.getCodecsOrExclude_();

videojs.log.warn(error);
this.error = error;
if (this.mediaSource.readyState !== 'open') {
this.trigger('error');
} else {
this.sourceUpdater_.endOfStream('decode');
}
// no codecs means that the playlist was excluded
if (!codecs) {
return;
}

this.sourceUpdater_.createSourceBuffers(codecs);

const codecString = [codecs.video, codecs.audio].filter(Boolean).join(',');

// TODO:
// blacklisting incompatible renditions will have to change
// once we add support for `changeType` on source buffers.
// We will have to not blacklist any rendition until we try to
// switch to it and learn that it is incompatible and if it is compatible
// we `changeType` on the sourceBuffer.
this.excludeIncompatibleVariants_(codecString);
}

Expand Down Expand Up @@ -1448,24 +1488,30 @@ export class MasterPlaylistController extends videojs.EventTarget {
variantCodecCount = Object.keys(variantCodecs).length;
}

// TODO: we can support this by removing the
// old media source and creating a new one, but it will take some work.
// The number of streams cannot change
if (variantCodecCount !== codecCount) {
blacklistReasons.push(`codec count "${variantCodecCount}" !== "${codecCount}"`);
variant.excludeUntil = Infinity;
}

// the video codec cannot change
if (variantCodecs.video && codecs.video &&
variantCodecs.video.type.toLowerCase() !== codecs.video.type.toLowerCase()) {
blacklistReasons.push(`video codec "${variantCodecs.video.type}" !== "${codecs.video.type}"`);
variant.excludeUntil = Infinity;
}
// only exclude playlists by codec change, if codecs cannot switch
// during playback.
if (!this.sourceUpdater_.canChangeType()) {
// the video codec cannot change
if (variantCodecs.video && codecs.video &&
variantCodecs.video.type.toLowerCase() !== codecs.video.type.toLowerCase()) {
blacklistReasons.push(`video codec "${variantCodecs.video.type}" !== "${codecs.video.type}"`);
variant.excludeUntil = Infinity;
}

// the audio codec cannot change
if (variantCodecs.audio && codecs.audio &&
variantCodecs.audio.type.toLowerCase() !== codecs.audio.type.toLowerCase()) {
variant.excludeUntil = Infinity;
blacklistReasons.push(`audio codec "${variantCodecs.audio.type}" !== "${codecs.audio.type}"`);
// the audio codec cannot change
if (variantCodecs.audio && codecs.audio &&
variantCodecs.audio.type.toLowerCase() !== codecs.audio.type.toLowerCase()) {
variant.excludeUntil = Infinity;
blacklistReasons.push(`audio codec "${variantCodecs.audio.type}" !== "${codecs.audio.type}"`);
}
}

if (blacklistReasons.length) {
Expand Down
9 changes: 6 additions & 3 deletions src/media-groups.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -736,10 +736,13 @@ export const setupMediaGroups = (settings) => {
// DO NOT enable the default subtitle or caption track.
// DO enable the default audio track
const audioGroup = mediaTypes.AUDIO.activeGroup();
const groupId = (audioGroup.filter(group => group.default)[0] || audioGroup[0]).id;

mediaTypes.AUDIO.tracks[groupId].enabled = true;
mediaTypes.AUDIO.onTrackChanged();
if (audioGroup) {
const groupId = (audioGroup.filter(group => group.default)[0] || audioGroup[0]).id;

mediaTypes.AUDIO.tracks[groupId].enabled = true;
mediaTypes.AUDIO.onTrackChanged();
}

masterPlaylistLoader.on('mediachange', () => {
['AUDIO', 'SUBTITLES'].forEach(type => mediaTypes[type].onGroupChanged());
Expand Down
6 changes: 5 additions & 1 deletion src/media-segment-request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,8 @@ const transmuxAndNotify = ({
if (probeResult) {
trackInfoFn(segment, {
hasAudio: probeResult.hasAudio,
hasVideo: probeResult.hasVideo
hasVideo: probeResult.hasVideo,
isMuxed
});
trackInfoFn = null;

Expand Down Expand Up @@ -318,6 +319,9 @@ const transmuxAndNotify = ({
},
onTrackInfo: (trackInfo) => {
if (trackInfoFn) {
if (isMuxed) {
trackInfo.isMuxed = true;
}
trackInfoFn(segment, trackInfo);
}
},
Expand Down
Loading

0 comments on commit 267cc34

Please sign in to comment.