From 267cc3452435f59106846b1b64c2c94a986fe9e7 Mon Sep 17 00:00:00 2001 From: Brandon Casey <2381475+brandonocasey@users.noreply.github.com> Date: Thu, 25 Jun 2020 17:32:50 -0400 Subject: [PATCH] feat: Support codecs switching when possible via sourceBuffer.changeType (#841) --- src/master-playlist-controller.js | 276 ++++++---- src/media-groups.js | 9 +- src/media-segment-request.js | 6 +- src/segment-loader.js | 19 +- src/source-updater.js | 454 ++++++++++++---- src/util/shallow-equal.js | 41 ++ src/util/to-title-case.js | 9 + test/manifests/dash-many-codecs.mpd | 4 +- test/master-playlist-controller.test.js | 690 +++++++++++++++++++++++- test/source-updater.test.js | 256 ++++++--- test/test-helpers.js | 10 + test/videojs-http-streaming.test.js | 78 ++- 12 files changed, 1513 insertions(+), 339 deletions(-) create mode 100644 src/util/shallow-equal.js create mode 100644 src/util/to-title-case.js diff --git a/src/master-playlist-controller.js b/src/master-playlist-controller.js index 66bf32afe..abb1c5a24 100644 --- a/src/master-playlist-controller.js +++ b/src/master-playlist-controller.js @@ -545,9 +545,23 @@ export class MasterPlaylistController extends videojs.EventTarget { }, ABORT_EARLY_BLACKLIST_SECONDS); }); - this.mainSegmentLoader_.on('trackinfo', () => { - this.tryToCreateSourceBuffers_(); - }); + const updateCodecs = () => { + if (!this.sourceUpdater_.ready()) { + return this.tryToCreateSourceBuffers_(); + } + + const codecs = this.getCodecsOrExclude_(); + + // no codecs means that the playlist was excluded + if (!codecs) { + return; + } + + this.sourceUpdater_.addOrChangeSourceBuffers(codecs); + }; + + this.mainSegmentLoader_.on('trackinfo', updateCodecs); + this.audioSegmentLoader_.on('trackinfo', updateCodecs); this.mainSegmentLoader_.on('fmp4', () => { if (!this.triggeredFmp4Usage) { @@ -569,10 +583,6 @@ export class MasterPlaylistController extends videojs.EventTarget { this.logger_('audioSegmentLoader ended'); this.onEndOfStream(); }); - - this.audioSegmentLoader_.on('trackinfo', () => { - this.tryToCreateSourceBuffers_(); - }); } mediaSecondsLoaded_() { @@ -732,17 +742,7 @@ export class MasterPlaylistController extends videojs.EventTarget { // Only attempt to create the source buffer if none already exist. // handleSourceOpen is also called when we are "re-opening" a source buffer // after `endOfStream` has been called (in response to a seek for instance) - try { - this.tryToCreateSourceBuffers_(); - } catch (e) { - videojs.log.warn('Failed to create Source Buffers', e); - if (this.mediaSource.readyState !== 'open') { - this.trigger('error'); - } else { - this.sourceUpdater_.endOfStream('decode'); - } - return; - } + this.tryToCreateSourceBuffers_(); // if autoplay is enabled, begin playback. This is duplicative of // code in video.js but is required because play() must be invoked @@ -1278,124 +1278,164 @@ export class MasterPlaylistController extends videojs.EventTarget { return this.masterPlaylistLoader_.media() || this.initialMedia_; } - /** - * Create source buffers and exlude any incompatible renditions. - * - * @private - */ - tryToCreateSourceBuffers_() { - // media source is not ready yet - if (this.mediaSource.readyState !== 'open') { - return; - } + areMediaTypesKnown_() { + const usingAudioLoader = !!this.mediaTypes_.AUDIO.activePlaylistLoader; - // source buffers are already created - if (this.sourceUpdater_.ready()) { - return; + // one or both loaders has not loaded sufficently to get codecs + if (!this.mainSegmentLoader_.startingMedia_ || (usingAudioLoader && !this.audioSegmentLoader_.startingMedia_)) { + return false; } - const mainStartingMedia = this.mainSegmentLoader_.startingMedia_; - const hasAltAudio = !!this.mediaTypes_.AUDIO.activePlaylistLoader; + return true; + } + + getCodecsOrExclude_() { + const media = { + main: this.mainSegmentLoader_.startingMedia_ || {}, + audio: this.audioSegmentLoader_.startingMedia_ || {} + }; - // Because a URI is required for EXT-X-STREAM-INF tags (therefore, there must always - // be a playlist, even for audio only playlists with alt audio), a segment will always - // be downloaded for the main segment loader, and the track info parsed from it. - // Therefore we must always wait for the segment loader's track info. - if (!mainStartingMedia || (hasAltAudio && !this.audioSegmentLoader_.startingMedia_)) { - return; - } - const audioStartingMedia = this.audioSegmentLoader_ && this.audioSegmentLoader_.startingMedia_ || {}; - const media = this.masterPlaylistLoader_.media(); - const playlistCodecs = codecsForPlaylist(this.masterPlaylistLoader_.master, media); + // set "main" media equal to video + media.video = media.main; + const playlistCodecs = codecsForPlaylist(this.master(), this.media()); const codecs = {}; + const usingAudioLoader = !!this.mediaTypes_.AUDIO.activePlaylistLoader; - // priority of codecs: playlist -> mux.js parsed codecs -> default - if (mainStartingMedia.isMuxed) { - codecs.video = playlistCodecs.video || mainStartingMedia.videoCodec || DEFAULT_VIDEO_CODEC; - codecs.video += ',' + (playlistCodecs.audio || mainStartingMedia.audioCodec || DEFAULT_AUDIO_CODEC); - if (hasAltAudio) { - codecs.audio = playlistCodecs.audio || - audioStartingMedia.audioCodec || - DEFAULT_AUDIO_CODEC; - } - } else { - if (mainStartingMedia.hasAudio || hasAltAudio) { - codecs.audio = playlistCodecs.audio || - mainStartingMedia.audioCodec || - audioStartingMedia.audioCodec || - DEFAULT_AUDIO_CODEC; - } + if (media.main.hasVideo) { + codecs.video = playlistCodecs.video || media.main.videoCodec || DEFAULT_VIDEO_CODEC; + } - if (mainStartingMedia.hasVideo) { - codecs.video = - playlistCodecs.video || - mainStartingMedia.videoCodec || - DEFAULT_VIDEO_CODEC; - } + if (media.main.isMuxed) { + codecs.video += `,${playlistCodecs.audio || media.main.audioCodec || DEFAULT_AUDIO_CODEC}`; + } + + if ((media.main.hasAudio && !media.main.isMuxed) || media.audio.hasAudio) { + codecs.audio = playlistCodecs.audio || media.main.audioCodec || media.audio.audioCodec || DEFAULT_AUDIO_CODEC; + // set audio isFmp4 so we use the correct "supports" function below + media.audio.isFmp4 = (media.main.hasAudio && !media.main.isMuxed) ? media.main.isFmp4 : media.audio.isFmp4; + } + + // no codecs, no playback. + if (!codecs.audio && !codecs.video) { + this.blacklistCurrentPlaylist({ + playlist: this.media(), + message: 'Could not determine codecs for playlist.', + blacklistDuration: Infinity + }); + return; } // fmp4 relies on browser support, while ts relies on muxer support - const supportFunction = mainStartingMedia.isFmp4 ? browserSupportsCodec : muxerSupportsCodec; - const unsupportedCodecs = []; + const supportFunction = (isFmp4, codec) => (isFmp4 ? browserSupportsCodec(codec) : muxerSupportsCodec(codec)); + const unsupportedCodecs = {}; + let unsupportedAudio; + + ['video', 'audio'].forEach(function(type) { + if (codecs.hasOwnProperty(type) && !supportFunction(media[type].isFmp4, codecs[type])) { + const supporter = media[type].isFmp4 ? 'browser' : 'muxer'; - ['audio', 'video'].forEach(function(type) { - if (codecs.hasOwnProperty(type) && !supportFunction(codecs[type])) { - unsupportedCodecs.push(codecs[type]); + unsupportedCodecs[supporter] = unsupportedCodecs[supporter] || []; + unsupportedCodecs[supporter].push(codecs[type]); + + if (type === 'audio') { + unsupportedAudio = supporter; + } } }); + if (usingAudioLoader && unsupportedAudio && this.media().attributes.AUDIO) { + const audioGroup = this.media().attributes.AUDIO; + + this.mediaTypes_.AUDIO.activePlaylistLoader.pause(); + this.audioSegmentLoader_.pause(); + this.audioSegmentLoader_.abort(); + this.master().playlists.forEach(variant => { + const variantAudioGroup = variant.attributes && variant.attributes.AUDIO; + + if (variantAudioGroup === audioGroup && variant !== this.media()) { + variant.excludeUntil = Infinity; + } + }); + this.logger_(`excluding audio group ${audioGroup} as ${unsupportedAudio} does not support codec(s): "${codecs.audio}"`); + } + // if we have any unsupported codecs blacklist this playlist. - if (unsupportedCodecs.length) { - const supporter = mainStartingMedia.isFmp4 ? 'browser' : 'muxer'; + if (Object.keys(unsupportedCodecs).length) { + const message = Object.keys(unsupportedCodecs).reduce((acc, supporter) => { + + if (acc) { + acc += ', '; + } - // reset startingMedia_ when the intial playlist is blacklisted. - this.mainSegmentLoader_.startingMedia_ = void 0; + acc += `${supporter} does not support codec(s): "${unsupportedCodecs[supporter].join(',')}"`; + + return acc; + }, '') + '.'; this.blacklistCurrentPlaylist({ - playlist: media, - message: `${supporter} does not support codec(s): "${unsupportedCodecs.join(',')}".`, - internal: true - }, Infinity); + playlist: this.media(), + internal: true, + message, + blacklistDuration: Infinity + }); return; } + // check if codec switching is happening + if (this.sourceUpdater_.ready() && !this.sourceUpdater_.canChangeType()) { + const switchMessages = []; - if (!codecs.video && !codecs.audio) { - const error = 'Failed to create SourceBuffers. No compatible SourceBuffer ' + - 'configuration for the variant stream:' + media.resolvedUri; + ['video', 'audio'].forEach((type) => { + const newCodec = (parseCodecs(this.sourceUpdater_.codecs[type] || '')[type] || {}).type; + const oldCodec = (parseCodecs(codecs[type] || '')[type] || {}).type; - videojs.log.warn(error); - this.error = error; + if (newCodec && oldCodec && newCodec.toLowerCase() !== oldCodec.toLowerCase()) { + switchMessages.push(`"${this.sourceUpdater_.codecs[type]}" -> "${codecs[type]}"`); + } + }); - if (this.mediaSource.readyState !== 'open') { - this.trigger('error'); - } else { - this.sourceUpdater_.endOfStream('decode'); + if (switchMessages.length) { + this.blacklistCurrentPlaylist({ + playlist: this.media(), + message: `Codec switching not supported: ${switchMessages.join(', ')}.`, + blacklistDuration: Infinity, + internal: true + }); + return; } } - try { - this.sourceUpdater_.createSourceBuffers(codecs); - } catch (e) { - const error = 'Failed to create SourceBuffers: ' + e; + // TODO: when using the muxer shouldn't we just return + // the codecs that the muxer outputs? + return codecs; + } + + /** + * Create source buffers and exlude any incompatible renditions. + * + * @private + */ + tryToCreateSourceBuffers_() { + // media source is not ready yet or sourceBuffers are already + // created. + if (this.mediaSource.readyState !== 'open' || this.sourceUpdater_.ready()) { + return; + } + + if (!this.areMediaTypesKnown_()) { + return; + } + + const codecs = this.getCodecsOrExclude_(); - videojs.log.warn(error); - this.error = error; - if (this.mediaSource.readyState !== 'open') { - this.trigger('error'); - } else { - this.sourceUpdater_.endOfStream('decode'); - } + // no codecs means that the playlist was excluded + if (!codecs) { return; } + this.sourceUpdater_.createSourceBuffers(codecs); + const codecString = [codecs.video, codecs.audio].filter(Boolean).join(','); - // TODO: - // blacklisting incompatible renditions will have to change - // once we add support for `changeType` on source buffers. - // We will have to not blacklist any rendition until we try to - // switch to it and learn that it is incompatible and if it is compatible - // we `changeType` on the sourceBuffer. this.excludeIncompatibleVariants_(codecString); } @@ -1448,24 +1488,30 @@ export class MasterPlaylistController extends videojs.EventTarget { variantCodecCount = Object.keys(variantCodecs).length; } + // TODO: we can support this by removing the + // old media source and creating a new one, but it will take some work. // The number of streams cannot change if (variantCodecCount !== codecCount) { blacklistReasons.push(`codec count "${variantCodecCount}" !== "${codecCount}"`); variant.excludeUntil = Infinity; } - // the video codec cannot change - if (variantCodecs.video && codecs.video && - variantCodecs.video.type.toLowerCase() !== codecs.video.type.toLowerCase()) { - blacklistReasons.push(`video codec "${variantCodecs.video.type}" !== "${codecs.video.type}"`); - variant.excludeUntil = Infinity; - } + // only exclude playlists by codec change, if codecs cannot switch + // during playback. + if (!this.sourceUpdater_.canChangeType()) { + // the video codec cannot change + if (variantCodecs.video && codecs.video && + variantCodecs.video.type.toLowerCase() !== codecs.video.type.toLowerCase()) { + blacklistReasons.push(`video codec "${variantCodecs.video.type}" !== "${codecs.video.type}"`); + variant.excludeUntil = Infinity; + } - // the audio codec cannot change - if (variantCodecs.audio && codecs.audio && - variantCodecs.audio.type.toLowerCase() !== codecs.audio.type.toLowerCase()) { - variant.excludeUntil = Infinity; - blacklistReasons.push(`audio codec "${variantCodecs.audio.type}" !== "${codecs.audio.type}"`); + // the audio codec cannot change + if (variantCodecs.audio && codecs.audio && + variantCodecs.audio.type.toLowerCase() !== codecs.audio.type.toLowerCase()) { + variant.excludeUntil = Infinity; + blacklistReasons.push(`audio codec "${variantCodecs.audio.type}" !== "${codecs.audio.type}"`); + } } if (blacklistReasons.length) { diff --git a/src/media-groups.js b/src/media-groups.js index c39861828..ea5f71724 100644 --- a/src/media-groups.js +++ b/src/media-groups.js @@ -736,10 +736,13 @@ export const setupMediaGroups = (settings) => { // DO NOT enable the default subtitle or caption track. // DO enable the default audio track const audioGroup = mediaTypes.AUDIO.activeGroup(); - const groupId = (audioGroup.filter(group => group.default)[0] || audioGroup[0]).id; - mediaTypes.AUDIO.tracks[groupId].enabled = true; - mediaTypes.AUDIO.onTrackChanged(); + if (audioGroup) { + const groupId = (audioGroup.filter(group => group.default)[0] || audioGroup[0]).id; + + mediaTypes.AUDIO.tracks[groupId].enabled = true; + mediaTypes.AUDIO.onTrackChanged(); + } masterPlaylistLoader.on('mediachange', () => { ['AUDIO', 'SUBTITLES'].forEach(type => mediaTypes[type].onGroupChanged()); diff --git a/src/media-segment-request.js b/src/media-segment-request.js index 8dc121a55..08acdae83 100644 --- a/src/media-segment-request.js +++ b/src/media-segment-request.js @@ -290,7 +290,8 @@ const transmuxAndNotify = ({ if (probeResult) { trackInfoFn(segment, { hasAudio: probeResult.hasAudio, - hasVideo: probeResult.hasVideo + hasVideo: probeResult.hasVideo, + isMuxed }); trackInfoFn = null; @@ -318,6 +319,9 @@ const transmuxAndNotify = ({ }, onTrackInfo: (trackInfo) => { if (trackInfoFn) { + if (isMuxed) { + trackInfo.isMuxed = true; + } trackInfoFn(segment, trackInfo); } }, diff --git a/src/segment-loader.js b/src/segment-loader.js index ddf6765b3..1d19465c6 100644 --- a/src/segment-loader.js +++ b/src/segment-loader.js @@ -21,6 +21,7 @@ import { removeCuesFromTrack } from './util/text-tracks'; import { gopsSafeToAlignWith, removeGopBuffer, updateGopBuffer } from './util/gops'; +import shallowEqual from './util/shallow-equal.js'; // in ms const CHECK_BUFFER_DELAY = 500; @@ -830,7 +831,6 @@ export default class SegmentLoader extends videojs.EventTarget { } if (!oldPlaylist || oldPlaylist.uri !== newPlaylist.uri) { - this.trigger('playlistupdate'); if (this.mediaIndex !== null || this.handlePartialData_) { // we must "resync" the segment loader when we switch renditions and // the segment loader is already synced to the previous rendition @@ -839,6 +839,8 @@ export default class SegmentLoader extends videojs.EventTarget { // out before we start adding more data this.resyncLoader(); } + this.startingMedia_ = void 0; + this.trigger('playlistupdate'); // the rest of this function depends on `oldPlaylist` being defined return; @@ -1441,18 +1443,21 @@ export default class SegmentLoader extends videojs.EventTarget { return; } + if (this.checkForIllegalMediaSwitch(trackInfo)) { + return; + } + + trackInfo = trackInfo || {}; + // When we have track info, determine what media types this loader is dealing with. // Guard against cases where we're not getting track info at all until we are // certain that all streams will provide it. - if (typeof this.startingMedia_ === 'undefined' && (trackInfo.hasAudio || trackInfo.hasVideo)) { + if ((trackInfo.hasVideo || trackInfo.hasAudio) && !shallowEqual(this.startingMedia_, trackInfo)) { this.startingMedia_ = trackInfo; + this.logger_('trackinfo update', trackInfo); + this.trigger('trackinfo'); } - this.trigger('trackinfo'); - - if (this.checkForIllegalMediaSwitch(trackInfo)) { - return; - } } handleTimingInfo_(simpleSegment, mediaType, timeType, time) { diff --git a/src/source-updater.js b/src/source-updater.js index 6faf2bcc7..c556adba7 100644 --- a/src/source-updater.js +++ b/src/source-updater.js @@ -6,6 +6,13 @@ import logger from './util/logger'; import noop from './util/noop'; import { bufferIntersection } from './ranges.js'; import {getMimeForCodec} from '@videojs/vhs-utils/dist/codecs.js'; +import window from 'global/window'; +import toTitleCase from './util/to-title-case.js'; + +const bufferTypes = [ + 'video', + 'audio' +]; const updating = (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; @@ -40,7 +47,7 @@ const shiftQueue = (type, sourceUpdater) => { let queueEntry = sourceUpdater.queue[queueIndex]; if (queueEntry.type === 'mediaSource') { - if (!sourceUpdater.updating()) { + if (!sourceUpdater.updating() && sourceUpdater.mediaSource.readyState !== 'closed') { sourceUpdater.queue.shift(); queueEntry.action(sourceUpdater); @@ -72,7 +79,7 @@ const shiftQueue = (type, sourceUpdater) => { // Media source queue entries don't need to consider whether the source updater is // started (i.e., source buffers are created) as they don't need the source buffers, but // source buffer queue entries do. - if (!sourceUpdater.started_ || updating(type, sourceUpdater)) { + if (!sourceUpdater.started_ || sourceUpdater.mediaSource.readyState === 'closed' || updating(type, sourceUpdater)) { return; } @@ -102,10 +109,34 @@ const shiftQueue = (type, sourceUpdater) => { sourceUpdater.queuePending[type] = queueEntry; }; +const cleanupBuffer = (type, sourceUpdater) => { + const buffer = sourceUpdater[`${type}Buffer`]; + const titleType = toTitleCase(type); + + if (!buffer) { + return; + } + + buffer.removeEventListener('updateend', sourceUpdater[`on${titleType}UpdateEnd_`]); + buffer.removeEventListener('error', sourceUpdater[`on${titleType}Error_`]); + + sourceUpdater.codecs[type] = null; + sourceUpdater[`${type}Buffer`] = null; +}; + +const inSourceBuffers = (mediaSource, sourceBuffer) => mediaSource && sourceBuffer && + Array.prototype.indexOf.call(mediaSource.sourceBuffers, sourceBuffer) !== -1; + const actions = { appendBuffer: (bytes, segmentInfo) => (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; + // can't do anything if the media source / source buffer is null + // or the media source does not contain this source buffer. + if (!inSourceBuffers(sourceUpdater.mediaSource, sourceBuffer)) { + return; + } + sourceUpdater.logger_(`Appending segment ${segmentInfo.mediaIndex}'s ${bytes.length} bytes to ${type}Buffer`); sourceBuffer.appendBuffer(bytes); @@ -113,7 +144,11 @@ const actions = { remove: (start, end) => (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; - sourceBuffer.removing = true; + // can't do anything if the media source / source buffer is null + // or the media source does not contain this source buffer. + if (!inSourceBuffers(sourceUpdater.mediaSource, sourceBuffer)) { + return; + } sourceUpdater.logger_(`Removing ${start} to ${end} from ${type}Buffer`); sourceBuffer.remove(start, end); @@ -121,6 +156,12 @@ const actions = { timestampOffset: (offset) => (type, sourceUpdater) => { const sourceBuffer = sourceUpdater[`${type}Buffer`]; + // can't do anything if the media source / source buffer is null + // or the media source does not contain this source buffer. + if (!inSourceBuffers(sourceUpdater.mediaSource, sourceBuffer)) { + return; + } + sourceUpdater.logger_(`Setting ${type}timestampOffset to ${offset}`); sourceBuffer.timestampOffset = offset; @@ -147,6 +188,77 @@ const actions = { } catch (e) { videojs.log.warn('Failed to set media source duration', e); } + }, + abort: () => (type, sourceUpdater) => { + if (sourceUpdater.mediaSource.readyState !== 'open') { + return; + } + const sourceBuffer = sourceUpdater[`${type}Buffer`]; + + // can't do anything if the media source / source buffer is null + // or the media source does not contain this source buffer. + if (!inSourceBuffers(sourceUpdater.mediaSource, sourceBuffer)) { + return; + } + + sourceUpdater.logger_(`calling abort on ${type}Buffer`); + try { + sourceBuffer.abort(); + } catch (e) { + videojs.log.warn(`Failed to abort on ${type}Buffer`, e); + } + }, + addSourceBuffer: (type, codec) => (sourceUpdater) => { + const titleType = toTitleCase(type); + const mime = getMimeForCodec(codec); + + sourceUpdater.logger_(`Adding ${type}Buffer with codec ${codec} to mediaSource`); + + const sourceBuffer = sourceUpdater.mediaSource.addSourceBuffer(mime); + + sourceBuffer.addEventListener('updateend', sourceUpdater[`on${titleType}UpdateEnd_`]); + sourceBuffer.addEventListener('error', sourceUpdater[`on${titleType}Error_`]); + sourceUpdater.codecs[type] = codec; + sourceUpdater[`${type}Buffer`] = sourceBuffer; + }, + removeSourceBuffer: (type) => (sourceUpdater) => { + const sourceBuffer = sourceUpdater[`${type}Buffer`]; + + cleanupBuffer(type, sourceUpdater); + + // can't do anything if the media source / source buffer is null + // or the media source does not contain this source buffer. + if (!inSourceBuffers(sourceUpdater.mediaSource, sourceBuffer)) { + return; + } + + sourceUpdater.logger_(`Removing ${type}Buffer with codec ${sourceUpdater.codecs[type]} from mediaSource`); + + try { + sourceUpdater.mediaSource.removeSourceBuffer(sourceBuffer); + } catch (e) { + videojs.log.warn(`Failed to removeSourceBuffer ${type}Buffer`, e); + } + }, + changeType: (codec) => (type, sourceUpdater) => { + const sourceBuffer = sourceUpdater[`${type}Buffer`]; + const mime = getMimeForCodec(codec); + + // can't do anything if the media source / source buffer is null + // or the media source does not contain this source buffer. + if (!inSourceBuffers(sourceUpdater.mediaSource, sourceBuffer)) { + return; + } + + // do not update codec if we don't need to. + if (sourceUpdater.codecs[type] === codec) { + return; + } + + sourceUpdater.logger_(`changing ${type}Buffer codec from ${sourceUpdater.codecs[type]} to ${codec}`); + + sourceBuffer.changeType(mime); + sourceUpdater.codecs[type] = codec; } }; @@ -168,7 +280,6 @@ const onUpdateend = (type, sourceUpdater) => (e) => { // if we encounter an updateend without a corresponding pending action from our queue // for that source buffer type, process the next action. if (sourceUpdater.queuePending[type]) { - sourceUpdater[`${type}Buffer`].removing = false; const doneFn = sourceUpdater.queuePending[type].doneFn; sourceUpdater.queuePending[type] = null; @@ -196,6 +307,8 @@ export default class SourceUpdater extends videojs.EventTarget { constructor(mediaSource) { super(); this.mediaSource = mediaSource; + this.sourceopenListener_ = () => shiftQueue('mediaSource', this); + this.mediaSource.addEventListener('sourceopen', this.sourceopenListener_); this.logger_ = logger('SourceUpdater'); // initial timestamp offset is 0 this.audioTimestampOffset_ = 0; @@ -207,10 +320,22 @@ export default class SourceUpdater extends videojs.EventTarget { }; this.delayedAudioAppendQueue_ = []; this.videoAppendQueued_ = false; + this.codecs = {}; + this.onVideoUpdateEnd_ = onUpdateend('video', this); + this.onAudioUpdateEnd_ = onUpdateend('audio', this); + this.onVideoError_ = (e) => { + // used for debugging + this.videoError_ = e; + }; + this.onAudioError_ = (e) => { + // used for debugging + this.audioError_ = e; + }; + this.started_ = false; } ready() { - return !!(this.audioBuffer || this.videoBuffer); + return this.started_; } createSourceBuffers(codecs) { @@ -219,55 +344,139 @@ export default class SourceUpdater extends videojs.EventTarget { return; } - if (this.mediaSource.readyState === 'closed') { - this.sourceopenListener_ = this.createSourceBuffers.bind(this, codecs); - this.mediaSource.addEventListener('sourceopen', this.sourceopenListener_); + // the intial addOrChangeSourceBuffers will always be + // two add buffers. + this.addOrChangeSourceBuffers(codecs); + this.started_ = true; + this.trigger('ready'); + } + + /** + * Add a type of source buffer to the media source. + * + * @param {string} type + * The type of source buffer to add. + * + * @param {string} codec + * The codec to add the source buffer with. + */ + addSourceBuffer(type, codec) { + pushQueue({ + type: 'mediaSource', + sourceUpdater: this, + action: actions.addSourceBuffer(type, codec), + name: 'addSourceBuffer' + }); + } + + /** + * call abort on a source buffer. + * + * @param {string} type + * The type of source buffer to call abort on. + */ + abort(type) { + pushQueue({ + type, + sourceUpdater: this, + action: actions.abort(type), + name: 'abort' + }); + } + + /** + * Call removeSourceBuffer and remove a specific type + * of source buffer on the mediaSource. + * + * @param {string} type + * The type of source buffer to remove. + */ + removeSourceBuffer(type) { + if (!this.canRemoveSourceBuffer()) { + videojs.log.error('removeSourceBuffer is not supported!'); return; } - if (codecs.audio) { - const mime = getMimeForCodec(codecs.audio); + pushQueue({ + type: 'mediaSource', + sourceUpdater: this, + action: actions.removeSourceBuffer(type), + name: 'removeSourceBuffer' + }); + } - this.audioBuffer = this.mediaSource.addSourceBuffer(mime); - this.audioBuffer.removing = false; - this.logger_(`created SourceBuffer ${mime}`); - } + /** + * Whether or not the removeSourceBuffer function is supported + * on the mediaSource. + * + * @return {boolean} + * if removeSourceBuffer can be called. + */ + canRemoveSourceBuffer() { + return window.MediaSource && + window.MediaSource.prototype && + typeof window.MediaSource.prototype.removeSourceBuffer === 'function'; + } - if (codecs.video) { - const mime = getMimeForCodec(codecs.video); + /** + * Whether or not the changeType function is supported + * on our SourceBuffers. + * + * @return {boolean} + * if changeType can be called. + */ + canChangeType() { + return window.SourceBuffer && + window.SourceBuffer.prototype && + typeof window.SourceBuffer.prototype.changeType === 'function'; + } - this.videoBuffer = this.mediaSource.addSourceBuffer(mime); - this.videoBuffer.removing = false; - this.logger_(`created SourceBuffer ${mime}`); + /** + * Call the changeType function on a source buffer, given the code and type. + * + * @param {string} type + * The type of source buffer to call changeType on. + * + * @param {string} codec + * The codec string to change type with on the source buffer. + */ + changeType(type, codec) { + if (!this.canChangeType()) { + videojs.log.error('changeType is not supported!'); + return; } - this.trigger('ready'); - this.start_(); + pushQueue({ + type, + sourceUpdater: this, + action: actions.changeType(codec), + name: 'changeType' + }); } - start_() { - this.started_ = true; - - if (this.audioBuffer) { - this.onAudioUpdateEnd_ = onUpdateend('audio', this); - this.audioBuffer.addEventListener('updateend', this.onAudioUpdateEnd_); - this.onAudioError_ = (e) => { - // used for debugging - this.audioError_ = e; - }; - this.audioBuffer.addEventListener('error', this.onAudioError_); - shiftQueue('audio', this); - } - if (this.videoBuffer) { - this.onVideoUpdateEnd_ = onUpdateend('video', this); - this.videoBuffer.addEventListener('updateend', this.onVideoUpdateEnd_); - this.onVideoError_ = (e) => { - // used for debugging - this.videoError_ = e; - }; - this.videoBuffer.addEventListener('error', this.onVideoError_); - shiftQueue('video', this); + /** + * Add source buffers with a codec or, if they are already created, + * call changeType on source buffers using changeType. + * + * @param {Object} codecs + * Codecs to switch to + */ + addOrChangeSourceBuffers(codecs) { + if (!codecs || typeof codecs !== 'object' || Object.keys(codecs).length === 0) { + throw new Error('Cannot addOrChangeSourceBuffers to undefined codecs'); } + + Object.keys(codecs).forEach((type) => { + const codec = codecs[type]; + + if (!this.ready()) { + return this.addSourceBuffer(type, codec); + } + + if (this.canChangeType()) { + this.changeType(type, codec); + } + }); } /** @@ -311,28 +520,69 @@ export default class SourceUpdater extends videojs.EventTarget { } } + /** + * Get the audio buffer's buffered timerange. + * + * @return {TimeRange} + * The audio buffer's buffered time range + */ audioBuffered() { - return this.audioBuffer && this.audioBuffer.buffered ? this.audioBuffer.buffered : + // no media source/source buffer or it isn't in the media sources + // source buffer list + if (!inSourceBuffers(this.mediaSource, this.audioBuffer)) { + return videojs.createTimeRange(); + } + + return this.audioBuffer.buffered ? this.audioBuffer.buffered : videojs.createTimeRange(); } + /** + * Get the video buffer's buffered timerange. + * + * @return {TimeRange} + * The video buffer's buffered time range + */ videoBuffered() { - return this.videoBuffer && this.videoBuffer.buffered ? this.videoBuffer.buffered : + // no media source/source buffer or it isn't in the media sources + // source buffer list + if (!inSourceBuffers(this.mediaSource, this.videoBuffer)) { + return videojs.createTimeRange(); + } + return this.videoBuffer.buffered ? this.videoBuffer.buffered : videojs.createTimeRange(); } + /** + * Get a combined video/audio buffer's buffered timerange. + * + * @return {TimeRange} + * the combined time range + */ buffered() { - if (this.audioBuffer && !this.videoBuffer) { + const video = inSourceBuffers(this.mediaSource, this.videoBuffer) ? this.videoBuffer : null; + const audio = inSourceBuffers(this.mediaSource, this.audioBuffer) ? this.audioBuffer : null; + + if (audio && !video) { return this.audioBuffered(); } - if (this.videoBuffer && !this.audioBuffer) { + if (video && !audio) { return this.videoBuffered(); } return bufferIntersection(this.audioBuffered(), this.videoBuffered()); } + /** + * Add a callback to the queue that will set duration on the mediaSource. + * + * @param {number} duration + * The duration to set + * + * @param {Function} [doneFn] + * function to run after duration has been set. + */ setDuration(duration, doneFn = noop) { // In order to set the duration on the media source, it's necessary to wait for all // source buffers to no longer be updating. "If the updating attribute equals true on @@ -347,6 +597,16 @@ export default class SourceUpdater extends videojs.EventTarget { }); } + /** + * Add a mediaSource endOfStream call to the queue + * + * @param {Error} [error] + * Call endOfStream with an error + * + * @param {Function} [doneFn] + * A function that should be called when the + * endOfStream call has finished. + */ endOfStream(error = null, doneFn = noop) { if (typeof error !== 'string') { error = undefined; @@ -468,26 +728,42 @@ export default class SourceUpdater extends videojs.EventTarget { return this.videoTimestampOffset_; } + /** + * Add a function to the queue that will be called + * when it is its turn to run in the audio queue. + * + * @param {Function} callback + * The callback to queue. + */ audioQueueCallback(callback) { - if (this.audioBuffer) { - pushQueue({ - type: 'audio', - sourceUpdater: this, - action: actions.callback(callback), - name: 'callback' - }); + if (!this.audioBuffer) { + return; } + pushQueue({ + type: 'audio', + sourceUpdater: this, + action: actions.callback(callback), + name: 'callback' + }); } + /** + * Add a function to the queue that will be called + * when it is its turn to run in the video queue. + * + * @param {Function} callback + * The callback to queue. + */ videoQueueCallback(callback) { - if (this.videoBuffer) { - pushQueue({ - type: 'video', - sourceUpdater: this, - action: actions.callback(callback), - name: 'callback' - }); + if (!this.videoBuffer) { + return; } + pushQueue({ + type: 'video', + sourceUpdater: this, + action: actions.callback(callback), + name: 'callback' + }); } /** @@ -495,59 +771,21 @@ export default class SourceUpdater extends videojs.EventTarget { */ dispose() { this.trigger('dispose'); - const audioDisposeFn = () => { - if (this.mediaSource.readyState === 'open') { - // ie 11 likes to throw on abort with InvalidAccessError or InvalidStateError - // dom exceptions - try { - this.audioBuffer.abort(); - } catch (e) { - videojs.log.warn('Failed to call abort on audio buffer', e); - } - } - this.audioBuffer.removeEventListener('updateend', this.onAudioUpdateEnd_); - this.audioBuffer.removeEventListener('updateend', audioDisposeFn); - this.audioBuffer.removeEventListener('error', this.onAudioError_); - this.audioBuffer = null; - }; - const videoDisposeFn = () => { - if (this.mediaSource.readyState === 'open') { - // ie 11 likes to throw on abort with InvalidAccessError or InvalidStateError - // dom exceptions - try { - this.videoBuffer.abort(); - } catch (e) { - videojs.log.warn('Failed to call abort on video buffer', e); - } - } - this.videoBuffer.removeEventListener('updateend', this.onVideoUpdateEnd_); - this.videoBuffer.removeEventListener('error', this.onVideoError_); - this.videoBuffer.removeEventListener('updateend', videoDisposeFn); - this.videoBuffer = null; - }; - - // TODO: can we just use "updating" rather than removing? - // this was implemented in https://github.com/videojs/http-streaming/pull/442 - if (this.audioBuffer) { - if (this.audioBuffer.removing) { - this.audioBuffer.addEventListener('updateend', audioDisposeFn); - } else { - audioDisposeFn(); - } - } - - if (this.videoBuffer) { - if (this.videoBuffer.removing) { - this.videoBuffer.addEventListener('updateend', videoDisposeFn); + bufferTypes.forEach((type) => { + this.abort(type); + if (this.canRemoveSourceBuffer()) { + this.removeSourceBuffer(type); } else { - videoDisposeFn(); + this[`${type}QueueCallback`](() => cleanupBuffer(type, this)); } - } + }); this.videoAppendQueued_ = false; this.delayedAudioAppendQueue_.length = 0; - this.mediaSource.removeEventListener('sourceopen', this.sourceopenListener_); + if (this.sourceopenListener_) { + this.mediaSource.removeEventListener('sourceopen', this.sourceopenListener_); + } this.off(); } diff --git a/src/util/shallow-equal.js b/src/util/shallow-equal.js new file mode 100644 index 000000000..e4b13bb32 --- /dev/null +++ b/src/util/shallow-equal.js @@ -0,0 +1,41 @@ +const shallowEqual = function(a, b) { + // if both are undefined + // or one or the other is undefined + // they are not equal + if ((!a && !b) || (!a && b) || (a && !b)) { + return false; + } + + // they are the same object and thus, equal + if (a === b) { + return true; + } + + // sort keys so we can make sure they have + // all the same keys later. + const akeys = Object.keys(a).sort(); + const bkeys = Object.keys(b).sort(); + + // different number of keys, not equal + if (akeys.length !== bkeys.length) { + return false; + } + + for (let i = 0; i < akeys.length; i++) { + const key = akeys[i]; + + // different sorted keys, not equal + if (key !== bkeys[i]) { + return false; + } + + // different values, not equal + if (a[key] !== b[key]) { + return false; + } + } + + return true; +}; + +export default shallowEqual; diff --git a/src/util/to-title-case.js b/src/util/to-title-case.js new file mode 100644 index 000000000..48856acf9 --- /dev/null +++ b/src/util/to-title-case.js @@ -0,0 +1,9 @@ +const toTitleCase = function(string) { + if (typeof string !== 'string') { + return string; + } + + return string.replace(/./, (w) => w.toUpperCase()); +}; + +export default toTitleCase; diff --git a/test/manifests/dash-many-codecs.mpd b/test/manifests/dash-many-codecs.mpd index 05ab8b721..9bc4ed208 100644 --- a/test/manifests/dash-many-codecs.mpd +++ b/test/manifests/dash-many-codecs.mpd @@ -3,7 +3,7 @@ - + @@ -13,7 +13,7 @@ - + diff --git a/test/master-playlist-controller.test.js b/test/master-playlist-controller.test.js index 97f2b685f..e686f487a 100644 --- a/test/master-playlist-controller.test.js +++ b/test/master-playlist-controller.test.js @@ -47,11 +47,12 @@ import { bandwidthWithinTolerance } from './custom-assertions.js'; -QUnit.module('MasterPlaylistController', { +const sharedHooks = { beforeEach(assert) { this.env = useFakeEnvironment(assert); this.clock = this.env.clock; this.requests = this.env.requests; + this.oldTypeSupported = window.MediaSource.isTypeSupported; this.mse = useFakeMediaSource(); if (!videojs.browser.IE_VERSION) { @@ -59,7 +60,7 @@ QUnit.module('MasterPlaylistController', { window.devicePixelRatio = 1; } - this.oldTypeSupported = window.MediaSource.isTypeSupported; + this.oldChangeType = window.SourceBuffer.prototype.changeType; // force the HLS tech to run this.origSupportsNativeHls = videojs.Vhs.supportsNativeHls; @@ -99,8 +100,12 @@ QUnit.module('MasterPlaylistController', { videojs.browser = this.oldBrowser; this.player.dispose(); window.MediaSource.isTypeSupported = this.oldTypeSupported; + window.SourceBuffer.prototype.changeType = this.oldChangeType; } -}); + +}; + +QUnit.module('MasterPlaylistController', sharedHooks); QUnit.test('throws error when given an empty URL', function(assert) { const options = { @@ -1026,20 +1031,19 @@ QUnit.test('waits for both main and audio loaders to finish before calling endOf // audio media this.standardXHRResponse(this.requests.shift(), audioMedia); - return requestAndAppendSegment({ + return Promise.all([requestAndAppendSegment({ request: this.requests.shift(), segment: videoSegment(), isOnlyVideo: true, segmentLoader: MPC.mainSegmentLoader_, clock: this.clock - }).then(() => requestAndAppendSegment({ + }), requestAndAppendSegment({ request: this.requests.shift(), segment: audioSegment(), isOnlyAudio: true, segmentLoader: MPC.audioSegmentLoader_, clock: this.clock - })).then(() => { - + })]).then(() => { assert.equal(videoEnded, 1, 'main segment loader did not trigger ended again'); assert.equal(audioEnded, 1, 'audio segment loader triggered ended'); assert.equal(MPC.mediaSource.readyState, 'ended', 'Media Source ended'); @@ -1379,6 +1383,8 @@ QUnit.test('blacklists switching between playlists with different codecs', funct const mpc = this.masterPlaylistController; + mpc.sourceUpdater_.canChangeType = () => false; + let debugLogs = []; mpc.logger_ = (...logs) => { @@ -3670,25 +3676,11 @@ QUnit.test('Uses audio codec from audio playlist for demuxed content', function( assert.deepEqual( createSourceBufferCalls[0], { - video: 'avc1.foo.bar', - audio: 'mp4a.foo.bar' + video: 'avc1.4d400d', + audio: 'mp4a.40.2' }, 'passed codecs from playlist' ); - - const playlists = mpc.master().playlists; - - assert.deepEqual(playlists[0], mpc.media(), '1st selected not blacklisted'); - assert.equal(playlists[1].excludeUntil, Infinity, 'blacklisted 2nd: codecs incompatible with selected.'); - assert.notOk(playlists[2].excludeUntil, '3rd not blacklisted: codecs compatable with selected'); - assert.equal(playlists[3].excludeUntil, Infinity, 'blacklisted 4th: codecs incompatible with selected.'); - - const secondBlacklistIndex = messages.indexOf('VHS: MPC > blacklisting 1-placeholder-uri-1: video codec "hvc1" !== "avc1"'); - const forthBlacklistIndex = messages.indexOf('VHS: MPC > blacklisting 3-placeholder-uri-3: video codec "av01" !== "avc1"'); - - assert.notEqual(secondBlacklistIndex, -1, '2nd codec blacklist is logged'); - assert.notEqual(forthBlacklistIndex, -1, '4th codec blacklist is logged'); - videojs.log.debug = oldDebug; done(); }; @@ -4544,3 +4536,655 @@ QUnit.test('can pass or select a playlist for smoothQualityChange_', function(as resyncLoader: 2 }, 'calls expected function when not passed a playlist'); }); + +QUnit.module('MasterPlaylistController codecs', { + beforeEach(assert) { + sharedHooks.beforeEach.call(this, assert); + this.mpc = this.masterPlaylistController; + + this.blacklists = []; + this.mpc.blacklistCurrentPlaylist = (blacklist) => this.blacklists.push(blacklist); + + this.contentSetup = (options) => { + const { + audioStartingMedia, + mainStartingMedia, + audioPlaylist, + mainPlaylist + } = options; + + if (mainStartingMedia) { + this.mpc.mainSegmentLoader_.startingMedia_ = mainStartingMedia; + } + + if (audioStartingMedia) { + this.mpc.audioSegmentLoader_.startingMedia_ = audioStartingMedia; + } + + this.master = {mediaGroups: {AUDIO: {}}, playlists: []}; + + this.mpc.master = () => this.master; + + if (mainPlaylist) { + this.mpc.media = () => mainPlaylist; + this.master.playlists.push(mainPlaylist); + } + + if (audioPlaylist) { + const mainAudioGroup = mainPlaylist && mainPlaylist.attributes.AUDIO; + + if (mainAudioGroup) { + this.master.mediaGroups.AUDIO[mainAudioGroup] = { + english: { + default: true, + playlists: [audioPlaylist] + } + }; + } + this.master.playlists.push(audioPlaylist); + this.mpc.mediaTypes_.AUDIO.activePlaylistLoader = {pause() {}}; + } + }; + }, + afterEach(assert) { + sharedHooks.afterEach.call(this, assert); + } +}); + +QUnit.test('can get demuxed codecs from the video/main', function(assert) { + this.contentSetup({ + audioStartingMedia: {hasAudio: true, hasVideo: false}, + mainStartingMedia: {hasVideo: true, hasAudio: false}, + audioPlaylist: {attributes: {}}, + mainPlaylist: {attributes: {CODECS: 'avc1.4c400d,mp4a.40.5', AUDIO: 'low-quality'}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {audio: 'mp4a.40.5', video: 'avc1.4c400d'}, 'codecs returned'); +}); + +QUnit.test('can get demuxed codecs from the video/main playlist and audio playlist', function(assert) { + this.contentSetup({ + audioStartingMedia: {hasAudio: true, hasVideo: false}, + mainStartingMedia: {hasVideo: true, hasAudio: false}, + audioPlaylist: {attributes: {CODECS: 'mp4a.40.5'}}, + mainPlaylist: {attributes: {CODECS: 'avc1.4c400d', AUDIO: 'low-quality'}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {audio: 'mp4a.40.5', video: 'avc1.4c400d'}, 'codecs returned'); +}); + +QUnit.test('can get demuxed codecs from the main and audio loaders', function(assert) { + this.contentSetup({ + audioStartingMedia: {hasAudio: true, hasVideo: false, audioCodec: 'mp4a.40.5'}, + mainStartingMedia: {hasVideo: true, hasAudio: false, videoCodec: 'avc1.4c400d'}, + audioPlaylist: {attributes: {}}, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {audio: 'mp4a.40.5', video: 'avc1.4c400d'}, 'codecs returned'); +}); + +QUnit.test('can get demuxed codecs from the main loader', function(assert) { + this.contentSetup({ + audioStartingMedia: {}, + mainStartingMedia: {hasVideo: true, hasAudio: true, videoCodec: 'avc1.4c400d', audioCodec: 'mp4a.40.5'}, + audioPlaylist: {attributes: {}}, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {audio: 'mp4a.40.5', video: 'avc1.4c400d'}, 'codecs returned'); +}); + +QUnit.test('can get muxed codecs from video/main playlist', function(assert) { + this.contentSetup({ + mainStartingMedia: {hasVideo: true, hasAudio: true, isMuxed: true}, + mainPlaylist: {attributes: {CODECS: 'avc1.4c400d,mp4a.40.5'}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4c400d,mp4a.40.5'}, 'codecs returned'); +}); + +QUnit.test('can get muxed codecs from video/main loader', function(assert) { + this.contentSetup({ + mainStartingMedia: { + hasVideo: true, + hasAudio: true, + isMuxed: true, + videoCodec: 'avc1.4c400d', + audioCodec: 'mp4a.40.5' + }, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4c400d,mp4a.40.5'}, 'codecs returned'); +}); + +QUnit.test('can get audio only codecs from main playlist ', function(assert) { + this.contentSetup({ + mainStartingMedia: {hasVideo: false, hasAudio: true}, + mainPlaylist: {attributes: {CODECS: 'mp4a.40.5'}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {audio: 'mp4a.40.5'}, 'codecs returned'); +}); + +QUnit.test('can get audio only codecs from main loader ', function(assert) { + this.contentSetup({ + mainStartingMedia: {hasVideo: false, hasAudio: true, audioCodec: 'mp4a.40.5'}, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {audio: 'mp4a.40.5'}, 'codecs returned'); +}); + +QUnit.test('can get video only codecs from main playlist', function(assert) { + this.contentSetup({ + mainStartingMedia: {hasVideo: true, hasAudio: false}, + mainPlaylist: {attributes: {CODECS: 'avc1.4c400d'}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4c400d'}, 'codecs returned'); +}); + +QUnit.test('can get video only codecs from main loader', function(assert) { + this.contentSetup({ + mainStartingMedia: {hasVideo: true, hasAudio: false, videoCodec: 'avc1.4c400d'}, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4c400d'}, 'codecs returned'); +}); + +QUnit.test('can get codecs from startingMedia', function(assert) { + this.contentSetup({ + mainStartingMedia: {videoCodec: 'avc1.4c400d', hasVideo: true, hasAudio: false}, + audioStartingMedia: {audioCodec: 'mp4a.40.5', hasVideo: false, hasAudio: true}, + mainPlaylist: {attributes: {}}, + audioPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4c400d', audio: 'mp4a.40.5'}, 'codecs returned'); +}); + +QUnit.test('playlist codecs take priority over others', function(assert) { + this.contentSetup({ + mainStartingMedia: {videoCodec: 'avc1.4c400d', hasVideo: true, hasAudio: false}, + audioStartingMedia: {audioCodec: 'mp4a.40.5', hasVideo: false, hasAudio: true}, + mainPlaylist: {attributes: {CODECS: 'avc1.4b400d', AUDIO: 'low-quality'}}, + audioPlaylist: {attributes: {CODECS: 'mp4a.40.20'}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4b400d', audio: 'mp4a.40.20'}, 'codecs returned'); +}); + +QUnit.test('uses default codecs if no codecs are found', function(assert) { + this.contentSetup({ + mainStartingMedia: {hasVideo: true, hasAudio: false}, + audioStartingMedia: {hasVideo: false, hasAudio: true}, + mainPlaylist: {attributes: {}}, + audioPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [], 'did not blacklist anything'); + assert.deepEqual(codecs, {video: 'avc1.4d400d', audio: 'mp4a.40.2'}, 'codecs returned'); +}); + +QUnit.test('excludes playlist without detected audio/video', function(assert) { + this.contentSetup({ + mainStartingMedia: {}, + audioStartingMedia: {}, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + message: 'Could not determine codecs for playlist.', + playlist: {attributes: {}} + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'no codecs returned'); +}); + +QUnit.test('excludes unsupported muxer codecs for ts', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: true, + audioCodec: 'ac-3' + }, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {}}, + internal: true, + message: 'muxer does not support codec(s): "hvc1.2.4.L123.B0,ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('excludes unsupported browser codecs for muxed fmp4', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: true, + isFmp4: true, + isMuxed: true, + audioCodec: 'ac-3' + }, + mainPlaylist: {attributes: {}} + }); + + window.MediaSource.isTypeSupported = (type) => (/(mp4a|avc1)/).test(type); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {}}, + internal: true, + message: 'browser does not support codec(s): "hvc1.2.4.L123.B0,ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('excludes unsupported muxer codecs for muxed ts', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: true, + isMuxed: true, + audioCodec: 'ac-3' + }, + mainPlaylist: {attributes: {}} + }); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {}}, + internal: true, + message: 'muxer does not support codec(s): "hvc1.2.4.L123.B0,ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('excludes unsupported browser codecs for fmp4', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: true, + audioCodec: 'ac-3', + isFmp4: true + }, + mainPlaylist: {attributes: {}} + }); + + window.MediaSource.isTypeSupported = (type) => (/(mp4a|avc1)/).test(type); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {}}, + internal: true, + message: 'browser does not support codec(s): "hvc1.2.4.L123.B0,ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('excludes unsupported codecs video ts, audio fmp4', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: false + }, + audioStartingMedia: { + hasVideo: false, + hasAudio: true, + audioCodec: 'ac-3', + isFmp4: true + }, + mainPlaylist: {attributes: {AUDIO: 'low-quality'}}, + audioPlaylist: {attributes: {}} + }); + + window.MediaSource.isTypeSupported = (type) => (/(mp4a|avc1)/).test(type); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {AUDIO: 'low-quality'}}, + internal: true, + message: 'muxer does not support codec(s): "hvc1.2.4.L123.B0", browser does not support codec(s): "ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('excludes unsupported codecs video fmp4, audio ts', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: false, + isFmp4: true + }, + audioStartingMedia: { + hasVideo: false, + hasAudio: true, + audioCodec: 'ac-3' + }, + mainPlaylist: {attributes: {AUDIO: 'low-quality'}}, + audioPlaylist: {attributes: {}} + }); + + window.MediaSource.isTypeSupported = (type) => (/(mp4a|avc1)/).test(type); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {AUDIO: 'low-quality'}}, + internal: true, + message: 'browser does not support codec(s): "hvc1.2.4.L123.B0", muxer does not support codec(s): "ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('excludes all of audio group on unsupported audio', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: false + }, + audioStartingMedia: { + hasVideo: false, + hasAudio: true, + audioCodec: 'ac-3' + }, + mainPlaylist: {id: 'bar', attributes: {AUDIO: 'low-quality'}}, + audioPlaylist: {attributes: {}} + }); + + this.master.playlists.push({id: 'foo', attributes: {AUDIO: 'low-quality'}}); + this.master.playlists.push({id: 'baz', attributes: {AUDIO: 'low-quality'}}); + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {AUDIO: 'low-quality'}, id: 'bar'}, + internal: true, + message: 'muxer does not support codec(s): "hvc1.2.4.L123.B0,ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); + assert.equal(this.master.playlists[2].id, 'foo', 'playlist 3 is the one we added'); + assert.equal(this.master.playlists[2].excludeUntil, Infinity, 'playlist 3 with same audio group excluded'); + assert.equal(this.master.playlists[3].id, 'baz', 'playlist 4 is the one we added'); + assert.equal(this.master.playlists[3].excludeUntil, Infinity, 'playlist 4 with same audio group excluded'); +}); + +QUnit.test('excludes on codec switch if codec switching not supported', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'hvc1.2.4.L123.B0', + hasVideo: true, + hasAudio: false, + isFmp4: true + }, + audioStartingMedia: { + hasVideo: false, + hasAudio: true, + audioCodec: 'ac-3', + isFmp4: true + }, + mainPlaylist: {attributes: {AUDIO: 'low-quality'}}, + audioPlaylist: {attributes: {}} + }); + + // sourceUpdater_ already setup + this.mpc.sourceUpdater_.ready = () => true; + this.mpc.sourceUpdater_.canChangeType = () => false; + this.mpc.sourceUpdater_.codecs = { + audio: 'mp4a.40.2', + video: 'avc1.4c400d' + }; + + // support all types + window.MediaSource.isTypeSupported = (type) => true; + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, [{ + blacklistDuration: Infinity, + playlist: {attributes: {AUDIO: 'low-quality'}}, + internal: true, + message: 'Codec switching not supported: "avc1.4c400d" -> "hvc1.2.4.L123.B0", "mp4a.40.2" -> "ac-3".' + }], 'blacklisted playlist'); + assert.deepEqual(codecs, void 0, 'codecs returned'); +}); + +QUnit.test('does not exclude on codec switch between the same base codec', function(assert) { + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'avc1.4d400e', + hasVideo: true, + hasAudio: false, + isFmp4: true + }, + audioStartingMedia: { + hasVideo: false, + hasAudio: true, + audioCodec: 'mp4a.40.5', + isFmp4: true + }, + mainPlaylist: {attributes: {AUDIO: 'low-quality'}}, + audioPlaylist: {attributes: {}} + }); + + // sourceUpdater_ already setup + this.mpc.sourceUpdater_.ready = () => true; + this.mpc.sourceUpdater_.canChangeType = () => false; + this.mpc.sourceUpdater_.codecs = { + audio: 'mp4a.40.2', + video: 'avc1.4c400d' + }; + + // support all types + window.MediaSource.isTypeSupported = (type) => true; + + const codecs = this.mpc.getCodecsOrExclude_(); + + assert.deepEqual(this.blacklists, []); + assert.deepEqual(codecs, {video: 'avc1.4d400e', audio: 'mp4a.40.5'}, 'codecs returned'); +}); + +QUnit.test('main loader only trackinfo works as expected', function(assert) { + this.mpc.mediaSource.readyState = 'open'; + let createBuffers = 0; + let switchBuffers = 0; + let expectedCodecs; + + this.mpc.sourceUpdater_.createSourceBuffers = (codecs) => { + assert.deepEqual(codecs, expectedCodecs, 'create source buffers codecs as expected'); + createBuffers++; + }; + this.mpc.sourceUpdater_.addOrChangeSourceBuffers = (codecs) => { + assert.deepEqual(codecs, expectedCodecs, 'codec switch as expected'); + switchBuffers++; + }; + + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'avc1.4d400e', + hasVideo: true, + hasAudio: true, + audioCodec: 'mp4a.40.2' + }, + mainPlaylist: {attributes: {}} + }); + + expectedCodecs = { + video: 'avc1.4d400e', + audio: 'mp4a.40.2' + }; + this.mpc.mainSegmentLoader_.trigger('trackinfo'); + + assert.equal(createBuffers, 1, 'createSourceBuffers called'); + assert.equal(switchBuffers, 0, 'addOrChangeSourceBuffers not called'); + + this.mpc.sourceUpdater_.ready = () => true; + this.mpc.sourceUpdater_.canChangeType = () => true; + + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'avc1.4c400e', + hasVideo: true, + hasAudio: true, + audioCodec: 'mp4a.40.5' + }, + mainPlaylist: {attributes: {}} + }); + + expectedCodecs = { + video: 'avc1.4c400e', + audio: 'mp4a.40.5' + }; + + this.mpc.mainSegmentLoader_.trigger('trackinfo'); + + assert.equal(createBuffers, 1, 'createBuffers not called'); + assert.equal(switchBuffers, 1, 'addOrChangeSourceBuffers called'); +}); + +QUnit.test('main & audio loader only trackinfo works as expected', function(assert) { + this.mpc.mediaSource.readyState = 'open'; + let createBuffers = 0; + let switchBuffers = 0; + let expectedCodecs; + + this.mpc.sourceUpdater_.createSourceBuffers = (codecs) => { + assert.deepEqual(codecs, expectedCodecs, 'create source buffers codecs as expected'); + createBuffers++; + }; + this.mpc.sourceUpdater_.addOrChangeSourceBuffers = (codecs) => { + assert.deepEqual(codecs, expectedCodecs, 'codec switch as expected'); + switchBuffers++; + }; + + this.contentSetup({ + mainStartingMedia: { + videoCodec: 'avc1.4d400e', + hasVideo: true, + hasAudio: false + }, + mainPlaylist: {attributes: {}}, + audioPlaylist: {attributes: {}} + }); + + expectedCodecs = { + video: 'avc1.4d400e', + audio: 'mp4a.40.2' + }; + + this.mpc.mainSegmentLoader_.trigger('trackinfo'); + + assert.equal(createBuffers, 0, 'createSourceBuffers not called'); + assert.equal(switchBuffers, 0, 'addOrChangeSourceBuffers not called'); + + this.mpc.audioSegmentLoader_.startingMedia_ = { + hasVideo: false, + hasAudio: true, + audioCodec: 'mp4a.40.2' + }; + + this.mpc.audioSegmentLoader_.trigger('trackinfo'); + + assert.equal(createBuffers, 1, 'createSourceBuffers called'); + assert.equal(switchBuffers, 0, 'addOrChangeSourceBuffers not called'); + + this.mpc.sourceUpdater_.ready = () => true; + this.mpc.sourceUpdater_.canChangeType = () => true; + + this.mpc.mainSegmentLoader_.startingMedia_ = { + videoCodec: 'avc1.4c400e', + hasVideo: true, + hasAudio: false + }; + + expectedCodecs = { + video: 'avc1.4c400e', + audio: 'mp4a.40.2' + }; + + this.mpc.mainSegmentLoader_.trigger('trackinfo'); + + assert.equal(createBuffers, 1, 'createBuffers not called'); + assert.equal(switchBuffers, 1, 'addOrChangeSourceBuffers called'); + + this.mpc.audioSegmentLoader_.startingMedia_ = { + hasVideo: false, + hasAudio: true, + audioCodec: 'mp4a.40.5' + }; + + expectedCodecs = { + video: 'avc1.4c400e', + audio: 'mp4a.40.5' + }; + + this.mpc.audioSegmentLoader_.trigger('trackinfo'); + + assert.equal(createBuffers, 1, 'createBuffers not called'); + assert.equal(switchBuffers, 2, 'addOrChangeSourceBuffers called'); +}); diff --git a/test/source-updater.test.js b/test/source-updater.test.js index abe4974ac..00d32be4c 100644 --- a/test/source-updater.test.js +++ b/test/source-updater.test.js @@ -65,7 +65,152 @@ QUnit.module('Source Updater', { } }); -QUnit.test('waits for sourceopen to createSourceBuffers', function(assert) { +QUnit.test('verifies that sourcebuffer is in source buffers list before attempting actions', function(assert) { + this.sourceUpdater.dispose(); + const actionCalls = { + videoRemoveSourceBuffer: 0, + videoAppendBuffer: 0, + videoRemove: 0, + videoTimestampOffset: 0, + videoBuffered: 0, + videoAbort: 0, + videoChangeType: 0, + audioRemoveSourceBuffer: 0, + audioAppendBuffer: 0, + audioRemove: 0, + audioTimestampOffset: 0, + audioBuffered: 0, + audioAbort: 0, + audioChangeType: 0 + }; + + const createMediaSource = () => { + + const mediaSource = new videojs.EventTarget(); + + mediaSource.readyState = 'open'; + mediaSource.sourceBuffers = []; + mediaSource.removeSourceBuffer = (sb) => { + if (sb.type_ === 'video') { + actionCalls.videoRemoveSourceBuffer++; + } else { + actionCalls.audioRemoveSourceBuffer++; + } + }; + + mediaSource.addSourceBuffer = (mime) => { + const type = (/^audio/).test(mime) ? 'audio' : 'video'; + + const sb = new videojs.EventTarget(); + + sb.appendBuffer = () => { + actionCalls[`${type}AppendBuffer`]++; + }; + sb.remove = () => { + actionCalls[`${type}Remove`]++; + }; + sb.abort = () => { + actionCalls[`${type}Abort`]++; + }; + sb.changeType = () => { + actionCalls[`${type}ChangeType`]++; + }; + sb.type_ = type; + Object.defineProperty(sb, 'buffered', { + get: () => { + actionCalls[`${type}Buffered`]++; + return videojs.createTimeRanges([0, 15]); + } + }); + + Object.defineProperty(sb, 'timestampOffset', { + get: () => { + return 444; + }, + set: () => { + actionCalls[`${type}TimestampOffset`]++; + } + }); + return sb; + }; + + return mediaSource; + }; + + const runTestFunctions = () => { + this.sourceUpdater.canChangeType = () => true; + this.sourceUpdater.canRemoveSourceBuffer = () => true; + this.sourceUpdater.appendBuffer({type: 'video', bytes: []}); + this.sourceUpdater.videoBuffer.trigger('updateend'); + this.sourceUpdater.appendBuffer({type: 'audio', bytes: []}); + this.sourceUpdater.audioBuffer.trigger('updateend'); + this.sourceUpdater.audioBuffered(); + this.sourceUpdater.videoBuffered(); + this.sourceUpdater.buffered(); + this.sourceUpdater.removeVideo(0, 1); + this.sourceUpdater.videoBuffer.trigger('updateend'); + this.sourceUpdater.removeAudio(0, 1); + this.sourceUpdater.audioBuffer.trigger('updateend'); + this.sourceUpdater.changeType('audio', 'foo'); + this.sourceUpdater.changeType('video', 'bar'); + this.sourceUpdater.abort('audio'); + this.sourceUpdater.abort('video'); + this.sourceUpdater.audioTimestampOffset(123); + this.sourceUpdater.videoTimestampOffset(123); + this.sourceUpdater.removeSourceBuffer('video'); + this.sourceUpdater.removeSourceBuffer('audio'); + }; + + this.sourceUpdater = new SourceUpdater(createMediaSource()); + this.sourceUpdater.createSourceBuffers({ + audio: 'mp4a.40.2', + video: 'avc1.4d400d' + }); + + assert.ok(this.sourceUpdater.videoBuffer, 'has video buffer'); + assert.ok(this.sourceUpdater.audioBuffer, 'has audio buffer'); + + this.sourceUpdater.mediaSource.sourceBuffers = []; + runTestFunctions(); + + Object.keys(actionCalls).forEach((name) => { + assert.equal(actionCalls[name], 0, `no ${name} without sourcebuffer in list`); + }); + + this.sourceUpdater.dispose(); + this.sourceUpdater = new SourceUpdater(createMediaSource()); + this.sourceUpdater.createSourceBuffers({ + audio: 'mp4a.40.2', + video: 'avc1.4d400d' + }); + + assert.ok(this.sourceUpdater.videoBuffer, 'has video buffer'); + assert.ok(this.sourceUpdater.audioBuffer, 'has audio buffer'); + + this.sourceUpdater.mediaSource.sourceBuffers = [ + this.sourceUpdater.videoBuffer, + this.sourceUpdater.audioBuffer + ]; + runTestFunctions(); + assert.deepEqual(actionCalls, { + audioAbort: 1, + audioAppendBuffer: 1, + audioBuffered: 8, + audioChangeType: 1, + audioRemove: 1, + audioRemoveSourceBuffer: 1, + audioTimestampOffset: 1, + videoAbort: 1, + videoAppendBuffer: 1, + videoBuffered: 8, + videoChangeType: 1, + videoRemove: 1, + videoRemoveSourceBuffer: 1, + videoTimestampOffset: 1 + }, 'calls functions correctly with sourcebuffer in list'); +}); + +QUnit.test('waits for sourceopen to create source buffers', function(assert) { this.sourceUpdater.dispose(); const video = document.createElement('video'); @@ -95,32 +240,38 @@ QUnit.test('waits for sourceopen to createSourceBuffers', function(assert) { }); }); -QUnit.test('runs callback when source buffer is created', function(assert) { - this.sourceUpdater.dispose(); - const video = document.createElement('video'); - - this.mediaSource = new window.MediaSource(); - // need to attach the real media source to a video element for the media source to - // change to an open ready state - video.src = URL.createObjectURL(this.mediaSource); - this.sourceUpdater = new SourceUpdater(this.mediaSource); - - this.sourceUpdater.createSourceBuffers({ - audio: 'mp4a.40.2', - video: 'avc1.4d400d' - }); - - this.sourceUpdater.appendBuffer({type: 'video', bytes: mp4VideoTotal()}); - +QUnit.test('source buffer creation is queued', function(assert) { // wait for the source to open (or error) before running through tests return new Promise((accept, reject) => { + this.sourceUpdater.dispose(); + const video = document.createElement('video'); + + this.mediaSource = new window.MediaSource(); + this.mediaSource.addEventListener('sourceopen', () => { - assert.equal(this.sourceUpdater.queue.length, 0, 'nothing in queue'); + assert.equal(this.sourceUpdater.queue.length, 3, 'three things in queue'); assert.equal(this.sourceUpdater.pendingQueue, null, 'nothing in pendingQueue'); + assert.deepEqual( + this.sourceUpdater.queue.map((i) => i.name), + ['addSourceBuffer', 'addSourceBuffer', 'appendBuffer'], + 'queue is as expected' + ); accept(); }); + // need to attach the real media source to a video element for the media source to + // change to an open ready state + video.src = URL.createObjectURL(this.mediaSource); + this.sourceUpdater = new SourceUpdater(this.mediaSource); this.mediaSource.addEventListener('error', reject); + + this.sourceUpdater.createSourceBuffers({ + audio: 'mp4a.40.2', + video: 'avc1.4d400d' + }); + + this.sourceUpdater.appendBuffer({type: 'video', bytes: mp4VideoTotal()}); }); + }); QUnit.test('initial values', function(assert) { @@ -205,53 +356,6 @@ QUnit.test('ready with both an audio and video buffer', function(assert) { assert.ok(this.sourceUpdater.ready(), 'source updater is ready'); }); -QUnit.test('waits for sourceopen to create source buffers', function(assert) { - const addEventListeners = []; - const addSourceBuffers = []; - const mockMediaSource = { - // readyState starts as closed, source updater has to wait for it to open - readyState: 'closed', - addEventListener: - (name, callback) => addEventListeners.push({ name, callback }), - removeEventListener() {}, - addSourceBuffer: (mimeType) => { - addSourceBuffers.push(mimeType); - return { - // source updater adds event listeners immediately after creation, mock out to - // prevent errors - addEventListener() {}, - removeEventListener() {}, - abort() {} - }; - } - }; - - // create new source update instance to allow for mocked media source - const sourceUpdater = new SourceUpdater(mockMediaSource); - - assert.equal(addEventListeners.length, 0, 'no event listener calls'); - assert.equal(addSourceBuffers.length, 0, 'no add source buffer calls'); - - sourceUpdater.createSourceBuffers({ - video: 'avc1.4d400d', - audio: 'mp4a.40.2' - }); - - assert.equal(addEventListeners.length, 1, 'one event listener'); - assert.equal(addEventListeners[0].name, 'sourceopen', 'listening on sourceopen'); - assert.equal(addSourceBuffers.length, 0, 'no add source buffer calls'); - - mockMediaSource.readyState = 'open'; - addEventListeners[0].callback(); - - assert.equal(addEventListeners.length, 1, 'one event listener'); - assert.equal(addSourceBuffers.length, 2, 'two add source buffer calls'); - assert.equal(addSourceBuffers[0], 'audio/mp4;codecs="mp4a.40.2"', 'added audio source buffer'); - assert.equal(addSourceBuffers[1], 'video/mp4;codecs="avc1.4d400d"', 'added video source buffer'); - - sourceUpdater.dispose(); -}); - QUnit.test('audioBuffered can append to and get the audio buffer', function(assert) { const done = assert.async(); @@ -327,6 +431,7 @@ QUnit.test('buffered returns video buffer when only video', function(assert) { QUnit.test('buffered returns intersection of audio and video buffers', function(assert) { const origAudioBuffer = this.sourceUpdater.audioBuffer; const origVideoBuffer = this.sourceUpdater.videoBuffer; + const origMediaSource = this.sourceUpdater.mediaSource; // mocking the buffered ranges in this test because it's tough to know how much each // browser will actually buffer @@ -337,6 +442,13 @@ QUnit.test('buffered returns intersection of audio and video buffers', function( buffered: videojs.createTimeRanges([[1.25, 1.5], [5.1, 6.1], [10.5, 10.9]]) }; + this.sourceUpdater.mediaSource = { + sourceBuffers: [ + this.sourceUpdater.audioBuffer, + this.sourceUpdater.videoBuffer + ] + }; + timeRangesEqual( this.sourceUpdater.buffered(), videojs.createTimeRanges([[1.25, 1.5], [5.5, 5.6], [10.5, 10.9]]), @@ -345,6 +457,7 @@ QUnit.test('buffered returns intersection of audio and video buffers', function( this.sourceUpdater.audioBuffer = origAudioBuffer; this.sourceUpdater.videoBuffer = origVideoBuffer; + this.sourceUpdater.mediaSource = origMediaSource; }); QUnit.test('buffered returns audio buffered if no video buffer', function(assert) { @@ -393,7 +506,7 @@ QUnit.test('removeAudio removes audio buffer', function(assert) { this.sourceUpdater.appendBuffer({type: 'audio', bytes: mp4AudioTotal()}, () => { assert.equal(this.sourceUpdater.buffered().length, 1, 'has buffered time range'); assert.ok(this.sourceUpdater.buffered().end(0) > 0, 'buffered content'); - this.sourceUpdater.removeAudio(0, Infinity, () => { + this.sourceUpdater.removeAudio(0, this.sourceUpdater.buffered().end(0), () => { assert.equal(this.sourceUpdater.buffered().length, 0, 'no buffered conent'); done(); }); @@ -410,7 +523,7 @@ QUnit.test('removeVideo removes video buffer', function(assert) { this.sourceUpdater.appendBuffer({type: 'video', bytes: mp4VideoTotal()}, () => { assert.equal(this.sourceUpdater.buffered().length, 1, 'has buffered time range'); assert.ok(this.sourceUpdater.buffered().end(0) > 0, 'buffered content'); - this.sourceUpdater.removeVideo(0, Infinity, () => { + this.sourceUpdater.removeVideo(0, this.sourceUpdater.buffered().end(0), () => { assert.equal(this.sourceUpdater.buffered().length, 0, 'no buffered content'); done(); }); @@ -429,7 +542,7 @@ QUnit.test('removeAudio does not remove video buffer', function(assert) { assert.ok(this.sourceUpdater.videoBuffered().end(0) > 0, 'buffered audio content'); this.sourceUpdater.appendBuffer({type: 'audio', bytes: mp4AudioTotal()}, () => { assert.ok(this.sourceUpdater.audioBuffered().end(0) > 0, 'buffered video content'); - this.sourceUpdater.removeAudio(0, Infinity, () => { + this.sourceUpdater.removeAudio(0, this.sourceUpdater.audioBuffered().end(0), () => { assert.equal(this.sourceUpdater.audioBuffered().length, 0, 'removed audio content'); assert.equal(this.sourceUpdater.videoBuffered().length, 1, 'has buffered video time range'); assert.ok(this.sourceUpdater.videoBuffered().end(0) > 0, 'did not remove video content'); @@ -451,7 +564,7 @@ QUnit.test('removeVideo does not remove audio buffer', function(assert) { assert.ok(this.sourceUpdater.videoBuffered().end(0) > 0, 'buffered audio content'); this.sourceUpdater.appendBuffer({type: 'audio', bytes: mp4AudioTotal()}, () => { assert.ok(this.sourceUpdater.audioBuffered().end(0) > 0, 'buffered video content'); - this.sourceUpdater.removeVideo(0, Infinity, () => { + this.sourceUpdater.removeVideo(0, this.sourceUpdater.videoBuffered().end(0), () => { assert.equal(this.sourceUpdater.videoBuffered().length, 0, 'removed video content'); assert.equal(this.sourceUpdater.audioBuffered().length, 1, 'has buffered audio time range'); assert.ok(this.sourceUpdater.audioBuffered().end(0) > 0, 'did not remove audio content'); @@ -1068,7 +1181,7 @@ QUnit.test('dispose removes sourceopen listener', function(assert) { // need to call createSourceBuffers before the source updater will check that the media // source is opened - sourceUpdater.createSourceBuffers({}); + sourceUpdater.createSourceBuffers({audio: 'mp4a.40.2'}); assert.equal(addEventListenerCalls.length, 1, 'added one event listener'); assert.equal(addEventListenerCalls[0].type, 'sourceopen', 'added sourceopen listener'); @@ -1140,7 +1253,6 @@ QUnit.test('dispose removes sourceopen listener', function(assert) { assert.ok(!abort, 'abort not called right after remove'); }); - assert.ok(this.sourceUpdater[`${type}Buffer`].removing, true, 'removing is set'); this.sourceUpdater.dispose(); this.sourceUpdater[`${type}Buffer`].addEventListener('updateend', () => { diff --git a/test/test-helpers.js b/test/test-helpers.js index 4d00e69b3..c562956b3 100644 --- a/test/test-helpers.js +++ b/test/test-helpers.js @@ -52,6 +52,8 @@ class MockSourceBuffer extends videojs.EventTarget { this.updating = true; } + changeType() {} + remove(start, end) { this.updates_.push({ remove: [start, end] @@ -101,6 +103,14 @@ class MockMediaSource extends videojs.EventTarget { return sourceBuffer; } + removeSourceBuffer(sourceBuffer) { + const index = this.sourceBuffers.indexOf(sourceBuffer); + + if (index !== -1) { + this.sourceBuffers.splice(index, 1); + } + } + endOfStream(error) { this.readyState = 'ended'; this.error_ = error; diff --git a/test/videojs-http-streaming.test.js b/test/videojs-http-streaming.test.js index 674ccfb6a..0187bcc60 100644 --- a/test/videojs-http-streaming.test.js +++ b/test/videojs-http-streaming.test.js @@ -670,14 +670,14 @@ QUnit.test('codecs are passed to the source buffer', function(assert) { this.player.tech(true).vhs.masterPlaylistController_.mainSegmentLoader_.one('appending', () => { // always create separate audio and video source buffers assert.equal(codecs.length, 2, 'created two source buffers'); - assert.equal( - codecs[0], - 'audio/mp4;codecs="mp4a.40.9"', + assert.notEqual( + codecs.indexOf('audio/mp4;codecs="mp4a.40.9"'), + -1, 'specified the audio codec' ); - assert.equal( - codecs[1], - 'video/mp4;codecs="avc1.dd00dd"', + assert.notEqual( + codecs.indexOf('video/mp4;codecs="avc1.dd00dd"'), + -1, 'specified the video codec' ); done(); @@ -1692,7 +1692,7 @@ QUnit.test('does not blacklist compatible AAC codec strings', function(assert) { ); }); -QUnit.test('blacklists incompatible playlists by codec', function(assert) { +QUnit.test('blacklists incompatible playlists by codec, without codec switching', function(assert) { this.player.src({ src: 'manifest/master.m3u8', type: 'application/vnd.apple.mpegurl' @@ -1736,6 +1736,8 @@ QUnit.test('blacklists incompatible playlists by codec', function(assert) { const loader = mpc.mainSegmentLoader_; const master = this.player.tech_.vhs.playlists.master; + mpc.sourceUpdater_.canChangeType = () => false; + loader.startingMedia_ = {hasVideo: true, hasAudio: true}; loader.trigger('trackinfo'); const playlists = master.playlists; @@ -1750,6 +1752,66 @@ QUnit.test('blacklists incompatible playlists by codec', function(assert) { assert.strictEqual(typeof playlists[6].excludeUntil, 'undefined', 'did not blacklist seventh playlist'); }); +QUnit.test('does not blacklist incompatible codecs with codec switching', function(assert) { + this.player.src({ + src: 'manifest/master.m3u8', + type: 'application/vnd.apple.mpegurl' + }); + + this.clock.tick(1); + + openMediaSource(this.player, this.clock); + + const playlistString = + '#EXTM3U\n' + + // selected playlist + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="avc1.4d400d,mp4a.40.2"\n' + + 'media.m3u8\n' + + // compatible with selected playlist + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="avc1.4d400d,mp4a.40.2"\n' + + 'media1.m3u8\n' + + // incompatible by audio codec difference + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="avc1.4d400d,ac-3"\n' + + 'media2.m3u8\n' + + // incompatible by video codec difference + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="hvc1.4d400d,mp4a.40.2"\n' + + 'media3.m3u8\n' + + // incompatible, only audio codec + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="mp4a.40.2"\n' + + 'media4.m3u8\n' + + // incompatible, only video codec + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="avc1.4d400d"\n' + + 'media5.m3u8\n' + + // compatible with selected playlist + '#EXT-X-STREAM-INF:BANDWIDTH=1,CODECS="avc1,mp4a"\n' + + 'media6.m3u8\n'; + + // master + this.requests.shift().respond(200, null, playlistString); + + // media + this.standardXHRResponse(this.requests.shift()); + + const mpc = this.player.tech_.vhs.masterPlaylistController_; + const loader = mpc.mainSegmentLoader_; + const master = this.player.tech_.vhs.playlists.master; + + mpc.sourceUpdater_.canChangeType = () => true; + + loader.startingMedia_ = {hasVideo: true, hasAudio: true}; + loader.trigger('trackinfo'); + const playlists = master.playlists; + + assert.strictEqual(playlists.length, 7, 'six playlists total'); + assert.strictEqual(typeof playlists[0].excludeUntil, 'undefined', 'did not blacklist first playlist'); + assert.strictEqual(typeof playlists[1].excludeUntil, 'undefined', 'did not blacklist second playlist'); + assert.strictEqual(typeof playlists[2].excludeUntil, 'undefined', 'blacklisted incompatible audio playlist'); + assert.strictEqual(typeof playlists[3].excludeUntil, 'undefined', 'blacklisted incompatible video playlist'); + assert.strictEqual(playlists[4].excludeUntil, Infinity, 'blacklisted audio only playlist'); + assert.strictEqual(playlists[5].excludeUntil, Infinity, 'blacklisted video only playlist'); + assert.strictEqual(typeof playlists[6].excludeUntil, 'undefined', 'did not blacklist seventh playlist'); +}); + QUnit.test('blacklists fmp4 playlists by browser support', function(assert) { const oldIsTypeSupported = window.MediaSource.isTypeSupported; @@ -1880,7 +1942,7 @@ QUnit.test('blacklists ts playlists by muxer support', function(assert) { assert.strictEqual(playlists.length, 3, 'three playlists total'); assert.strictEqual(playlists[0].excludeUntil, Infinity, 'blacklisted first playlist'); assert.strictEqual(playlists[1].excludeUntil, Infinity, 'blacklisted second playlist'); - assert.strictEqual(typeof playlists[2].excludeUntil, 'undefined', 'did not blacklist second playlist'); + assert.strictEqual(typeof playlists[2].excludeUntil, 'undefined', 'did not blacklist third playlist'); assert.deepEqual(debugLogs, [ `Internal problem encountered with playlist ${playlists[0].id}. muxer does not support codec(s): "hvc1". Switching to playlist ${playlists[1].id}.`, `Internal problem encountered with playlist ${playlists[1].id}. muxer does not support codec(s): "ac-3". Switching to playlist ${playlists[2].id}.`