Skip to content

Commit

Permalink
Chore: Support auto captions in legacy and edited captions (#825)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jeremy Press authored Jul 26, 2018
1 parent 26b8c9b commit 7b23933
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 109 deletions.
82 changes: 51 additions & 31 deletions src/lib/viewers/media/DashViewer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,57 +451,77 @@ class DashViewer extends VideoBaseViewer {
this.textTracks.map((track) => getLanguageName(track.language) || track.language),
getLanguageName(this.options.location.locale.substring(0, 2))
);
return;
}

// Attempt to load auto-generated captions
// @TODO 07-07-18: Support both auto-generated captions and subtitles from videos
this.loadAutoGeneratedCaptions();
}

/**
* Loads auto-generated captions
* Loads auto-generated captions from skills using a shaka SimpleTextDisplayer
* @TODO 07-07-18: Support both auto-generated captions and subtitles from videos
*
* @public
* @param {Object} transcriptCard - transcript card from Box Skills
* @return {void}
*/
loadAutoGeneratedCaptions() {
const textCues = [];

// Convert Box Skill transcript cards to Shaka Text Cues
const skillsCards = getProp(this.options, 'file.metadata.global.boxSkillsCards.cards');
// No skills are available on the file
if (!skillsCards) {
loadAutoGeneratedCaptions(transcriptCard) {
// Avoid regenerating captions if the object has not changed
if (this.transcript === transcriptCard) {
return;
}

const transcriptCard = skillsCards.find((card) => card.skill_card_type === 'transcript');
// No transcript can be found in the skills data
if (!transcriptCard) {
this.transcript = transcriptCard;
const textCues = this.createTextCues(transcriptCard);

// Don't do anything if there are no cues
if (!textCues.length) {
return;
}

const entries = transcriptCard.entries || [];
entries.forEach((entry) => {
// Set defaults if transcript data is malformed (start/end: 0s, text: '')
const { appears = [{}], text = '' } = entry;
const { start = 0, end = 0 } = Array.isArray(appears) && appears.length > 0 ? appears[0] : {};
textCues.push(new shaka.text.Cue(start, end, text));
});

if (textCues.length > 0) {
this.autoCaptionDisplayer = new shaka.text.SimpleTextDisplayer(this.mediaEl);
this.autoCaptionDisplayer.append(textCues);
this.player.configure({ textDisplayFactory: this.autoCaptionDisplayer });
// We know we are editing the transcript if we already have created an autoCaptionDisplayer
if (this.autoCaptionDisplayer) {
const areAutoCaptionsVisible = this.autoCaptionDisplayer.isTextVisible();
this.autoCaptionDisplayer.destroy();
this.setupAutoCaptionDisplayer(textCues);
this.autoCaptionDisplayer.setTextVisibility(areAutoCaptionsVisible);
} else {
this.setupAutoCaptionDisplayer(textCues);
// Update the subtitles/caption button to reflect auto-translation
this.mediaControls.setLabel(this.mediaControls.subtitlesButtonEl, __('media_auto_generated_captions'));
this.mediaControls.initSubtitles(
[__('auto_generated')],
getLanguageName(this.options.location.locale.substring(0, 2))
);

// Update the subtitles/caption button to reflect auto-translation
this.mediaControls.setLabel(this.mediaControls.subtitlesButtonEl, __('media_auto_generated_captions'));
}
}

/**
* Turns a Box Skills transcript card into an array of shaka text cues
*
* @param {Object} transcriptCard - transcript card from Box Skills
* @return {Array} Array of text cues
*/
createTextCues(transcriptCard) {
const entries = getProp(transcriptCard, 'entries', []);
return entries.map((entry) => {
// Set defaults if transcript data is malformed (start/end: 0s, text: '')
const { appears = [{}], text = '' } = entry;
const { start = 0, end = 0 } = Array.isArray(appears) && appears.length > 0 ? appears[0] : {};
return new shaka.text.Cue(start, end, text);
});
}

/**
* Sets up the autoCaption displayer using a shaka SimpleTextDisplayer
*
* @public
* @param {Array} textCues - Array of text cues which map text to a timestamp
* @return {void}
*/
setupAutoCaptionDisplayer(textCues) {
this.autoCaptionDisplayer = new shaka.text.SimpleTextDisplayer(this.mediaEl);
this.autoCaptionDisplayer.append(textCues);
this.player.configure({ textDisplayFactory: this.autoCaptionDisplayer });
}

/**
* Loads alternate audio streams
*
Expand Down
150 changes: 72 additions & 78 deletions src/lib/viewers/media/__tests__/DashViewer-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -794,100 +794,94 @@ describe('lib/viewers/media/DashViewer', () => {

expect(dash.textTracks).to.deep.equal([russian, foo, und, empty, doesntmatter, zero]);
});
});

it('should attempt to load auto-generated subtitles if no there are no subtitles from the video', () => {
sandbox.stub(dash, 'loadAutoGeneratedCaptions');
describe('loadAutoGeneratedCaptions', () => {
beforeEach(() => {
dash.autoCaptionDisplayer = {
append: () => {},
setTextVisibility: () => {},
isTextVisible: () => {},
destroy: () => {}
};
dash.createTextCues = sandbox.stub();
dash.setupAutoCaptionDisplayer = sandbox.stub();
stubs.mockPlayer = sandbox.mock(dash.player);
stubs.mockDisplayer = sandbox.mock(dash.autoCaptionDisplayer);
});

const subs = [];
stubs.mockPlayer.expects('getTextTracks').returns(subs);
stubs.mockControls.expects('initSubtitles').never();
const transcript = {
appears: [
{
start: 0,
end: 1
}
],
text: 'sometext'
};

dash.loadSubtitles();
const cues = [{ 1: 'foo' }, { 2: 'bar' }];

expect(dash.loadAutoGeneratedCaptions).to.be.called;
it('should do nothing if the transcript has not changed', () => {
dash.transcript = transcript;
dash.loadAutoGeneratedCaptions(transcript);
expect(dash.createTextCues).to.not.be.called;
});
});

describe('loadAutoGeneratedCaptions()', () => {
it('should convert to Shaka Cues, initialize subtitles, set button label & return true if there are auto-generated subtitles', () => {
dash.options = {
file: {
metadata: {
global: {
boxSkillsCards: {
cards: [
{
skill_card_type: 'transcript',
entries: [
{
appears: [
{
start: 0,
end: 1
}
],
text: 'sometext'
}
]
}
]
}
}
}
},
location: {
locale: 'en-US'
}
};
const appendStub = sandbox.stub();
sandbox.stub(shaka.text, 'SimpleTextDisplayer').returns({
append: appendStub
});
it('should do nothing if no text cues are found', () => {
dash.createTextCues.returns([]);
dash.setupAutoCaptionDisplayer = sandbox.stub();

stubs.mockPlayer.expects('configure').withArgs({
textDisplayFactory: sandbox.match.any
});
dash.loadAutoGeneratedCaptions(transcript);
expect(dash.setupAutoCaptionDisplayer).to.not.be.called;
});

it('should destroy and reset an existing autoCaptionDisplayer', () => {
stubs.mockDisplayer.expects('destroy');
stubs.mockDisplayer.expects('setTextVisibility');
dash.createTextCues.returns(cues);
dash.loadAutoGeneratedCaptions(transcript);

expect(dash.setupAutoCaptionDisplayer).to.be.calledWith(cues);
});

it('should setup a new autoCaptionDisplayer if setting up for first time', () => {
dash.autoCaptionDisplayer = null;
dash.createTextCues.returns(cues);
stubs.mockControls.expects('initSubtitles').withArgs(['Auto-Generated'], 'English');
stubs.mockControls.expects('setLabel').withArgs(sandbox.match.any, 'Auto-Generated Captions');

dash.loadAutoGeneratedCaptions();

expect(appendStub).to.be.called;
dash.loadAutoGeneratedCaptions(transcript);
expect(dash.setupAutoCaptionDisplayer).to.be.calledWith(cues);
});
});

it('should not set a custom text displayer if there are no transcript cards', () => {
stubs.mockPlayer.expects('configure').never();
stubs.mockControls.expects('initSubtitles').never();
stubs.mockControls.expects('setLabel').never();
describe('createTextCues()', () => {
it('should correctly map cues', () => {
const transcript = { entries: [{ appears: [{ start: 1, end: 2 }], text: 'foo' }] };
const result = dash.createTextCues(transcript)[0];
expect(result.startTime).to.equal(1);
expect(result.endTime).to.equal(2);

dash.loadAutoGeneratedCaptions();
expect(result.payload).to.equal('foo');
});
});

it('should convert to Shaka Cues, initialize subtitles, set button label & return true if there are auto-generated subtitles', () => {
dash.options = {
file: {
metadata: {
global: {
boxSkillsCards: {
cards: [
{
skill_card_type: 'not a transcript',
entries: []
}
]
}
}
}
},
location: {
locale: 'en-US'
}
};
stubs.mockPlayer.expects('configure').never();
stubs.mockControls.expects('initSubtitles').never();
stubs.mockControls.expects('setLabel').never();
describe('setupAutoCaptionDisplayer()', () => {
beforeEach(() => {
stubs.appendStub = sandbox.stub();
sandbox.stub(shaka.text, 'SimpleTextDisplayer').returns({
append: stubs.appendStub
});
});

it('should setup a simpleTextDisplayer and configure the player', () => {
stubs.mockPlayer.expects('configure').withArgs({
textDisplayFactory: sandbox.match.any
});

dash.loadAutoGeneratedCaptions();
dash.setupAutoCaptionDisplayer('foo');
expect(stubs.appendStub).to.be.called;
});
});

Expand Down

0 comments on commit 7b23933

Please sign in to comment.