Merge pull request #6157 from thornbill/media-segment-actions

Add media segment skipping

src/apps/stable/features/playback/constants/mediaSegmentAction.ts

@@ -0,0 +1,7 @@
+/**
+ * Actions that are triggered for media segments.
+ */
+export enum MediaSegmentAction {
+    None = 'None',
+    Skip = 'Skip'
+}

src/apps/stable/features/playback/utils/mediaSegmentManager.ts

@@ -0,0 +1,131 @@
+import type { Api } from '@jellyfin/sdk/lib/api';
+import type { MediaSegmentDto } from '@jellyfin/sdk/lib/generated-client/models/media-segment-dto';
+import { MediaSegmentType } from '@jellyfin/sdk/lib/generated-client/models/media-segment-type';
+import { MediaSegmentsApi } from '@jellyfin/sdk/lib/generated-client/api/media-segments-api';
+
+import type { PlaybackManager } from 'components/playback/playbackmanager';
+import ServerConnections from 'components/ServerConnections';
+import { TICKS_PER_MILLISECOND, TICKS_PER_SECOND } from 'constants/time';
+import { currentSettings as userSettings } from 'scripts/settings/userSettings';
+import type { PlayerState } from 'types/playbackStopInfo';
+import type { Event } from 'utils/events';
+import { toApi } from 'utils/jellyfin-apiclient/compat';
+
+import { getMediaSegmentAction } from './mediaSegmentSettings';
+import { findCurrentSegment } from './mediaSegments';
+import { PlaybackSubscriber } from './playbackSubscriber';
+import { MediaSegmentAction } from '../constants/mediaSegmentAction';
+
+class MediaSegmentManager extends PlaybackSubscriber {
+    private hasSegments = false;
+    private isLastSegmentIgnored = false;
+    private lastSegmentIndex = 0;
+    private lastTime = -1;
+    private mediaSegmentTypeActions: Record<Partial<MediaSegmentType>, MediaSegmentAction> | undefined;
+    private mediaSegments: MediaSegmentDto[] = [];
+
+    private async fetchMediaSegments(api: Api, itemId: string, includeSegmentTypes: MediaSegmentType[]) {
+        // FIXME: Replace with SDK getMediaSegmentsApi function when available in stable
+        const mediaSegmentsApi = new MediaSegmentsApi(api.configuration, undefined, api.axiosInstance);
+
+        try {
+            const { data: mediaSegments } = await mediaSegmentsApi.getItemSegments({ itemId, includeSegmentTypes });
+            this.mediaSegments = mediaSegments.Items || [];
+        } catch (err) {
+            console.error('[MediaSegmentManager] failed to fetch segments', err);
+            this.mediaSegments = [];
+        }
+    }
+
+    private performAction(mediaSegment: MediaSegmentDto) {
+        if (!this.mediaSegmentTypeActions || !mediaSegment.Type || !this.mediaSegmentTypeActions[mediaSegment.Type]) {
+            console.error('[MediaSegmentManager] segment type missing from action map', mediaSegment, this.mediaSegmentTypeActions);
+            return;
+        }
+
+        const action = this.mediaSegmentTypeActions[mediaSegment.Type];
+        if (action === MediaSegmentAction.Skip) {
+            // Ignore segment if playback progress has passed the segment's start time
+            if (mediaSegment.StartTicks !== undefined && this.lastTime > mediaSegment.StartTicks) {
+                console.info('[MediaSegmentManager] ignoring skipping segment that has been seeked back into', mediaSegment);
+                this.isLastSegmentIgnored = true;
+                return;
+            } else if (mediaSegment.EndTicks) {
+                // If there is an end time, seek to it
+                // Do not skip if duration < 1s to avoid slow stream changes
+                if (mediaSegment.StartTicks && mediaSegment.EndTicks - mediaSegment.StartTicks < TICKS_PER_SECOND) {
+                    console.info('[MediaSegmentManager] ignoring skipping segment with duration <1s', mediaSegment);
+                    this.isLastSegmentIgnored = true;
+                    return;
+                }
+
+                console.debug('[MediaSegmentManager] skipping to %s ms', mediaSegment.EndTicks / TICKS_PER_MILLISECOND);
+                this.playbackManager.seek(mediaSegment.EndTicks, this.player);
+            } else {
+                // If there is no end time, skip to the next track
+                console.debug('[MediaSegmentManager] skipping to next item in queue');
+                this.playbackManager.nextTrack(this.player);
+            }
+        }
+    }
+
+    onPlayerPlaybackStart(_e: Event, state: PlayerState) {
+        this.isLastSegmentIgnored = false;
+        this.lastSegmentIndex = 0;
+        this.lastTime = -1;
+        this.hasSegments = !!state.MediaSource?.HasSegments;
+
+        const itemId = state.MediaSource?.Id;
+        const serverId = state.NowPlayingItem?.ServerId || ServerConnections.currentApiClient()?.serverId();
+
+        if (!this.hasSegments || !serverId || !itemId) return;
+
+        // Get the user settings for media segment actions
+        this.mediaSegmentTypeActions = Object.values(MediaSegmentType)
+            .map(type => ({
+                type,
+                action: getMediaSegmentAction(userSettings, type)
+            }))
+            .filter(({ action }) => !!action && action !== MediaSegmentAction.None)
+            .reduce((acc, { type, action }) => {
+                if (action) acc[type] = action;
+                return acc;
+            }, {} as Record<Partial<MediaSegmentType>, MediaSegmentAction>);
+
+        if (!Object.keys(this.mediaSegmentTypeActions).length) {
+            console.info('[MediaSegmentManager] user has no media segment actions enabled');
+            return;
+        }
+
+        const api = toApi(ServerConnections.getApiClient(serverId));
+        void this.fetchMediaSegments(
+            api,
+            itemId,
+            Object.keys(this.mediaSegmentTypeActions).map(t => t as keyof typeof MediaSegmentType));
+    }
+
+    onPlayerTimeUpdate() {
+        if (this.hasSegments && this.mediaSegments.length) {
+            const time = this.playbackManager.currentTime(this.player) * TICKS_PER_MILLISECOND;
+            const currentSegmentDetails = findCurrentSegment(this.mediaSegments, time, this.lastSegmentIndex);
+            if (
+                // The current time falls within a segment
+                currentSegmentDetails
+                // and the last segment is not ignored or the segment index has changed
+                && (!this.isLastSegmentIgnored || this.lastSegmentIndex !== currentSegmentDetails.index)
+            ) {
+                console.debug(
+                    '[MediaSegmentManager] found %s segment at %s ms',
+                    currentSegmentDetails.segment.Type,
+                    time / TICKS_PER_MILLISECOND,
+                    currentSegmentDetails);
+                this.isLastSegmentIgnored = false;
+                this.performAction(currentSegmentDetails.segment);
+                this.lastSegmentIndex = currentSegmentDetails.index;
+            }
+            this.lastTime = time;
+        }
+    }
+}
+
+export const bindMediaSegmentManager = (playbackManager: PlaybackManager) => new MediaSegmentManager(playbackManager);

src/apps/stable/features/playback/utils/mediaSegmentSettings.ts

@@ -0,0 +1,14 @@
+import { MediaSegmentType } from '@jellyfin/sdk/lib/generated-client/models/media-segment-type';
+
+import { UserSettings } from 'scripts/settings/userSettings';
+
+import { MediaSegmentAction } from '../constants/mediaSegmentAction';
+
+const PREFIX = 'segmentTypeAction';
+
+export const getId = (type: MediaSegmentType) => `${PREFIX}__${type}`;
+
+export function getMediaSegmentAction(userSettings: UserSettings, type: MediaSegmentType): MediaSegmentAction | undefined {
+    const action = userSettings.get(getId(type), false);
+    return action ? action as MediaSegmentAction : undefined;
+}

src/apps/stable/features/playback/utils/mediaSegments.test.ts

@@ -0,0 +1,68 @@
+import type { MediaSegmentDto } from '@jellyfin/sdk/lib/generated-client/models/media-segment-dto';
+import { MediaSegmentType } from '@jellyfin/sdk/lib/generated-client/models/media-segment-type';
+import { describe, expect, it } from 'vitest';
+
+import { findCurrentSegment } from './mediaSegments';
+
+const TEST_SEGMENTS: MediaSegmentDto[] = [
+    {
+        Id: 'intro',
+        Type: MediaSegmentType.Intro,
+        StartTicks: 0,
+        EndTicks: 10
+    },
+    {
+        Id: 'preview',
+        Type: MediaSegmentType.Preview,
+        StartTicks: 20,
+        EndTicks: 30
+    },
+    {
+        Id: 'recap',
+        Type: MediaSegmentType.Recap,
+        StartTicks: 30,
+        EndTicks: 40
+    },
+    {
+        Id: 'commercial',
+        Type: MediaSegmentType.Commercial,
+        StartTicks: 40,
+        EndTicks: 50
+    },
+    {
+        Id: 'outro',
+        Type: MediaSegmentType.Outro,
+        StartTicks: 50,
+        EndTicks: 60
+    }
+];
+
+describe('findCurrentSegment()', () => {
+    it('Should return the current segment', () => {
+        let segmentDetails = findCurrentSegment(TEST_SEGMENTS, 23);
+        expect(segmentDetails).toBeDefined();
+        expect(segmentDetails?.index).toBe(1);
+        expect(segmentDetails?.segment?.Id).toBe('preview');
+
+        segmentDetails = findCurrentSegment(TEST_SEGMENTS, 5, 1);
+        expect(segmentDetails).toBeDefined();
+        expect(segmentDetails?.index).toBe(0);
+        expect(segmentDetails?.segment?.Id).toBe('intro');
+
+        segmentDetails = findCurrentSegment(TEST_SEGMENTS, 42, 3);
+        expect(segmentDetails).toBeDefined();
+        expect(segmentDetails?.index).toBe(3);
+        expect(segmentDetails?.segment?.Id).toBe('commercial');
+    });
+
+    it('Should return undefined if not in a segment', () => {
+        let segmentDetails = findCurrentSegment(TEST_SEGMENTS, 16);
+        expect(segmentDetails).toBeUndefined();
+
+        segmentDetails = findCurrentSegment(TEST_SEGMENTS, 10, 1);
+        expect(segmentDetails).toBeUndefined();
+
+        segmentDetails = findCurrentSegment(TEST_SEGMENTS, 100);
+        expect(segmentDetails).toBeUndefined();
+    });
+});

src/apps/stable/features/playback/utils/mediaSegments.ts

@@ -0,0 +1,41 @@
+import type { MediaSegmentDto } from '@jellyfin/sdk/lib/generated-client/models/media-segment-dto';
+
+const isBeforeSegment = (segment: MediaSegmentDto, time: number, direction: number) => {
+    if (direction === -1) {
+        return (
+            typeof segment.EndTicks !== 'undefined'
+            && segment.EndTicks <= time
+        );
+    }
+    return (
+        typeof segment.StartTicks !== 'undefined'
+        && segment.StartTicks > time
+    );
+};
+
+const isInSegment = (segment: MediaSegmentDto, time: number) => (
+    typeof segment.StartTicks !== 'undefined'
+    && segment.StartTicks <= time
+    && (typeof segment.EndTicks === 'undefined' || segment.EndTicks > time)
+);
+
+export const findCurrentSegment = (segments: MediaSegmentDto[], time: number, lastIndex = 0) => {
+    const lastSegment = segments[lastIndex];
+    if (isInSegment(lastSegment, time)) {
+        return { index: lastIndex, segment: lastSegment };
+    }
+
+    let direction = 1;
+    if (lastIndex > 0 && lastSegment.StartTicks && lastSegment.StartTicks > time) {
+        direction = -1;
+    }
+
+    for (
+        let index = lastIndex, segment = segments[index];
+        index >= 0 && index < segments.length;
+        index += direction, segment = segments[index]
+    ) {
+        if (isBeforeSegment(segment, time, direction)) return;
+        if (isInSegment(segment, time)) return { index, segment };
+    }
+};

src/components/playback/playbackmanager.js

@@ -1,5 +1,6 @@
 import { PlaybackErrorCode } from '@jellyfin/sdk/lib/generated-client/models/playback-error-code.js';
 import { getMediaInfoApi } from '@jellyfin/sdk/lib/utils/api/media-info-api';
+import { MediaType } from '@jellyfin/sdk/lib/generated-client/models/media-type';
 import merge from 'lodash-es/merge';
 import Screenfull from 'screenfull';
 
@@ -19,8 +20,8 @@ import { PluginType } from '../../types/plugin.ts';
 import { includesAny } from '../../utils/container.ts';
 import { getItems } from '../../utils/jellyfin-apiclient/getItems.ts';
 import { getItemBackdropImageUrl } from '../../utils/jellyfin-apiclient/backdropImage';
-import { MediaType } from '@jellyfin/sdk/lib/generated-client/models/media-type';
 
+import { bindMediaSegmentManager } from 'apps/stable/features/playback/utils/mediaSegmentManager';
 import { MediaError } from 'types/mediaError';
 import { getMediaError } from 'utils/mediaError';
 import { toApi } from 'utils/jellyfin-apiclient/compat';
@@ -3663,6 +3664,8 @@ export class PlaybackManager {
                 Events.on(serverNotifications, 'ServerRestarting', self.setDefaultPlayerActive.bind(self));
             });
         }
+
+        bindMediaSegmentManager(self);
     }
 
     getCurrentPlayer() {

src/components/playbackSettings/playbackSettings.js

@@ -1,3 +1,9 @@
+import { MediaSegmentType } from '@jellyfin/sdk/lib/generated-client/models/media-segment-type';
+import escapeHTML from 'escape-html';
+
+import { MediaSegmentAction } from 'apps/stable/features/playback/constants/mediaSegmentAction';
+import { getId, getMediaSegmentAction } from 'apps/stable/features/playback/utils/mediaSegmentSettings';
+
 import appSettings from '../../scripts/settings/appSettings';
 import { appHost } from '../apphost';
 import browser from '../../scripts/browser';
@@ -6,12 +12,12 @@ import qualityoptions from '../qualityOptions';
 import globalize from '../../lib/globalize';
 import loading from '../loading/loading';
 import Events from '../../utils/events.ts';
-import '../../elements/emby-select/emby-select';
-import '../../elements/emby-checkbox/emby-checkbox';
 import ServerConnections from '../ServerConnections';
 import toast from '../toast/toast';
 import template from './playbackSettings.template.html';
-import escapeHTML from 'escape-html';
+
+import '../../elements/emby-select/emby-select';
+import '../../elements/emby-checkbox/emby-checkbox';
 
 function fillSkipLengths(select) {
     const options = [5, 10, 15, 20, 25, 30];
@@ -40,6 +46,42 @@ function populateLanguages(select, languages) {
     select.innerHTML = html;
 }
 
+function populateMediaSegments(container, userSettings) {
+    const selectedValues = {};
+    const actionOptions = Object.values(MediaSegmentAction)
+        .map(action => {
+            const actionLabel = globalize.translate(`MediaSegmentAction.${action}`);
+            return `<option value='${action}'>${actionLabel}</option>`;
+        })
+        .join('');
+
+    const segmentSettings = [
+        // List the types in a logical order (and exclude "Unknown" type)
+        MediaSegmentType.Intro,
+        MediaSegmentType.Preview,
+        MediaSegmentType.Recap,
+        MediaSegmentType.Commercial,
+        MediaSegmentType.Outro
+    ].map(segmentType => {
+        const segmentTypeLabel = globalize.translate('LabelMediaSegmentsType', globalize.translate(`MediaSegmentType.${segmentType}`));
+        const id = getId(segmentType);
+        selectedValues[id] = getMediaSegmentAction(userSettings, segmentType) || MediaSegmentAction.None;
+        return `<div class="selectContainer">
+<select is="emby-select" id="${id}" class="segmentTypeAction" label="${segmentTypeLabel}">
+    ${actionOptions}
+</select>
+</div>`;
+    }).join('');
+
+    container.innerHTML = segmentSettings;
+
+    Object.entries(selectedValues)
+        .forEach(([id, value]) => {
+            const field = container.querySelector(`#${id}`);
+            if (field) field.value = value;
+        });
+}
+
 function fillQuality(select, isInNetwork, mediatype, maxVideoWidth) {
     const options = mediatype === 'Audio' ? qualityoptions.getAudioQualityOptions({
 
@@ -219,6 +261,9 @@ function loadForm(context, user, userSettings, systemInfo, apiClient) {
     fillSkipLengths(selectSkipBackLength);
     selectSkipBackLength.value = userSettings.skipBackLength();
 
+    const mediaSegmentContainer = context.querySelector('.mediaSegmentActionContainer');
+    populateMediaSegments(mediaSegmentContainer, userSettings);
+
     loading.hide();
 }
 
@@ -257,6 +302,11 @@ function saveUser(context, user, userSettingsInstance, apiClient) {
     userSettingsInstance.skipForwardLength(context.querySelector('.selectSkipForwardLength').value);
     userSettingsInstance.skipBackLength(context.querySelector('.selectSkipBackLength').value);
 
+    const segmentTypeActions = context.querySelectorAll('.segmentTypeAction') || [];
+    Array.prototype.forEach.call(segmentTypeActions, actionEl => {
+        userSettingsInstance.set(actionEl.id, actionEl.value, false);
+    });
+
     return apiClient.updateUserConfiguration(user.Id, user.Configuration);
 }
 

src/components/playbackSettings/playbackSettings.template.html

@@ -156,6 +156,9 @@ <h2 class="sectionTitle">
         <div class="selectContainer">
             <select is="emby-select" class="selectSkipBackLength" label="${LabelSkipBackLength}"></select>
         </div>
+
+        <h3 class="sectionTitle">${HeaderMediaSegmentActions}</h3>
+        <div class="mediaSegmentActionContainer"></div>
     </div>
 
     <div class="verticalSection verticalSection-extrabottompadding">