From aee5c560d7f8620c30bd594e5be2b3cf70ce7e6d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Sun, 30 Jul 2023 13:20:27 +0300 Subject: [PATCH] Major refactoring of decoding module, added cached frames indication in player --- cvat-canvas/src/typescript/canvasModel.ts | 7 +- cvat-core/src/frames.ts | 755 +++++------------- cvat-data/src/ts/cvat-data.ts | 337 ++++---- cvat-ui/src/actions/annotation-actions.ts | 4 + .../components/annotation-page/styles.scss | 22 +- .../top-bar/player-navigation.tsx | 13 + .../annotation-page/top-bar/top-bar.tsx | 3 + .../annotation-page/top-bar/top-bar.tsx | 5 + cvat-ui/src/reducers/annotation-reducer.ts | 3 + cvat-ui/src/reducers/index.ts | 1 + 10 files changed, 414 insertions(+), 736 deletions(-) diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index aaa21282ea3..104eb71a2db 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -566,10 +566,9 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { this.data.objects = objectStates; this.notify(UpdateReasons.OBJECTS_UPDATED); }) - .catch((exception: any): void => { - this.data.exception = exception; - // don't notify when the frame is no longer needed - if (typeof exception !== 'number' || exception === this.data.imageID) { + .catch((exception: Error | number): void => { + if (typeof exception !== 'number') { + this.data.exception = exception; this.notify(UpdateReasons.DATA_FAILED); } }); diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index b1ed902d3ea..6a811f5a88d 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -3,28 +3,29 @@ // // SPDX-License-Identifier: MIT -import { isBrowser, isNode } from 'browser-or-node'; - -import * as cvatData from 'cvat-data'; -import { DimensionType } from 'enums'; +import _ from 'lodash'; +import { + FrameDecoder, BlockType, DimensionType, decodeContextImages, RequestOutdatedError, +} from 'cvat-data'; import PluginRegistry from './plugins'; import serverProxy, { RawFramesMetaData } from './server-proxy'; -import { - Exception, ArgumentError, DataError, ServerError, -} from './exceptions'; +import { Exception, ArgumentError, DataError } from './exceptions'; // frame storage by job id const frameDataCache: Record }; chunkSize: number; mode: 'annotation' | 'interpolation'; startFrame: number; stopFrame: number; - provider: cvatData.FrameProvider; - frameBuffer: FrameBuffer; + decodeForward: boolean; + forwardStep: number; + latestFrameDecodeRequest: number | null; + provider: FrameDecoder; decodedBlocksCacheSize: number; - activeChunkRequest: null; - nextChunkRequest: null; + activeChunkRequest: Promise | null; + activeContextRequest: Promise | null; + contextCache: Record; }> = {}; export class FramesMetaData { @@ -98,15 +99,20 @@ export class FramesMetaData { } export class FrameData { + public readonly filename: string; + public readonly width: number; + public readonly height: number; + public readonly number: number; + public readonly relatedFiles: number; + public readonly deleted: boolean; + public readonly jobID: number; + constructor({ width, height, name, jobID, frameNumber, - startFrame, - stopFrame, - decodeForward, deleted, related_files: relatedFiles, }) { @@ -125,7 +131,7 @@ export class FrameData { value: height, writable: false, }, - jid: { + jobID: { value: jobID, writable: false, }, @@ -137,18 +143,6 @@ export class FrameData { value: relatedFiles, writable: false, }, - startFrame: { - value: startFrame, - writable: false, - }, - stopFrame: { - value: stopFrame, - writable: false, - }, - decodeForward: { - value: decodeForward, - writable: false, - }, deleted: { value: deleted, writable: false, @@ -157,233 +151,160 @@ export class FrameData { ); } - async data(onServerRequest = () => {}) { + async data(onServerRequest = () => {}): Promise { const result = await PluginRegistry.apiWrapper.call(this, FrameData.prototype.data, onServerRequest); return result; } get imageData() { + // todo: check where it is used return this._data.imageData; } set imageData(imageData) { + // todo: check where it is used this._data.imageData = imageData; } } -FrameData.prototype.data.implementation = async function (onServerRequest) { - return new Promise((resolve, reject) => { - const resolveWrapper = (data) => { - this._data = { - imageData: data, - renderWidth: this.width, - renderHeight: this.height, - }; - return resolve(this._data); - }; - - if (this._data) { - resolve(this._data); - return; - } +Object.defineProperty(FrameData.prototype.data, 'implementation', { + value(this: FrameData, onServerRequest) { + return new Promise<{ + renderWidth: number; + renderHeight: number; + imageData: ImageBitmap | Blob; + } | Blob>((resolve, reject) => { + const { + provider, chunkSize, stopFrame, decodeForward, forwardStep, decodedBlocksCacheSize, + } = frameDataCache[this.jobID]; + const requestId = +_.uniqueId(); + const chunkNumber = Math.floor(this.number / chunkSize); + const frame = provider.frame(this.number); + if (frame) { + if (decodeForward && decodedBlocksCacheSize > 1 && !frameDataCache[this.jobID].activeChunkRequest) { + let firstFrameInNextChunk = this.number + forwardStep; + let nextChunkNumber = Math.floor(firstFrameInNextChunk / chunkSize); + while (nextChunkNumber === chunkNumber) { + firstFrameInNextChunk += forwardStep; + nextChunkNumber = Math.floor(firstFrameInNextChunk / chunkSize); + } - const { provider } = frameDataCache[this.jid]; - const { chunkSize } = frameDataCache[this.jid]; - const start = parseInt(this.number / chunkSize, 10) * chunkSize; - const stop = Math.min(this.stopFrame, (parseInt(this.number / chunkSize, 10) + 1) * chunkSize - 1); - const chunkNumber = Math.floor(this.number / chunkSize); - - const onDecodeAll = async (frameNumber) => { - if ( - frameDataCache[this.jid].activeChunkRequest && - chunkNumber === frameDataCache[this.jid].activeChunkRequest.chunkNumber - ) { - const callbackArray = frameDataCache[this.jid].activeChunkRequest.callbacks; - for (let i = callbackArray.length - 1; i >= 0; --i) { - if (callbackArray[i].frameNumber === frameNumber) { - const callback = callbackArray[i]; - callbackArray.splice(i, 1); - callback.resolve(await provider.frame(callback.frameNumber)); + // TODO: implement better decode forward + if (nextChunkNumber * chunkSize <= stopFrame && !provider.isChunkCached(nextChunkNumber)) { + frameDataCache[this.jobID].activeChunkRequest = new Promise((resolveForward) => { + serverProxy.frames + .getData(this.jobID, nextChunkNumber).then((chunk: ArrayBuffer) => { + provider.requestDecodeBlock( + chunk, + nextChunkNumber * chunkSize, + Math.min(stopFrame, (nextChunkNumber + 1) * chunkSize - 1), + () => {}, + () => { + resolveForward(); + frameDataCache[this.jobID].activeChunkRequest = null; + }, + () => { + resolveForward(); + frameDataCache[this.jobID].activeChunkRequest = null; + }, + ); + }); + }); } } - if (callbackArray.length === 0) { - frameDataCache[this.jid].activeChunkRequest = null; - } + + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: frame, + }); + return; } - }; - const rejectRequestAll = () => { - if ( - frameDataCache[this.jid].activeChunkRequest && - chunkNumber === frameDataCache[this.jid].activeChunkRequest.chunkNumber - ) { - for (const r of frameDataCache[this.jid].activeChunkRequest.callbacks) { - r.reject(r.frameNumber); + onServerRequest(); + frameDataCache[this.jobID].latestFrameDecodeRequest = requestId; + (frameDataCache[this.jobID].activeChunkRequest || Promise.resolve()).finally(() => { + if (frameDataCache[this.jobID].latestFrameDecodeRequest !== requestId) { + // not relevant request anymore + reject(this.number); + return; } - frameDataCache[this.jid].activeChunkRequest = null; - } - }; - const makeActiveRequest = () => { - const taskDataCache = frameDataCache[this.jid]; - const activeChunk = taskDataCache.activeChunkRequest; - activeChunk.request = serverProxy.frames - .getData(this.jid, activeChunk.chunkNumber) - .then((chunk) => { - frameDataCache[this.jid].activeChunkRequest.completed = true; - if (!taskDataCache.nextChunkRequest) { - provider.requestDecodeBlock( - chunk, - taskDataCache.activeChunkRequest.start, - taskDataCache.activeChunkRequest.stop, - taskDataCache.activeChunkRequest.onDecodeAll, - taskDataCache.activeChunkRequest.rejectRequestAll, - ); - } - }) - .catch((exception) => { - if (exception instanceof Exception) { - reject(exception); - } else { - reject(new Exception(exception.message)); - } - }) - .finally(() => { - if (taskDataCache.nextChunkRequest) { - if (taskDataCache.activeChunkRequest) { - for (const r of taskDataCache.activeChunkRequest.callbacks) { - r.reject(r.frameNumber); - } - } - taskDataCache.activeChunkRequest = taskDataCache.nextChunkRequest; - taskDataCache.nextChunkRequest = null; - makeActiveRequest(); - } - }); - }; + // it might appear during decoding, so, check again + const currentFrame = provider.frame(this.number); + if (currentFrame) { + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: currentFrame, + }); + return; + } - if (isNode) { - resolve('Dummy data'); - } else if (isBrowser) { - provider - .frame(this.number) - .then((frame) => { - if (frame === null) { - onServerRequest(); - const activeRequest = frameDataCache[this.jid].activeChunkRequest; - if (!provider.isChunkCached(start, stop)) { - if ( - !activeRequest || - (activeRequest && - activeRequest.completed && - activeRequest.chunkNumber !== chunkNumber) - ) { - if (activeRequest && activeRequest.rejectRequestAll) { - activeRequest.rejectRequestAll(); - } - frameDataCache[this.jid].activeChunkRequest = { - request: null, - chunkNumber, - start, - stop, - onDecodeAll, - rejectRequestAll, - completed: false, - callbacks: [ - { - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }, - ], - }; - makeActiveRequest(); - } else if (activeRequest.chunkNumber === chunkNumber) { - if (!activeRequest.onDecodeAll && !activeRequest.rejectRequestAll) { - activeRequest.onDecodeAll = onDecodeAll; - activeRequest.rejectRequestAll = rejectRequestAll; - } - activeRequest.callbacks.push({ - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }); - } else { - if (frameDataCache[this.jid].nextChunkRequest) { - const { callbacks } = frameDataCache[this.jid].nextChunkRequest; - for (const r of callbacks) { - r.reject(r.frameNumber); - } - } - frameDataCache[this.jid].nextChunkRequest = { - request: null, - chunkNumber, - start, - stop, - onDecodeAll, - rejectRequestAll, - completed: false, - callbacks: [ - { - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }, - ], - }; - } - } else { - activeRequest.callbacks.push({ - resolve: resolveWrapper, - reject, - frameNumber: this.number, - }); - provider.requestDecodeBlock(null, start, stop, onDecodeAll, rejectRequestAll); - } - } else { - if ( - this.number % chunkSize > chunkSize / 4 && - provider.decodedBlocksCacheSize > 1 && - this.decodeForward && - !provider.isNextChunkExists(this.number) - ) { - const nextChunkNumber = Math.floor(this.number / chunkSize) + 1; - if (nextChunkNumber * chunkSize < this.stopFrame) { - provider.setReadyToLoading(nextChunkNumber); - const nextStart = nextChunkNumber * chunkSize; - const nextStop = Math.min(this.stopFrame, (nextChunkNumber + 1) * chunkSize - 1); - if (!provider.isChunkCached(nextStart, nextStop)) { - if (!frameDataCache[this.jid].activeChunkRequest) { - frameDataCache[this.jid].activeChunkRequest = { - request: null, - chunkNumber: nextChunkNumber, - start: nextStart, - stop: nextStop, - onDecodeAll: null, - rejectRequestAll: null, - completed: false, - callbacks: [], - }; - makeActiveRequest(); - } - } else { - provider.requestDecodeBlock(null, nextStart, nextStop, null, null); - } - } + frameDataCache[this.jobID].activeChunkRequest = new Promise(( + resolveLoadAndDecode, + ) => { + let wasResolved = false; + serverProxy.frames.getData(this.jobID, chunkNumber).then((chunk: ArrayBuffer) => { + try { + provider + .requestDecodeBlock( + chunk, + chunkNumber * chunkSize, + Math.min(stopFrame, (chunkNumber + 1) * chunkSize - 1), + (_frame: number, bitmap: ImageBitmap | Blob) => { + if (decodeForward) { + // resolve immediately only if is not playing + return; + } + + if (frameDataCache[this.jobID].latestFrameDecodeRequest === requestId && + this.number === _frame + ) { + wasResolved = true; + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: bitmap, + }); + } + }, () => { + frameDataCache[this.jobID].activeChunkRequest = null; + resolveLoadAndDecode(); + const decodedFrame = provider.frame(this.number); + if (decodeForward) { + // resolve after decoding everything if playing + resolve({ + renderWidth: this.width, + renderHeight: this.height, + imageData: decodedFrame, + }); + } else if (!wasResolved) { + reject(this.number); + } + }, (error: Error | RequestOutdatedError) => { + frameDataCache[this.jobID].activeChunkRequest = null; + resolveLoadAndDecode(); + if (error instanceof RequestOutdatedError) { + reject(this.number); + } else { + reject(error); + } + }, + ); + } catch (error) { + reject(error); } - resolveWrapper(frame); - } - }) - .catch((exception) => { - if (exception instanceof Exception) { - reject(exception); - } else { - reject(new Exception(exception.message)); - } + }).catch((error) => { + reject(error); + resolveLoadAndDecode(error); + }); }); - } - }); -}; + }); + }); + }, + writable: false, +}); function getFrameMeta(jobID, frame): RawFramesMetaData['frames'][0] { const { meta, mode, startFrame } = frameDataCache[jobID]; @@ -403,284 +324,37 @@ function getFrameMeta(jobID, frame): RawFramesMetaData['frames'][0] { return frameMeta; } -class FrameBuffer { - constructor(size, chunkSize, stopFrame, jobID) { - this._size = size; - this._buffer = {}; - this._contextImage = {}; - this._requestedChunks = {}; - this._chunkSize = chunkSize; - this._stopFrame = stopFrame; - this._activeFillBufferRequest = false; - this._jobID = jobID; - } - - addContextImage(frame, data): void { - const promise = new Promise((resolve, reject) => { - data.then((resolvedData) => { - const meta = getFrameMeta(this._jobID, frame); - return cvatData - .decodeZip(resolvedData, 0, meta.related_files, cvatData.DimensionType.DIMENSION_2D); - }).then((decodedData) => { - this._contextImage[frame] = decodedData; - resolve(); - }).catch((error: Error) => { - if (error instanceof ServerError && (error as any).code === 404) { - this._contextImage[frame] = {}; - resolve(); - } else { - reject(error); - } - }); - }); - - this._contextImage[frame] = promise; - } - - isContextImageAvailable(frame): boolean { - return frame in this._contextImage; - } - - getContextImage(frame): Promise { - return new Promise((resolve) => { - if (frame in this._contextImage) { - if (this._contextImage[frame] instanceof Promise) { - this._contextImage[frame].then(() => { - resolve(this.getContextImage(frame)); - }); - } else { - resolve({ ...this._contextImage[frame] }); - } - } else { - resolve([]); - } - }); +export async function getContextImage(jobID: number, frame: number): Promise> { + if (!(jobID in frameDataCache)) { + throw new Error('Frame data was not initialized for this job. Try first requesting any frame.'); } - getFreeBufferSize() { - let requestedFrameCount = 0; - for (const chunk of Object.values(this._requestedChunks)) { - requestedFrameCount += chunk.requestedFrames.size; - } - - return this._size - Object.keys(this._buffer).length - requestedFrameCount; + const { related_files: relatedFiles } = frameDataCache[jobID] + .meta.frames[frame - frameDataCache[jobID].startFrame]; + if (relatedFiles === 0) { + return {}; } - requestOneChunkFrames(chunkIdx) { - return new Promise((resolve, reject) => { - this._requestedChunks[chunkIdx] = { - ...this._requestedChunks[chunkIdx], - resolve, - reject, - }; - for (const frame of this._requestedChunks[chunkIdx].requestedFrames.entries()) { - const requestedFrame = frame[1]; - const frameMeta = getFrameMeta(this._jobID, requestedFrame); - const frameData = new FrameData({ - ...frameMeta, - jobID: this._jobID, - frameNumber: requestedFrame, - startFrame: frameDataCache[this._jobID].startFrame, - stopFrame: frameDataCache[this._jobID].stopFrame, - decodeForward: false, - deleted: requestedFrame in frameDataCache[this._jobID].meta, - }); - - frameData - .data() - .then(() => { - if ( - !(chunkIdx in this._requestedChunks) || - !this._requestedChunks[chunkIdx].requestedFrames.has(requestedFrame) - ) { - reject(chunkIdx); - } else { - this._requestedChunks[chunkIdx].requestedFrames.delete(requestedFrame); - this._requestedChunks[chunkIdx].buffer[requestedFrame] = frameData; - if (this._requestedChunks[chunkIdx].requestedFrames.size === 0) { - const bufferedframes = Object.keys(this._requestedChunks[chunkIdx].buffer).map( - (f) => +f, - ); - this._requestedChunks[chunkIdx].resolve(new Set(bufferedframes)); - } - } - }) - .catch(() => { - reject(chunkIdx); - }); - } - }); - } - - fillBuffer(startFrame, frameStep = 1, count = null) { - const freeSize = this.getFreeBufferSize(); - const requestedFrameCount = count ? count * frameStep : freeSize * frameStep; - const stopFrame = Math.min(startFrame + requestedFrameCount, this._stopFrame + 1); - - for (let i = startFrame; i < stopFrame; i += frameStep) { - const chunkIdx = Math.floor(i / this._chunkSize); - if (!(chunkIdx in this._requestedChunks)) { - this._requestedChunks[chunkIdx] = { - requestedFrames: new Set(), - resolve: null, - reject: null, - buffer: {}, - }; - } - this._requestedChunks[chunkIdx].requestedFrames.add(i); - } - - let bufferedFrames = new Set(); - - // if we send one request to get frame 1 with filling the buffer - // then quicky send one more request to get frame 1 - // frame 1 will be already decoded and written to buffer - // the second request gets frame 1 from the buffer, removes it from there and returns - // after the first request finishes decoding it tries to get frame 1, but failed - // because frame 1 was already removed from the buffer by the second request - // to prevent this behavior we do not write decoded frames to buffer till the end of decoding all chunks - const buffersToBeCommited = []; - const commitBuffers = () => { - for (const buffer of buffersToBeCommited) { - this._buffer = { - ...this._buffer, - ...buffer, - }; - } - }; - - // Need to decode chunks in sequence - // eslint-disable-next-line no-async-promise-executor - return new Promise(async (resolve, reject) => { - for (const chunkIdx of Object.keys(this._requestedChunks)) { - try { - const chunkFrames = await this.requestOneChunkFrames(chunkIdx); - if (chunkIdx in this._requestedChunks) { - bufferedFrames = new Set([...bufferedFrames, ...chunkFrames]); - - buffersToBeCommited.push(this._requestedChunks[chunkIdx].buffer); - delete this._requestedChunks[chunkIdx]; - if (Object.keys(this._requestedChunks).length === 0) { - commitBuffers(); - resolve(bufferedFrames); - } - } else { - commitBuffers(); - reject(chunkIdx); - break; - } - } catch (error) { - commitBuffers(); - reject(error); - break; - } - } - }); + if (frameDataCache[jobID].activeContextRequest) { + await (frameDataCache[jobID].activeContextRequest); } - async makeFillRequest(start, step, count = null) { - if (!this._activeFillBufferRequest) { - this._activeFillBufferRequest = true; - try { - await this.fillBuffer(start, step, count); - this._activeFillBufferRequest = false; - } catch (error) { - if (typeof error === 'number' && error in this._requestedChunks) { - this._activeFillBufferRequest = false; - } - throw error; - } - } + if (frame in frameDataCache[jobID].contextCache) { + return frameDataCache[jobID].contextCache[frame]; } - async require(frameNumber: number, jobID: number, fillBuffer: boolean, frameStep: number): FrameData { - for (const frame in this._buffer) { - if (+frame < frameNumber || +frame >= frameNumber + this._size * frameStep) { - delete this._buffer[frame]; - } - } - - this._required = frameNumber; - const frameMeta = getFrameMeta(jobID, frameNumber); - let frame = new FrameData({ - ...frameMeta, - jobID, - frameNumber, - startFrame: frameDataCache[jobID].startFrame, - stopFrame: frameDataCache[jobID].stopFrame, - decodeForward: !fillBuffer, - deleted: frameNumber in frameDataCache[jobID].meta.deleted_frames, + const promise = serverProxy.frames.getImageContext(jobID, frame) + .then((encodedImages) => decodeContextImages(encodedImages, 0, relatedFiles)); + frameDataCache[jobID].activeContextRequest = new Promise((resolve) => { + promise.finally(() => { + resolve(); + frameDataCache[jobID].activeContextRequest = null; }); - - if (frameNumber in this._buffer) { - frame = this._buffer[frameNumber]; - delete this._buffer[frameNumber]; - const cachedFrames = this.cachedFrames(); - if ( - fillBuffer && - !this._activeFillBufferRequest && - this._size > this._chunkSize && - cachedFrames.length < (this._size * 3) / 4 - ) { - const maxFrame = cachedFrames ? Math.max(...cachedFrames) : frameNumber; - if (maxFrame < this._stopFrame) { - this.makeFillRequest(maxFrame + 1, frameStep).catch((e) => { - if (e !== 'not needed') { - throw e; - } - }); - } - } - } else if (fillBuffer) { - this.clear(); - await this.makeFillRequest(frameNumber, frameStep, fillBuffer ? null : 1); - frame = this._buffer[frameNumber]; - } else { - this.clear(); - } - - return frame; - } - - clear() { - for (const chunkIdx in this._requestedChunks) { - if ( - Object.prototype.hasOwnProperty.call(this._requestedChunks, chunkIdx) && - this._requestedChunks[chunkIdx].reject - ) { - this._requestedChunks[chunkIdx].reject('not needed'); - } - } - this._activeFillBufferRequest = false; - this._requestedChunks = {}; - this._buffer = {}; - } - - cachedFrames() { - return Object.keys(this._buffer).map((f) => +f); - } -} - -async function getImageContext(jobID, frame) { - return new Promise((resolve, reject) => { - serverProxy.frames - .getImageContext(jobID, frame) - .then((result) => { - resolve(result); - }) - .catch((error) => { - reject(error); - }); }); -} -export async function getContextImage(jobID, frame) { - if (frameDataCache[jobID].frameBuffer.isContextImageAvailable(frame)) { - return frameDataCache[jobID].frameBuffer.getContextImage(frame); - } - const response = getImageContext(jobID, frame); - await frameDataCache[jobID].frameBuffer.addContextImage(frame, response); - return frameDataCache[jobID].frameBuffer.getContextImage(frame); + const images = await promise; + frameDataCache[jobID].contextCache[frame] = images; + return images; } export function decodePreview(preview: Blob): Promise { @@ -707,60 +381,68 @@ export async function getFrame( isPlaying: boolean, step: number, dimension: DimensionType, -) { +): Promise { if (!(jobID in frameDataCache)) { - const blockType = chunkType === 'video' ? cvatData.BlockType.MP4VIDEO : cvatData.BlockType.ARCHIVE; + const blockType = chunkType === 'video' ? BlockType.MP4VIDEO : BlockType.ARCHIVE; const meta = await serverProxy.frames.getMeta('job', jobID); - meta.deleted_frames = Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])); - const mean = meta.frames.reduce((a, b) => a + b.width * b.height, 0) / meta.frames.length; + const updatedMeta = { + ...meta, + deleted_frames: Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])), + }; + const mean = updatedMeta.frames.reduce((a, b) => a + b.width * b.height, 0) / updatedMeta.frames.length; const stdDev = Math.sqrt( - meta.frames.map((x) => (x.width * x.height - mean) ** 2).reduce((a, b) => a + b) / - meta.frames.length, + updatedMeta.frames.map((x) => (x.width * x.height - mean) ** 2).reduce((a, b) => a + b) / + updatedMeta.frames.length, ); // limit of decoded frames cache by 2GB - const decodedBlocksCacheSize = Math.floor(2147483648 / (mean + stdDev) / 4 / chunkSize) || 1; - + const decodedBlocksCacheSize = Math.floor(2147483648 / ((mean + stdDev) * 4 * chunkSize)) || 1; frameDataCache[jobID] = { - meta, + meta: updatedMeta, chunkSize, mode, startFrame, stopFrame, - provider: new cvatData.FrameProvider( + decodeForward: isPlaying, + forwardStep: step, + provider: new FrameDecoder( blockType, chunkSize, Math.max(decodedBlocksCacheSize, 9), - decodedBlocksCacheSize, - 1, dimension, ), - frameBuffer: new FrameBuffer( - Math.min(180, decodedBlocksCacheSize * chunkSize), - chunkSize, - stopFrame, - jobID, - ), decodedBlocksCacheSize, activeChunkRequest: null, - nextChunkRequest: null, + activeContextRequest: null, + latestFrameDecodeRequest: null, + contextCache: {}, }; - - const frameMeta = getFrameMeta(jobID, frame); - frameDataCache[jobID].provider.setRenderSize(frameMeta.width, frameMeta.height); } - return frameDataCache[jobID].frameBuffer.require(frame, jobID, isPlaying, step); + const frameMeta = getFrameMeta(jobID, frame); + frameDataCache[jobID].provider.setRenderSize(frameMeta.width, frameMeta.height); + frameDataCache[jobID].decodeForward = isPlaying; + frameDataCache[jobID].forwardStep = step; + + return new FrameData({ + width: frameMeta.width, + height: frameMeta.height, + name: frameMeta.name, + related_files: frameMeta.related_files, + frameNumber: frame, + deleted: frame in frameDataCache[jobID].meta.deleted_frames, + jobID, + }); } -export async function getDeletedFrames(instanceType, id) { +export async function getDeletedFrames(instanceType: 'job' | 'task', id) { if (instanceType === 'job') { const { meta } = frameDataCache[id]; return meta.deleted_frames; } if (instanceType === 'task') { - const meta = await serverProxy.frames.getMeta('job', id); + const meta = await serverProxy.frames.getMeta('task', id); meta.deleted_frames = Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])); return meta; } @@ -768,19 +450,19 @@ export async function getDeletedFrames(instanceType, id) { throw new Exception(`getDeletedFrames is not implemented for ${instanceType}`); } -export function deleteFrame(jobID, frame) { +export function deleteFrame(jobID: number, frame: number): void { const { meta } = frameDataCache[jobID]; meta.deleted_frames[frame] = true; } -export function restoreFrame(jobID, frame) { +export function restoreFrame(jobID: number, frame: number): void { const { meta } = frameDataCache[jobID]; if (frame in meta.deleted_frames) { delete meta.deleted_frames[frame]; } } -export async function patchMeta(jobID) { +export async function patchMeta(jobID: number): Promise { const { meta } = frameDataCache[jobID]; const newMeta = await serverProxy.frames.saveMeta('job', jobID, { deleted_frames: Object.keys(meta.deleted_frames), @@ -799,7 +481,9 @@ export async function patchMeta(jobID) { frameDataCache[jobID].meta.deleted_frames = prevDeletedFrames; } -export async function findFrame(jobID, frameFrom, frameTo, filters) { +export async function findFrame( + jobID: number, frameFrom: number, frameTo: number, filters: { offset?: number, notDeleted: boolean }, +): Promise { const offset = filters.offset || 1; let meta; if (!frameDataCache[jobID]) { @@ -836,23 +520,16 @@ export async function findFrame(jobID, frameFrom, frameTo, filters) { return lastUndeletedFrame; } -export function getRanges(jobID) { +export function getRanges(jobID): Array { if (!(jobID in frameDataCache)) { - return { - decoded: [], - buffered: [], - }; + return []; } - return { - decoded: frameDataCache[jobID].provider.cachedFrames, - buffered: frameDataCache[jobID].frameBuffer.cachedFrames(), - }; + return frameDataCache[jobID].provider.cachedFrames; } -export function clear(jobID) { +export function clear(jobID: number): void { if (jobID in frameDataCache) { - frameDataCache[jobID].frameBuffer.clear(); delete frameDataCache[jobID]; } } diff --git a/cvat-data/src/ts/cvat-data.ts b/cvat-data/src/ts/cvat-data.ts index 78d0bcf558b..f6ace8cff88 100644 --- a/cvat-data/src/ts/cvat-data.ts +++ b/cvat-data/src/ts/cvat-data.ts @@ -8,6 +8,8 @@ import { MP4Reader, Bytestream } from './3rdparty/mp4'; import ZipDecoder from './unzip_imgs.worker'; import H264Decoder from './3rdparty/Decoder.worker'; +export class RequestOutdatedError extends Error {} + export enum BlockType { MP4VIDEO = 'mp4video', ARCHIVE = 'archive', @@ -18,25 +20,25 @@ export enum DimensionType { DIMENSION_2D = '2d', } -export function decodeZip( - block: any, start: number, end: number, dimension: any, +const decodeZipWorker = new (ZipDecoder as any)() as any as Worker; +export function decodeContextImages( + block: any, start: number, end: number, ): Promise> { return new Promise((resolve, reject) => { - decodeZip.mutex.acquire().then((release) => { - const worker = new ZipDecoder(); + decodeContextImages.mutex.acquire().then((release) => { const result: Record = {}; let decoded = 0; - worker.onerror = (e: ErrorEvent) => { + decodeZipWorker.onerror = (e: ErrorEvent) => { release(); - worker.terminate(); reject(new Error(`Archive can not be decoded. ${e.message}`)); }; - worker.onmessage = async (event) => { + decodeZipWorker.onmessage = async (event) => { const { error, fileName } = event.data; if (error) { - worker.onerror(new ErrorEvent('error', { message: error.toString() })); + decodeZipWorker.onerror(new ErrorEvent('error', { message: error.toString() })); + return; } const { data } = event.data; @@ -45,174 +47,143 @@ export function decodeZip( if (decoded === end) { release(); - worker.terminate(); resolve(result); } }; - worker.postMessage({ + decodeZipWorker.postMessage({ block, start, end, - dimension, + dimension: DimensionType.DIMENSION_2D, dimension2D: DimensionType.DIMENSION_2D, }); }); }); } -decodeZip.mutex = new Mutex(); +decodeContextImages.mutex = new Mutex(); interface BlockToDecode { start: number; end: number; block: ArrayBuffer; - resolveCallback: (frame: number) => void; - rejectCallback: (e: ErrorEvent) => void; + onDecodeAll(): void; + onDecode(frame: number, bitmap: ImageBitmap | Blob): void; + onReject(e: Error): void; } -export class FrameProvider { - private blocksRanges: string[]; - private blockSize: number; +export class FrameDecoder { private blockType: BlockType; - + private chunkSize: number; /* ImageBitmap when decode zip or video chunks Blob when 3D dimension null when not decoded yet */ - private frames: Record; - private requestedBlockToDecode: null | BlockToDecode; - private blocksAreBeingDecoded: Record; - private promisedFrames: Record void; - reject: () => void; - }>; - private currentDecodingThreads: number; + private decodedChunks: Record>; + private chunkIsBeingDecoded: BlockToDecode | null; + private requestedChunkToDecode: BlockToDecode | null; private currentFrame: number; private mutex: Mutex; - private dimension: DimensionType; - private workerThreadsLimit: number; - private cachedEncodedBlocksLimit: number; - private cachedDecodedBlocksLimit: number; - + private cachedChunksLimit: number; // used for video chunks to get correct side after decoding private renderWidth: number; private renderHeight: number; + private zipWorker: Worker; + + private promisedFrames: Record void; + reject: () => void; + }>; constructor( blockType: BlockType, - blockSize: number, + chunkSize: number, cachedBlockCount: number, - decodedBlocksCacheSize = 5, - maxWorkerThreadCount = 2, dimension: DimensionType = DimensionType.DIMENSION_2D, ) { this.mutex = new Mutex(); - this.blocksRanges = []; - this.frames = {}; this.promisedFrames = {}; - this.currentDecodingThreads = 0; this.currentFrame = -1; - this.cachedEncodedBlocksLimit = Math.max(1, cachedBlockCount); // number of stored blocks - this.cachedDecodedBlocksLimit = decodedBlocksCacheSize; - this.workerThreadsLimit = maxWorkerThreadCount; + this.cachedChunksLimit = Math.max(1, cachedBlockCount); this.dimension = dimension; this.renderWidth = 1920; this.renderHeight = 1080; - this.blockSize = blockSize; + this.chunkSize = chunkSize; this.blockType = blockType; - // todo: sort out with logic of blocks - this._blocks = {}; - this.requestedBlockToDecode = null; - this.blocksAreBeingDecoded = {}; + this.decodedChunks = {}; + this.requestedChunkToDecode = null; + this.chunkIsBeingDecoded = null; - setTimeout(this._checkDecodeRequests.bind(this), 100); - } - - _checkDecodeRequests(): void { - if (this.requestedBlockToDecode !== null && this.currentDecodingThreads < this.workerThreadsLimit) { - this.startDecode().then(() => { - setTimeout(this._checkDecodeRequests.bind(this), 100); - }); - } else { - setTimeout(this._checkDecodeRequests.bind(this), 100); - } + this.zipWorker = new (ZipDecoder as any)() as any as Worker; } - isChunkCached(start: number, end: number): boolean { - // todo: always returns false because this.blocksRanges is Array, not dictionary - // but if try to correct other errors happens, need to debug.. - return `${start}:${end}` in this.blocksRanges; + isChunkCached(chunkNumber: number): boolean { + return chunkNumber in this.decodedChunks; } - /* This method removes extra data from a cache when memory overflow */ async _cleanup(): Promise { - if (this.blocksRanges.length > this.cachedEncodedBlocksLimit) { - const shifted = this.blocksRanges.shift(); // get the oldest block - const [start, end] = shifted.split(':').map((el) => +el); - delete this._blocks[Math.floor(start / this.blockSize)]; - for (let i = start; i <= end; i++) { - delete this.frames[i]; - } - } - - // delete frames whose are not in areas of current frame - const distance = Math.floor(this.cachedDecodedBlocksLimit / 2); - for (let i = 0; i < this.blocksRanges.length; i++) { - const [start, end] = this.blocksRanges[i].split(':').map((el) => +el); - if ( - end < this.currentFrame - distance * this.blockSize || - start > this.currentFrame + distance * this.blockSize - ) { - for (let j = start; j <= end; j++) { - delete this.frames[j]; - } - } - } + // removes chunks around the current one exceeding the limit + const currentChunkNumber = Math.floor(this.currentFrame / this.chunkSize); + const distance = Math.floor((this.cachedChunksLimit - 1) / 2); + const chunks = Object.keys(this.decodedChunks).map((chunk: string) => +chunk); + const prevChunks = chunks.filter((chunk) => chunk < currentChunkNumber); + const nextChunks = chunks.filter((chunk) => chunk > currentChunkNumber); + const prevChunksToRemove = prevChunks.slice(0, prevChunks.length - distance); + const nextChunksToRemove = nextChunks.slice(distance, nextChunks.length); + + [].concat(...prevChunksToRemove, ...nextChunksToRemove).forEach((chunk: number) => { + // TODO: think how to improve if one of sides does not have decoded chunks + // probably we need to know direction here + delete this.decodedChunks[chunk]; + }); } - async requestDecodeBlock( + requestDecodeBlock( block: ArrayBuffer, start: number, end: number, - resolveCallback: () => void, - rejectCallback: () => void, - ): Promise { - const release = await this.mutex.acquire(); - try { - if (this.requestedBlockToDecode !== null) { - if (start === this.requestedBlockToDecode.start && end === this.requestedBlockToDecode.end) { - // only rewrite callbacks if the same block was requested again - this.requestedBlockToDecode.resolveCallback = resolveCallback; - this.requestedBlockToDecode.rejectCallback = rejectCallback; - - // todo: should we reject the previous request here? - } else if (this.requestedBlockToDecode.rejectCallback) { - // if another block requested, the previous request should be rejected - this.requestedBlockToDecode.rejectCallback(); - } + onDecode: (frame: number, bitmap: ImageBitmap | Blob) => void, + onDecodeAll: () => void, + onReject: (e: Error) => void, + ): void { + if (this.requestedChunkToDecode !== null) { + // a chunk was already requested to be decoded, but decoding didn't start yet + if (start === this.requestedChunkToDecode.start && end === this.requestedChunkToDecode.end) { + // it was the same chunk + this.requestedChunkToDecode.onReject(new RequestOutdatedError()); + + this.requestedChunkToDecode.onDecode = onDecode; + this.requestedChunkToDecode.onReject = onReject; + } else if (this.requestedChunkToDecode.onReject) { + // it was other chunk + this.requestedChunkToDecode.onReject(new RequestOutdatedError()); } + } else if (this.chunkIsBeingDecoded === null || this.chunkIsBeingDecoded.start !== start) { + // everything was decoded or decoding other chunk is in process + this.requestedChunkToDecode = { + block, + start, + end, + onDecode, + onDecodeAll, + onReject, + }; + } else { + // the same chunk is being decoded right now + // reject previous decoding request + this.chunkIsBeingDecoded.onReject(new RequestOutdatedError()); - if (!(`${start}:${end}` in this.blocksAreBeingDecoded)) { - this.requestedBlockToDecode = { - block: block || this._blocks[Math.floor(start / this.blockSize)], - start, - end, - resolveCallback, - rejectCallback, - }; - } else { - this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback = rejectCallback; - this.blocksAreBeingDecoded[`${start}:${end}`].resolveCallback = resolveCallback; - } - } finally { - release(); + this.chunkIsBeingDecoded.onReject = onReject; + this.chunkIsBeingDecoded.onDecode = onDecode; } + + this.startDecode(); } setRenderSize(width: number, height: number): void { @@ -220,50 +191,37 @@ export class FrameProvider { this.renderHeight = height; } - /* Method returns frame from collection. Else method returns null */ - async frame(frameNumber: number): Promise { + frame(frameNumber: number): ImageBitmap | Blob | null { + const chunkNumber = Math.floor(frameNumber / this.chunkSize); this.currentFrame = frameNumber; - return new Promise((resolve, reject) => { - if (frameNumber in this.frames) { - if (this.frames[frameNumber] !== null) { - resolve(this.frames[frameNumber]); - } else { - this.promisedFrames[frameNumber] = { resolve, reject }; - } - } else { - resolve(null); - } - }); - } - - isNextChunkExists(frameNumber: number): boolean { - const nextChunkNum = Math.floor(frameNumber / this.blockSize) + 1; - return nextChunkNum in this._blocks; - } + if (chunkNumber in this.decodedChunks) { + return this.decodedChunks[chunkNumber][frameNumber]; + } - setReadyToLoading(chunkNumber: number): void { - this._blocks[chunkNumber] = 'loading'; + return null; } async startDecode(): Promise { + const blockToDecode = { ...this.requestedChunkToDecode }; const release = await this.mutex.acquire(); try { - const { start, end, block } = this.requestedBlockToDecode; - - this.blocksRanges.push(`${start}:${end}`); - this.blocksAreBeingDecoded[`${start}:${end}`] = this.requestedBlockToDecode; - this.requestedBlockToDecode = null; - this._blocks[Math.floor((start + 1) / this.blockSize)] = block; - - for (let i = start; i <= end; i++) { - this.frames[i] = null; + const { start, end, block } = this.requestedChunkToDecode; + if (start !== blockToDecode.start) { + // request is not relevant, another block was already requested + // it happens when A is being decoded, B comes and wait for mutex, C comes and wait for mutex + // B is not necessary anymore, because C already was requested + blockToDecode.onReject(new RequestOutdatedError()); + throw new RequestOutdatedError(); } this._cleanup(); - this.currentDecodingThreads++; + const chunkNumber = Math.floor(start / this.chunkSize); + const decodedFrames: Record = {}; + this.chunkIsBeingDecoded = this.requestedChunkToDecode; + this.requestedChunkToDecode = null; if (this.blockType === BlockType.MP4VIDEO) { - const worker = new H264Decoder(); + const worker = new H264Decoder() as any as Worker; let index = start; worker.onmessage = (e) => { @@ -281,31 +239,30 @@ export class FrameProvider { const array = new Uint8ClampedArray(e.data.buf.slice(0, width * height * 4)); createImageBitmap(new ImageData(array, width)).then((bitmap) => { - this.frames[keptIndex] = bitmap; - const { resolveCallback } = this.blocksAreBeingDecoded[`${start}:${end}`]; - if (resolveCallback) { - resolveCallback(keptIndex); - } + decodedFrames[keptIndex] = bitmap; + this.chunkIsBeingDecoded.onDecode(keptIndex, decodedFrames[keptIndex]); if (keptIndex in this.promisedFrames) { const { resolve } = this.promisedFrames[keptIndex]; delete this.promisedFrames[keptIndex]; - resolve(this.frames[keptIndex]); + resolve(decodedFrames[keptIndex]); } if (keptIndex === end) { + this.decodedChunks[chunkNumber] = decodedFrames; + this.chunkIsBeingDecoded.onDecodeAll(); + this.chunkIsBeingDecoded = null; worker.terminate(); - this.currentDecodingThreads--; - delete this.blocksAreBeingDecoded[`${start}:${end}`]; + release(); } }); index++; }; - worker.onerror = (e: ErrorEvent) => { + worker.onerror = () => { + release(); worker.terminate(); - this.currentDecodingThreads--; for (let i = index; i <= end; i++) { // reject all the following frames @@ -316,11 +273,8 @@ export class FrameProvider { } } - if (this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback) { - this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback(e); - } - - delete this.blocksAreBeingDecoded[`${start}:${end}`]; + this.chunkIsBeingDecoded.onReject(new Error('Error occured during decode')); + this.chunkIsBeingDecoded = null; }; worker.postMessage({ @@ -339,58 +293,58 @@ export class FrameProvider { const sps = avc.sps[0]; const pps = avc.pps[0]; - /* Decode Sequence & Picture Parameter Sets */ worker.postMessage({ buf: sps, offset: 0, length: sps.length }); worker.postMessage({ buf: pps, offset: 0, length: pps.length }); - /* Decode Pictures */ for (let sample = 0; sample < video.getSampleCount(); sample++) { video.getSampleNALUnits(sample).forEach((nal) => { worker.postMessage({ buf: nal, offset: 0, length: nal.length }); }); } } else { - const worker = new ZipDecoder(); let index = start; - worker.onmessage = async (event) => { - this.frames[event.data.index] = event.data.data; - - const { resolveCallback } = this.blocksAreBeingDecoded[`${start}:${end}`]; - if (resolveCallback) { - resolveCallback(event.data.index); + this.zipWorker.onmessage = async (event) => { + if (event.data.error) { + this.zipWorker.onerror(new ErrorEvent('error', { message: event.data.error.toString() })); + return; } + decodedFrames[event.data.index] = event.data.data as ImageBitmap | Blob; + this.chunkIsBeingDecoded.onDecode(event.data.index, decodedFrames[event.data.index]); + if (event.data.index in this.promisedFrames) { const { resolve } = this.promisedFrames[event.data.index]; delete this.promisedFrames[event.data.index]; - resolve(this.frames[event.data.index]); + resolve(decodedFrames[event.data.index]); } if (index === end) { - worker.terminate(); - this.currentDecodingThreads--; - delete this.blocksAreBeingDecoded[`${start}:${end}`]; + this.decodedChunks[chunkNumber] = decodedFrames; + this.chunkIsBeingDecoded.onDecodeAll(); + this.chunkIsBeingDecoded = null; + release(); } index++; }; - worker.onerror = (e: ErrorEvent) => { - for (let i = start; i <= end; i++) { + this.zipWorker.onerror = () => { + release(); + + for (let i = index; i <= end; i++) { + // reject all the following frames if (i in this.promisedFrames) { const { reject } = this.promisedFrames[i]; delete this.promisedFrames[i]; reject(); } } - if (this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback) { - this.blocksAreBeingDecoded[`${start}:${end}`].rejectCallback(e); - } - this.currentDecodingThreads--; - worker.terminate(); + + this.chunkIsBeingDecoded.onReject(new Error('Error occured during decode')); + this.chunkIsBeingDecoded = null; }; - worker.postMessage({ + this.zipWorker.postMessage({ block, start, end, @@ -398,20 +352,19 @@ export class FrameProvider { dimension2D: DimensionType.DIMENSION_2D, }); } - } finally { + } catch (error) { + this.chunkIsBeingDecoded = null; release(); } } - get decodedBlocksCacheSize(): number { - return this.cachedDecodedBlocksLimit; - } - - /* - Method returns a list of cached ranges - Is an array of strings like "start:end" - */ get cachedFrames(): string[] { - return [...this.blocksRanges].sort((a, b) => +a.split(':')[0] - +b.split(':')[0]); + const chunks = Object.keys(this.decodedChunks).map((chunkNumber: string) => +chunkNumber).sort((a, b) => a - b); + return chunks.map((chunk) => { + const frames = Object.keys(this.decodedChunks[chunk]).map((frame) => +frame); + const min = Math.min(...frames); + const max = Math.max(...frames); + return `${min}:${max}`; + }); } } diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 8b7a899b9ec..e00e1a87156 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -624,6 +624,7 @@ export function changeFrameAsync( relatedFiles: currentState.annotation.player.frame.relatedFiles, delay: currentState.annotation.player.frame.delay, changeTime: currentState.annotation.player.frame.changeTime, + ranges: currentState.annotation.player.ranges, states: currentState.annotation.annotations.states, minZ: currentState.annotation.annotations.zLayer.min, maxZ: currentState.annotation.annotations.zLayer.max, @@ -645,6 +646,8 @@ export function changeFrameAsync( } const data = await job.frames.get(toFrame, fillBuffer, frameStep); + const ranges = await job.frames.ranges(); + console.log(ranges); const states = await job.annotations.get(toFrame, showAllInterpolationTracks, filters); if (!isAbleToChangeFrame() || statisticsVisible || propagateVisible) { @@ -695,6 +698,7 @@ export function changeFrameAsync( curZ: maxZ, changeTime: currentTime + delay, delay, + ranges, }, }); } catch (error) { diff --git a/cvat-ui/src/components/annotation-page/styles.scss b/cvat-ui/src/components/annotation-page/styles.scss index 7d75f57d8f7..8e39fd5a64b 100644 --- a/cvat-ui/src/components/annotation-page/styles.scss +++ b/cvat-ui/src/components/annotation-page/styles.scss @@ -126,15 +126,35 @@ } } -.cvat-player-slider { +.cvat-player-slider.ant-slider { width: 350px; margin: 0; + margin-top: $grid-unit-size * -0.5; + > .ant-slider-handle { + z-index: 100; + margin-top: -3.5px; + } + > .ant-slider-track { + background: none; + } > .ant-slider-rail { + height: $grid-unit-size; background-color: $player-slider-color; } } +.cvat-player-slider-progress { + width: 350px; + position: absolute; + top: 0; + pointer-events: none; + + > rect { + fill: #ff4136; + } +} + .cvat-player-filename-wrapper { max-width: $grid-unit-size * 30; max-height: $grid-unit-size * 3; diff --git a/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx b/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx index e95d15cf33d..f39f4df2162 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/player-navigation.tsx @@ -21,6 +21,7 @@ interface Props { startFrame: number; stopFrame: number; playing: boolean; + ranges: string; frameNumber: number; frameFilename: string; frameDeleted: boolean; @@ -47,6 +48,7 @@ function PlayerNavigation(props: Props): JSX.Element { deleteFrameShortcut, focusFrameInputShortcut, inputFrameRef, + ranges, onSliderChange, onInputChange, onURLIconClick, @@ -105,6 +107,17 @@ function PlayerNavigation(props: Props): JSX.Element { value={frameNumber || 0} onChange={onSliderChange} /> + + {ranges.split(';').map((range) => { + let [start, end] = range.split(':').map((num) => +num); + start = Math.max(0, start - 1); + const totalSegments = stopFrame - startFrame; + const segmentWidth = 1000 / totalSegments; + const width = Math.max((end - start), 1) * segmentWidth; + const offset = (Math.max((start - startFrame), 0) / totalSegments) * 1000; + return (); + })} + diff --git a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx index ee800ce6861..7c88063bf21 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx @@ -69,6 +69,7 @@ interface Props { onRestoreFrame(): void; switchNavigationBlocked(blocked: boolean): void; jobInstance: any; + ranges: string; } export default function AnnotationTopBarComponent(props: Props): JSX.Element { @@ -77,6 +78,7 @@ export default function AnnotationTopBarComponent(props: Props): JSX.Element { undoAction, redoAction, playing, + ranges, frameNumber, frameFilename, frameDeleted, @@ -168,6 +170,7 @@ export default function AnnotationTopBarComponent(props: Props): JSX.Element { startFrame={startFrame} stopFrame={stopFrame} playing={playing} + ranges={ranges} frameNumber={frameNumber} frameFilename={frameFilename} frameDeleted={frameDeleted} diff --git a/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx b/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx index f53c5f61cdb..e2ebd27e1fb 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/top-bar.tsx @@ -65,6 +65,7 @@ interface StateToProps { normalizedKeyMap: Record; canvasInstance: Canvas | Canvas3d; forceExit: boolean; + ranges: string; activeControl: ActiveControl; } @@ -91,6 +92,7 @@ function mapStateToProps(state: CombinedState): StateToProps { annotation: { player: { playing, + ranges, frame: { data: { deleted: frameIsDeleted }, filename: frameFilename, @@ -142,6 +144,7 @@ function mapStateToProps(state: CombinedState): StateToProps { canvasInstance, forceExit, activeControl, + ranges: ranges.join(';'), }; } @@ -638,6 +641,7 @@ class AnnotationTopBarContainer extends React.PureComponent { workspace, canvasIsReady, keyMap, + ranges, normalizedKeyMap, activeControl, searchAnnotations, @@ -766,6 +770,7 @@ class AnnotationTopBarContainer extends React.PureComponent { workspace={workspace} playing={playing} saving={saving} + ranges={ranges} startFrame={startFrame} stopFrame={stopFrame} frameNumber={frameNumber} diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index 71524879f8b..15fc05c617f 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -72,6 +72,7 @@ const defaultState: AnnotationState = { delay: 0, changeTime: null, }, + ranges: [], playing: false, frameAngles: [], navigationBlocked: false, @@ -275,12 +276,14 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { maxZ, curZ, delay, + ranges, changeTime, } = action.payload; return { ...state, player: { ...state.player, + ranges, frame: { data, filename, diff --git a/cvat-ui/src/reducers/index.ts b/cvat-ui/src/reducers/index.ts index 89d4bb14f3f..69619e4e9d6 100644 --- a/cvat-ui/src/reducers/index.ts +++ b/cvat-ui/src/reducers/index.ts @@ -692,6 +692,7 @@ export interface AnnotationState { delay: number; changeTime: number | null; }; + ranges: string[]; navigationBlocked: boolean; playing: boolean; frameAngles: number[];