From 7552c906319ccd9e3464c270580b930d63aec8ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 22:03:13 +0300 Subject: [PATCH 1/9] Bump gitpython from 3.1.3 to 3.1.8 in /cvat/requirements (#2143) Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.1.3 to 3.1.8. - [Release notes](https://github.com/gitpython-developers/GitPython/releases) - [Changelog](https://github.com/gitpython-developers/GitPython/blob/master/CHANGES) - [Commits](https://github.com/gitpython-developers/GitPython/compare/3.1.3...3.1.8) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 1cd368a6433..8ee8172638a 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -24,7 +24,7 @@ dj-pagination==2.5.0 python-logstash==0.4.6 django-revproxy==0.10.0 rules==2.2 -GitPython==3.1.3 +GitPython==3.1.8 coreapi==2.3.3 django-filter==2.3.0 Markdown==3.2.2 From 6f635436a9f66019348af97a36fbd0b422190463 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 22:03:50 +0300 Subject: [PATCH 2/9] Bump isort from 4.3.21 to 5.5.1 in /cvat/requirements (#2142) Bumps [isort](https://github.com/pycqa/isort) from 4.3.21 to 5.5.1. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/4.3.21...5.5.1) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index 990bf33600d..4b43f3e70a7 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -1,6 +1,6 @@ -r base.txt astroid==2.4.2 -isort==4.3.21 +isort==5.5.1 lazy-object-proxy==1.5.1 mccabe==0.6.1 pylint==2.6.0 From 67de3f86dd7df6887e0bce2b5db20f99086d17d9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 22:37:33 +0300 Subject: [PATCH 3/9] Bump diskcache from 4.1.0 to 5.0.2 in /cvat/requirements (#2145) Bumps [diskcache](https://github.com/grantjenks/python-diskcache) from 4.1.0 to 5.0.2. - [Release notes](https://github.com/grantjenks/python-diskcache/releases) - [Commits](https://github.com/grantjenks/python-diskcache/compare/v4.1.0...v5.0.2) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 8ee8172638a..d44579482e1 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -44,4 +44,4 @@ tensorflow==2.2.0 # Optional requirement of Datumaro # The package is used by pyunpack as a command line tool to support multiple # archives. Don't use as a python module because it has GPL license. patool==1.12 -diskcache==4.1.0 \ No newline at end of file +diskcache==5.0.2 \ No newline at end of file From 21e67c20474556c14fdd9d4d24089658fb816483 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 22:38:07 +0300 Subject: [PATCH 4/9] Bump django-extensions from 3.0.6 to 3.0.8 in /cvat/requirements (#2144) Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.0.6 to 3.0.8. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.0.6...3.0.8) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index 4b43f3e70a7..b6ac4f7260d 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -8,6 +8,6 @@ pylint-django==2.3.0 pylint-plugin-utils==0.6 rope==0.17.0 wrapt==1.12.1 -django-extensions==3.0.6 +django-extensions==3.0.8 Werkzeug==1.0.1 snakeviz==2.1.0 From 416df8980afa18b3a686e63405f5cc9a946c4e95 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Mon, 7 Sep 2020 22:38:38 +0300 Subject: [PATCH 5/9] Don't allow lambda manager to return objects with a label which doesn't exist in the task. (#2131) --- cvat/apps/lambda_manager/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index d307ca51231..52c1de419ce 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -134,6 +134,16 @@ def invoke(self, db_task, data): }) quality = data.get("quality") mapping = data.get("mapping") + mapping_by_default = {db_label.name:db_label.name + for db_label in db_task.label_set.all()} + if not mapping: + # use mapping by default to avoid labels in mapping which + # don't exist in the task + mapping = mapping_by_default + else: + # filter labels in mapping which don't exist in the task + mapping = {k:v for k,v in mapping.items() if v in mapping_by_default} + if self.kind == LambdaType.DETECTOR: payload.update({ "image": self._get_image(db_task, data["frame"], quality) From 0efc11d2b03278e9a0e4cc1f5759a480be930df1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2020 06:50:42 +0300 Subject: [PATCH 6/9] Bump psycopg2-binary from 2.8.5 to 2.8.6 in /cvat/requirements (#2146) Bumps [psycopg2-binary](https://github.com/psycopg/psycopg2) from 2.8.5 to 2.8.6. - [Release notes](https://github.com/psycopg/psycopg2/releases) - [Changelog](https://github.com/psycopg/psycopg2/blob/master/NEWS) - [Commits](https://github.com/psycopg/psycopg2/commits) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/production.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index 36e64e29737..cfc5a3509ca 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -1,3 +1,3 @@ -r base.txt -psycopg2-binary==2.8.5 +psycopg2-binary==2.8.6 mod-wsgi==4.7.1 \ No newline at end of file From 0933ee236263013f04082dfa600d577fe649bbbf Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 8 Sep 2020 22:44:13 +0300 Subject: [PATCH 7/9] Fix CVAT format import for frame stepped tasks (#2151) * Fix cvat format import with frame step * update changelog --- CHANGELOG.md | 1 + cvat/apps/dataset_manager/formats/cvat.py | 4 +++- cvat/apps/engine/tests/_test_rest_api.py | 13 ++++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 844cc02fe6f..c7f5e7a2548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed multiple errors which arises when polygon is of length 5 or less () +- Fixed CVAT format import for frame stepped tasks () ### Security - diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index ac5823b6f60..3c349947b76 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -442,7 +442,9 @@ def load(file_object, annotations): ) elif el.tag == 'image': image_is_opened = True - frame_id = match_dm_item(DatasetItem(id=el.attrib['id'], image=el.attrib['name']), annotations) + frame_id = annotations.abs_frame_id(match_dm_item( + DatasetItem(id=el.attrib['id'], image=el.attrib['name']), + annotations)) elif el.tag in supported_shapes and (track is not None or image_is_opened): attributes = [] shape = { diff --git a/cvat/apps/engine/tests/_test_rest_api.py b/cvat/apps/engine/tests/_test_rest_api.py index 1e773b2f3f0..374a4cd2e21 100644 --- a/cvat/apps/engine/tests/_test_rest_api.py +++ b/cvat/apps/engine/tests/_test_rest_api.py @@ -2082,7 +2082,14 @@ def _create_task(self, owner, assignee): "client_files[0]": generate_image_file("test_1.jpg")[1], "client_files[1]": generate_image_file("test_2.jpg")[1], "client_files[2]": generate_image_file("test_3.jpg")[1], + "client_files[4]": generate_image_file("test_4.jpg")[1], + "client_files[5]": generate_image_file("test_5.jpg")[1], + "client_files[6]": generate_image_file("test_6.jpg")[1], + "client_files[7]": generate_image_file("test_7.jpg")[1], + "client_files[8]": generate_image_file("test_8.jpg")[1], + "client_files[9]": generate_image_file("test_9.jpg")[1], "image_quality": 75, + "frame_filter": "step=3", } response = self.client.post("/api/v1/tasks/{}/data".format(tid), data=images) assert response.status_code == status.HTTP_202_ACCEPTED @@ -2202,7 +2209,7 @@ def _run_api_v1_jobs_id_annotations(self, owner, assignee, annotator): "occluded": False }, { - "frame": 1, + "frame": 2, "label_id": task["labels"][1]["id"], "group": None, "source": "manual", @@ -2239,7 +2246,7 @@ def _run_api_v1_jobs_id_annotations(self, owner, assignee, annotator): ] }, { - "frame": 1, + "frame": 2, "attributes": [], "points": [2.0, 2.1, 100, 300.222], "type": "rectangle", @@ -2256,7 +2263,7 @@ def _run_api_v1_jobs_id_annotations(self, owner, assignee, annotator): "attributes": [], "shapes": [ { - "frame": 1, + "frame": 2, "attributes": [], "points": [1.0, 2.1, 100, 300.222], "type": "rectangle", From a5b63a4f53335fb69f36188ceed1243c879bed66 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 8 Sep 2020 20:59:00 +0100 Subject: [PATCH 8/9] Add python3-setuptools to install list (#2153) Installing python3-setuptools was required on a fresh ubuntu 18.04 VM image. Otherwise the next line fails with "no module setuptools" --- cvat/apps/documentation/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md index ce2e9008dc5..bc83b7c084e 100644 --- a/cvat/apps/documentation/installation.md +++ b/cvat/apps/documentation/installation.md @@ -67,7 +67,7 @@ server. Proxy is an advanced topic and it is not covered by the guide. defining and running multi-container docker applications. ```bash - sudo apt-get --no-install-recommends install -y python3-pip + sudo apt-get --no-install-recommends install -y python3-pip python3-setuptools sudo python3 -m pip install setuptools docker-compose ``` From 4e219299e1ef3f209b43daa7a0896c7f1005d00d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 8 Sep 2020 23:01:35 +0300 Subject: [PATCH 9/9] UI Tracking with serverless functions (#2136) * tmp * Refactored * Refactoring & added button to context menu * Updated changelog, updated versions * Improved styles * Removed outdated code * Updated icon --- CHANGELOG.md | 1 + .../src/typescript/interactionHandler.ts | 4 - cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- cvat-ui/src/actions/annotation-actions.ts | 33 +- cvat-ui/src/actions/plugins-actions.ts | 5 +- .../controls-side-bar/controls-side-bar.tsx | 4 +- .../controls-side-bar/tools-control.tsx | 346 ++++++++++++++++-- .../objects-side-bar/object-item-basics.tsx | 3 + .../objects-side-bar/object-item-menu.tsx | 32 +- .../objects-side-bar/object-item.tsx | 3 + .../standard-workspace/standard-workspace.tsx | 1 + .../standard-workspace/styles.scss | 3 +- .../objects-side-bar/object-item.tsx | 14 +- cvat-ui/src/reducers/annotation-reducer.ts | 4 +- cvat-ui/src/reducers/interfaces.ts | 4 +- cvat-ui/src/utils/range.ts | 27 ++ 19 files changed, 431 insertions(+), 61 deletions(-) create mode 100644 cvat-ui/src/utils/range.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c7f5e7a2548..7d22ae580ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007) - Annotation in process outline color wheel () - On the fly annotation using DL detectors () +- Automatic tracking of bounding boxes using serverless functions () - [Datumaro] CLI command for dataset equality comparison () - [Datumaro] Merging of datasets with different labels () diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index 76237cf06ed..7ede21e3265 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -155,10 +155,6 @@ export class InteractionHandlerImpl implements InteractionHandler { this.shapesWereUpdated = true; this.canvas.off('mousedown.interaction', eventListener); - if (this.shouldRaiseEvent(false)) { - this.onInteraction(this.prepareResult(), true, false); - } - this.interact({ enabled: false }); }).addClass('cvat_canvas_shape_drawing').attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 91ac8a0f822..55038b63bea 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.6.0", + "version": "3.6.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 4418f157449..869464ac5c1 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.6.0", + "version": "3.6.1", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 5f7a0b86a67..5c8e3541d84 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.9.2", + "version": "1.9.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index f91eabd335a..f504a65bbd7 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.9.2", + "version": "1.9.3", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index efeaa81e8ff..e6b3ef2f6bc 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -27,6 +27,7 @@ import getCore from 'cvat-core-wrapper'; import logger, { LogType } from 'cvat-logger'; import { RectDrawingMethod } from 'cvat-canvas-wrapper'; import { getCVATStore } from 'cvat-store'; +import { MutableRefObject } from 'react'; interface AnnotationsParameters { filters: string[]; @@ -189,6 +190,7 @@ export enum AnnotationActionTypes { SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS', SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED', INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS', + SET_AI_TOOLS_REF = 'SET_AI_TOOLS_REF', } export function saveLogsAsync(): ThunkAction { @@ -1397,6 +1399,16 @@ export function interactWithCanvas(activeInteractor: Model, activeLabelID: numbe }; } +export function setAIToolsRef(ref: MutableRefObject): AnyAction { + return { + type: AnnotationActionTypes.SET_AI_TOOLS_REF, + payload: { + aiToolsRef: ref, + }, + }; +} + + export function repeatDrawShapeAsync(): ThunkAction { return async (dispatch: ActionCreator): Promise => { const { @@ -1424,12 +1436,21 @@ export function repeatDrawShapeAsync(): ThunkAction { let activeControl = ActiveControl.CURSOR; if (activeInteractor) { - canvasInstance.interact({ - enabled: true, - shapeType: 'points', - minPosVertices: 4, // TODO: Add parameter to interactor - }); - dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + if (activeInteractor.type === 'tracker') { + canvasInstance.interact({ + enabled: true, + shapeType: 'rectangle', + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + } else { + canvasInstance.interact({ + enabled: true, + shapeType: 'points', + minPosVertices: 4, // TODO: Add parameter to interactor + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + } + return; } diff --git a/cvat-ui/src/actions/plugins-actions.ts b/cvat-ui/src/actions/plugins-actions.ts index 6b5ca2c93ad..c1c739f6aa3 100644 --- a/cvat-ui/src/actions/plugins-actions.ts +++ b/cvat-ui/src/actions/plugins-actions.ts @@ -30,19 +30,16 @@ export function checkPluginsAsync(): ThunkAction { const plugins: PluginObjects = { ANALYTICS: false, GIT_INTEGRATION: false, - DEXTR_SEGMENTATION: false, }; const promises: Promise[] = [ // check must return true/false with no exceptions PluginChecker.check(SupportedPlugins.ANALYTICS), PluginChecker.check(SupportedPlugins.GIT_INTEGRATION), - PluginChecker.check(SupportedPlugins.DEXTR_SEGMENTATION), ]; const values = await Promise.all(promises); - [plugins.ANALYTICS, plugins.GIT_INTEGRATION, - plugins.DEXTR_SEGMENTATION] = values; + [plugins.ANALYTICS, plugins.GIT_INTEGRATION] = values; dispatch(pluginActions.checkedAllPlugins(plugins)); }; } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx index a202d5c99ac..ef4b457e11d 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx @@ -85,7 +85,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { preventDefault(event); const drawing = [ActiveControl.DRAW_POINTS, ActiveControl.DRAW_POLYGON, ActiveControl.DRAW_POLYLINE, ActiveControl.DRAW_RECTANGLE, - ActiveControl.DRAW_CUBOID, ActiveControl.INTERACTION].includes(activeControl); + ActiveControl.DRAW_CUBOID, ActiveControl.AI_TOOLS].includes(activeControl); if (!drawing) { canvasInstance.cancel(); @@ -98,7 +98,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { repeatDrawShape(); } } else { - if (activeControl === ActiveControl.INTERACTION) { + if (activeControl === ActiveControl.AI_TOOLS) { // separated API method canvasInstance.interact({ enabled: false }); return; diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 195466b26b6..83db38e21c0 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; +import React, { MutableRefObject } from 'react'; import { connect } from 'react-redux'; import Icon from 'antd/lib/icon'; import Popover from 'antd/lib/popover'; @@ -13,9 +13,11 @@ import Text from 'antd/lib/typography/Text'; import Tabs from 'antd/lib/tabs'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; +import Progress from 'antd/lib/progress'; import { AIToolsIcon } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; +import range from 'utils/range'; import getCore from 'cvat-core-wrapper'; import { CombinedState, @@ -32,6 +34,7 @@ import { } from 'actions/annotation-actions'; import { InteractionResult } from 'cvat-canvas/src/typescript/canvas'; import DetectorRunner from 'components/model-runner-modal/detector-runner'; +import InputNumber from 'antd/lib/input-number'; interface StateToProps { canvasInstance: Canvas; @@ -39,10 +42,13 @@ interface StateToProps { states: any[]; activeLabelID: number; jobInstance: any; - isInteraction: boolean; + isActivated: boolean; frame: number; interactors: Model[]; detectors: Model[]; + trackers: Model[]; + curZOrder: number; + aiToolsRef: MutableRefObject; } interface DispatchToProps { @@ -60,18 +66,21 @@ function mapStateToProps(state: CombinedState): StateToProps { const { instance: jobInstance } = annotation.job; const { instance: canvasInstance, activeControl } = annotation.canvas; const { models } = state; - const { interactors, detectors } = models; + const { interactors, detectors, trackers } = models; return { interactors, detectors, - isInteraction: activeControl === ActiveControl.INTERACTION, + trackers, + isActivated: activeControl === ActiveControl.AI_TOOLS, activeLabelID: annotation.drawing.activeLabelID, labels: annotation.job.labels, states: annotation.annotations.states, canvasInstance, jobInstance, frame, + curZOrder: annotation.annotations.zLayer.cur, + aiToolsRef: annotation.aiToolsRef, }; } @@ -103,10 +112,14 @@ interface State { activeInteractor: Model | null; activeLabelID: number; interactiveStateID: number | null; + activeTracker: Model | null; + trackingProgress: number | null; + trackingFrames: number; fetching: boolean; + mode: 'detection' | 'interaction' | 'tracking'; } -class ToolsControlComponent extends React.PureComponent { +export class ToolsControlComponent extends React.PureComponent { private interactionIsAborted: boolean; private interactionIsDone: boolean; @@ -114,9 +127,13 @@ class ToolsControlComponent extends React.PureComponent { super(props); this.state = { activeInteractor: props.interactors.length ? props.interactors[0] : null, + activeTracker: props.trackers.length ? props.trackers[0] : null, activeLabelID: props.labels[0].id, interactiveStateID: null, + trackingProgress: null, + trackingFrames: 10, fetching: false, + mode: 'interaction', }; this.interactionIsAborted = false; @@ -124,16 +141,18 @@ class ToolsControlComponent extends React.PureComponent { } public componentDidMount(): void { - const { canvasInstance } = this.props; + const { canvasInstance, aiToolsRef } = this.props; + aiToolsRef.current = this; canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener); canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener); } public componentDidUpdate(prevProps: Props): void { - const { isInteraction } = this.props; - if (prevProps.isInteraction && !isInteraction) { + const { isActivated } = this.props; + if (prevProps.isActivated && !isActivated) { window.removeEventListener('contextmenu', this.contextmenuDisabler); - } else if (!prevProps.isInteraction && isInteraction) { + } else if (!prevProps.isActivated && isActivated) { + // reset flags when start interaction/tracking this.interactionIsDone = false; this.interactionIsAborted = false; window.addEventListener('contextmenu', this.contextmenuDisabler); @@ -141,7 +160,8 @@ class ToolsControlComponent extends React.PureComponent { } public componentWillUnmount(): void { - const { canvasInstance } = this.props; + const { canvasInstance, aiToolsRef } = this.props; + aiToolsRef.current = undefined; canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener); } @@ -162,14 +182,14 @@ class ToolsControlComponent extends React.PureComponent { private cancelListener = async (): Promise => { const { - isInteraction, + isActivated, jobInstance, frame, fetchAnnotations, } = this.props; const { interactiveStateID, fetching } = this.state; - if (isInteraction) { + if (isActivated) { if (fetching && !this.interactionIsDone) { // user pressed ESC this.setState({ fetching: false }); @@ -187,12 +207,13 @@ class ToolsControlComponent extends React.PureComponent { } }; - private interactionListener = async (e: Event): Promise => { + private onInteraction = async (e: Event): Promise => { const { frame, labels, + curZOrder, jobInstance, - isInteraction, + isActivated, activeLabelID, fetchAnnotations, updateAnnotations, @@ -200,8 +221,8 @@ class ToolsControlComponent extends React.PureComponent { const { activeInteractor, interactiveStateID, fetching } = this.state; try { - if (!isInteraction) { - throw Error('Canvas raises event "canvas.interacted" when interaction is off'); + if (!isActivated) { + throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); } if (fetching) { @@ -216,7 +237,6 @@ class ToolsControlComponent extends React.PureComponent { this.setState({ fetching: true }); try { result = await core.lambda.call(jobInstance.task, interactor, { - task: jobInstance.task, frame, points: convertShapesForInteractor((e as CustomEvent).detail.shapes), }); @@ -241,7 +261,7 @@ class ToolsControlComponent extends React.PureComponent { shapeType: ShapeType.POLYGON, points: result.flat(), occluded: false, - zOrder: (e as CustomEvent).detail.zOrder, + zOrder: curZOrder, }); await jobInstance.annotations.put([object]); @@ -260,7 +280,7 @@ class ToolsControlComponent extends React.PureComponent { shapeType: ShapeType.POLYGON, points: result.flat(), occluded: false, - zOrder: (e as CustomEvent).detail.zOrder, + zOrder: curZOrder, }); // need a clientID of a created object to interact with it further // so, we do not use createAnnotationAction @@ -302,6 +322,71 @@ class ToolsControlComponent extends React.PureComponent { } }; + private onTracking = async (e: Event): Promise => { + const { + isActivated, + jobInstance, + frame, + curZOrder, + fetchAnnotations, + } = this.props; + const { activeLabelID } = this.state; + const [label] = jobInstance.task.labels.filter( + (_label: any): boolean => _label.id === activeLabelID, + ); + + if (!(e as CustomEvent).detail.isDone) { + return; + } + + this.interactionIsDone = true; + try { + if (!isActivated) { + throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); + } + + const { points } = (e as CustomEvent).detail.shapes[0]; + const state = new core.classes.ObjectState({ + shapeType: ShapeType.RECTANGLE, + objectType: ObjectType.TRACK, + zOrder: curZOrder, + label, + points, + frame, + occluded: false, + source: 'auto', + attributes: {}, + }); + + const [clientID] = await jobInstance.annotations.put([state]); + + // update annotations on a canvas + fetchAnnotations(); + + const states = await jobInstance.annotations.get(frame); + const [objectState] = states + .filter((_state: any): boolean => _state.clientID === clientID); + await this.trackState(objectState); + } catch (err) { + notification.error({ + description: err.toString(), + message: 'Tracking error occured', + }); + } + }; + + private interactionListener = async (e: Event): Promise => { + const { mode } = this.state; + + if (mode === 'interaction') { + await this.onInteraction(e); + } + + if (mode === 'tracking') { + await this.onTracking(e); + } + }; + private setActiveInteractor = (key: string): void => { const { interactors } = this.props; this.setState({ @@ -311,6 +396,72 @@ class ToolsControlComponent extends React.PureComponent { }); }; + private setActiveTracker = (key: string): void => { + const { trackers } = this.props; + this.setState({ + activeTracker: trackers.filter( + (tracker: Model) => tracker.id === key, + )[0], + }); + }; + + public async trackState(state: any): Promise { + const { jobInstance, frame } = this.props; + const { activeTracker, trackingFrames } = this.state; + const { clientID, points } = state; + + const tracker = activeTracker as Model; + try { + this.setState({ trackingProgress: 0, fetching: true }); + let response = await core.lambda.call(jobInstance.task, tracker, { + task: jobInstance.task, + frame, + shape: points, + }); + + for (const offset of range(1, trackingFrames + 1)) { + /* eslint-disable no-await-in-loop */ + const states = await jobInstance.annotations.get(frame + offset); + const [objectState] = states + .filter((_state: any): boolean => _state.clientID === clientID); + response = await core.lambda.call(jobInstance.task, tracker, { + task: jobInstance.task, + frame: frame + offset, + shape: response.points, + state: response.state, + }); + + const reduced = response.shape + .reduce((acc: number[], value: number, index: number): number[] => { + if (index % 2) { // y + acc[1] = Math.min(acc[1], value); + acc[3] = Math.max(acc[3], value); + } else { // x + acc[0] = Math.min(acc[0], value); + acc[2] = Math.max(acc[2], value); + } + return acc; + }, [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, + Number.MIN_SAFE_INTEGER, Number.MIN_SAFE_INTEGER, + ]); + + objectState.points = reduced; + await objectState.save(); + + this.setState({ trackingProgress: offset / trackingFrames }); + } + } finally { + this.setState({ trackingProgress: null, fetching: false }); + } + } + + public trackingAvailable(): boolean { + const { activeTracker, trackingFrames } = this.state; + const { trackers } = this.props; + + return !!trackingFrames && !!trackers.length && activeTracker !== null; + } + private renderLabelBlock(): JSX.Element { const { labels } = this.props; const { activeLabelID } = this.state; @@ -355,10 +506,119 @@ class ToolsControlComponent extends React.PureComponent { ); } + private renderTrackerBlock(): JSX.Element { + const { + trackers, + canvasInstance, + jobInstance, + frame, + onInteractionStart, + } = this.props; + const { + activeTracker, + activeLabelID, + fetching, + trackingFrames, + } = this.state; + + if (!trackers.length) { + return ( + + + No available trackers found + + + ); + } + + return ( + <> + + + Tracker + + + + + + + + + + Tracking frames + + + { + if (typeof (value) !== 'undefined') { + this.setState({ + trackingFrames: value, + }); + } + }} + /> + + + + + + + + + ); + } + private renderInteractorBlock(): JSX.Element { const { interactors, canvasInstance, onInteractionStart } = this.props; const { activeInteractor, activeLabelID, fetching } = this.state; + if (!interactors.length) { + return ( + + + No available interactors found + + + ); + } + return ( <> @@ -389,6 +649,10 @@ class ToolsControlComponent extends React.PureComponent { className='cvat-tools-interact-button' disabled={!activeInteractor || fetching} onClick={() => { + this.setState({ + mode: 'interaction', + }); + if (activeInteractor) { canvasInstance.cancel(); canvasInstance.interact({ @@ -413,10 +677,21 @@ class ToolsControlComponent extends React.PureComponent { const { jobInstance, detectors, + curZOrder, frame, fetchAnnotations, } = this.props; + if (!detectors.length) { + return ( + + + No available detectors found + + + ); + } + return ( { task={jobInstance.task} runInference={async (task: any, model: Model, body: object) => { try { + this.setState({ + mode: 'detection', + }); + this.setState({ fetching: true }); const result = await core.lambda.call(task, model, { ...body, @@ -444,7 +723,7 @@ class ToolsControlComponent extends React.PureComponent { occluded: false, source: 'auto', attributes: {}, - zOrder: 0, // TODO: get current z order + zOrder: curZOrder, }) )); @@ -471,7 +750,7 @@ class ToolsControlComponent extends React.PureComponent { AI Tools - + { this.renderLabelBlock() } { this.renderInteractorBlock() } @@ -479,24 +758,34 @@ class ToolsControlComponent extends React.PureComponent { { this.renderDetectorBlock() } + + { this.renderLabelBlock() } + { this.renderTrackerBlock() } + ); } public render(): JSX.Element | null { - const { interactors, isInteraction, canvasInstance } = this.props; - const { fetching } = this.state; + const { + interactors, + detectors, + trackers, + isActivated, + canvasInstance, + } = this.props; + const { fetching, trackingProgress } = this.state; - if (!interactors.length) return null; + if (![...interactors, ...detectors, ...trackers].length) return null; - const dynamcPopoverPros = isInteraction ? { + const dynamcPopoverPros = isActivated ? { overlayStyle: { display: 'none', }, } : {}; - const dynamicIconProps = isInteraction ? { + const dynamicIconProps = isActivated ? { className: 'cvat-active-canvas-control cvat-tools-control', onClick: (): void => { canvasInstance.interact({ enabled: false }); @@ -517,12 +806,15 @@ class ToolsControlComponent extends React.PureComponent { > Waiting for a server response.. + { trackingProgress !== null && ( + + )} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx index bbfc0e034ff..9976518754e 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx @@ -41,6 +41,7 @@ interface Props { toBackground(): void; toForeground(): void; resetCuboidPerspective(): void; + activateTracking(): void; } function ItemTopComponent(props: Props): JSX.Element { @@ -72,6 +73,7 @@ function ItemTopComponent(props: Props): JSX.Element { toBackground, toForeground, resetCuboidPerspective, + activateTracking, } = props; const [menuVisible, setMenuVisible] = useState(false); @@ -150,6 +152,7 @@ function ItemTopComponent(props: Props): JSX.Element { toForeground, resetCuboidPerspective, changeColorPickerVisible, + activateTracking, })} > diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx index 99db3b84891..5fe8af31255 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx @@ -33,16 +33,17 @@ interface Props { toBackgroundShortcut: string; toForegroundShortcut: string; removeShortcut: string; - changeColor: (value: string) => void; - copy: (() => void); - remove: (() => void); - propagate: (() => void); - createURL: (() => void); - switchOrientation: (() => void); - toBackground: (() => void); - toForeground: (() => void); - resetCuboidPerspective: (() => void); - changeColorPickerVisible: (visible: boolean) => void; + changeColor(value: string): void; + copy(): void; + remove(): void; + propagate(): void; + createURL(): void; + switchOrientation(): void; + toBackground(): void; + toForeground(): void; + resetCuboidPerspective(): void; + changeColorPickerVisible(visible: boolean): void; + activateTracking(): void; } export default function ItemMenu(props: Props): JSX.Element { @@ -71,6 +72,7 @@ export default function ItemMenu(props: Props): JSX.Element { toForeground, resetCuboidPerspective, changeColorPickerVisible, + activateTracking, } = props; return ( @@ -94,6 +96,16 @@ export default function ItemMenu(props: Props): JSX.Element { + {objectType === ObjectType.TRACK && shapeType === ShapeType.RECTANGLE && ( + + + + + + )} { [ShapeType.POLYGON, ShapeType.POLYLINE, ShapeType.CUBOID].includes(shapeType) && (