Running detectors on the fly (#2102)

* Draft version

* Removed extra file

* Removed extra code

* Updated icon: magic wand

* Ctrl modifier, fixed some cases when interaction event isn't raised

* Added tooltip description of an interactor

* Locking UI while server fetching

* Removing old code & refactoring

* Fixed couple of bugs

* Updated CHANGELOG.md, updated versions

* Update crosshair.ts

* Minor fixes

* Fixed eslint issues

* Prevent default action

* Added minNegVertices=0 by default, ignored negative points for dextr, fixed context menu in some cases

* On the fly annotations draft

* Initial version of FBRS interactive segmentation

* Fix fbrs model_handler

* Fixed couple of minor bugs

* Added ability to interrupt interaction

* Do not show reid on annotation view

* Prettified UI

* Updated changelog, increased version

* Removed extra files

* Removed extra code

* Fixed changelog

Co-authored-by: Nikita Manovich <nikita.manovich@intel.com>

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added password reset functionality (<https://github.com/opencv/cvat/pull/2058>)
 - Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007)
 - Annotation in process outline color wheel (<https://github.com/opencv/cvat/pull/2084>)
+- On the fly annotation using DL detectors (<https://github.com/opencv/cvat/pull/2102>)
 - [Datumaro] CLI command for dataset equality comparison (<https://github.com/opencv/cvat/pull/1989>)
 - [Datumaro] Merging of datasets with different labels (<https://github.com/opencv/cvat/pull/2098>)
 

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.9.1",
+  "version": "1.9.2",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx

@@ -10,6 +10,7 @@ import Select, { OptionProps } from 'antd/lib/select';
 import Button from 'antd/lib/button';
 import Modal from 'antd/lib/modal';
 import Text from 'antd/lib/typography/Text';
+import Tabs from 'antd/lib/tabs';
 import { Row, Col } from 'antd/lib/grid';
 import notification from 'antd/lib/notification';
 
@@ -23,8 +24,14 @@ import {
     ObjectType,
     ShapeType,
 } from 'reducers/interfaces';
-import { interactWithCanvas, fetchAnnotationsAsync, updateAnnotationsAsync } from 'actions/annotation-actions';
+import {
+    interactWithCanvas,
+    fetchAnnotationsAsync,
+    updateAnnotationsAsync,
+    createAnnotationsAsync,
+} from 'actions/annotation-actions';
 import { InteractionResult } from 'cvat-canvas/src/typescript/canvas';
+import DetectorRunner from 'components/model-runner-modal/detector-runner';
 
 interface StateToProps {
     canvasInstance: Canvas;
@@ -35,11 +42,13 @@ interface StateToProps {
     isInteraction: boolean;
     frame: number;
     interactors: Model[];
+    detectors: Model[];
 }
 
 interface DispatchToProps {
     onInteractionStart(activeInteractor: Model, activeLabelID: number): void;
     updateAnnotations(statesToUpdate: any[]): void;
+    createAnnotations(sessionInstance: any, frame: number, statesToCreate: any[]): void;
     fetchAnnotations(): void;
 }
 
@@ -51,10 +60,11 @@ function mapStateToProps(state: CombinedState): StateToProps {
     const { instance: jobInstance } = annotation.job;
     const { instance: canvasInstance, activeControl } = annotation.canvas;
     const { models } = state;
-    const { interactors } = models;
+    const { interactors, detectors } = models;
 
     return {
         interactors,
+        detectors,
         isInteraction: activeControl === ActiveControl.INTERACTION,
         activeLabelID: annotation.drawing.activeLabelID,
         labels: annotation.job.labels,
@@ -65,19 +75,12 @@ function mapStateToProps(state: CombinedState): StateToProps {
     };
 }
 
-function mapDispatchToProps(dispatch: any): DispatchToProps {
-    return {
-        onInteractionStart(activeInteractor: Model, activeLabelID: number): void {
-            dispatch(interactWithCanvas(activeInteractor, activeLabelID));
-        },
-        updateAnnotations(statesToUpdate: any[]): void {
-            dispatch(updateAnnotationsAsync(statesToUpdate));
-        },
-        fetchAnnotations(): void {
-            dispatch(fetchAnnotationsAsync());
-        },
-    };
-}
+const mapDispatchToProps = {
+    onInteractionStart: interactWithCanvas,
+    updateAnnotations: updateAnnotationsAsync,
+    fetchAnnotations: fetchAnnotationsAsync,
+    createAnnotations: createAnnotationsAsync,
+};
 
 function convertShapesForInteractor(shapes: InteractionResult[]): number[][] {
     const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => {
@@ -378,9 +381,10 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                         </Select>
                     </Col>
                 </Row>
-                <Row type='flex' align='middle' justify='center'>
-                    <Col offset={4} span={16}>
+                <Row type='flex' align='middle' justify='end'>
+                    <Col>
                         <Button
+                            type='primary'
                             loading={fetching}
                             className='cvat-tools-interact-button'
                             disabled={!activeInteractor || fetching}
@@ -405,6 +409,60 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         );
     }
 
+    private renderDetectorBlock(): JSX.Element {
+        const {
+            jobInstance,
+            detectors,
+            frame,
+            fetchAnnotations,
+        } = this.props;
+
+        return (
+            <DetectorRunner
+                withCleanup={false}
+                models={detectors}
+                task={jobInstance.task}
+                runInference={async (task: any, model: Model, body: object) => {
+                    try {
+                        this.setState({ fetching: true });
+                        const result = await core.lambda.call(task, model, {
+                            ...body,
+                            frame,
+                        });
+
+                        const states = result
+                            .map((data: any): any => (
+                                new core.classes.ObjectState({
+                                    shapeType: data.type,
+                                    label: task.labels
+                                        .filter(
+                                            (label: any): boolean => label.name === data.label,
+                                        )[0],
+                                    points: data.points,
+                                    objectType: ObjectType.SHAPE,
+                                    frame,
+                                    occluded: false,
+                                    source: 'auto',
+                                    attributes: {},
+                                    zOrder: 0, // TODO:  get current z order
+                                })
+                            ));
+
+                        await jobInstance.annotations.put(states);
+                        fetchAnnotations();
+                    } catch (error) {
+                        notification.error({
+                            description: error.toString(),
+                            message: 'Detection error occured',
+                        });
+                    } finally {
+                        this.setState({ fetching: false });
+                    }
+                }}
+            />
+        );
+    }
+
     private renderPopoverContent(): JSX.Element {
         return (
             <div className='cvat-tools-control-popover-content'>
@@ -413,8 +471,15 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                         <Text className='cvat-text-color' strong>AI Tools</Text>
                     </Col>
                 </Row>
-                { this.renderLabelBlock() }
-                { this.renderInteractorBlock() }
+                <Tabs>
+                    <Tabs.TabPane key='interactors' tab='Interactors'>
+                        { this.renderLabelBlock() }
+                        { this.renderInteractorBlock() }
+                    </Tabs.TabPane>
+                    <Tabs.TabPane key='detectors' tab='Detectors'>
+                        { this.renderDetectorBlock() }
+                    </Tabs.TabPane>
+                </Tabs>
             </div>
         );
     }
@@ -443,7 +508,7 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         return (
             <>
                 <Modal
-                    title='Interaction request'
+                    title='Making a server request'
                     zIndex={Number.MAX_SAFE_INTEGER}
                     visible={fetching}
                     closable={false}

cvat-ui/src/components/annotation-page/standard-workspace/styles.scss

@@ -102,10 +102,10 @@
 }
 
 .cvat-tools-control-popover-content {
+    width: 350px;
     padding: 10px;
     border-radius: 5px;
     background: $background-color-2;
-    width: 270px;
 }
 
 .cvat-draw-shape-popover-content {