Merge branch 'develop' into az/av

cvat-ai · Jun 16, 2020 · 63fcdec · 63fcdec
2 parents cf2408b + 587ca4d
commit 63fcdec
Show file tree

Hide file tree

Showing 9 changed files with 114 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Shortcut to change color of an activated shape in new UI (Enter) (<https://github.com/opencv/cvat/pull/1683>)
 - Shortcut to switch split mode (<https://github.com/opencv/cvat/pull/1683>)
 - Built-in search for labels when create an object or change a label (<https://github.com/opencv/cvat/pull/1683>)
+- Better validation of labels and attributes in raw viewer (<https://github.com/opencv/cvat/pull/1727>)
 - ClamAV antivirus integration (<https://github.com/opencv/cvat/pull/1712>)
 
 ### Changed
@@ -26,12 +27,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -
 
 ### Fixed
+- Problem with exported frame stepped image task (<https://github.com/opencv/cvat/issues/1613>)
 - Fixed dataset filter item representation for imageless dataset items (<https://github.com/opencv/cvat/pull/1593>)
 - Fixed interpreter crash when trying to import `tensorflow` with no AVX instructions available (<https://github.com/opencv/cvat/pull/1567>)
 - Kibana wrong working time calculation with new annotation UI use (<https://github.com/opencv/cvat/pull/1654>)
 - Wrong rexex for account name validation (<https://github.com/opencv/cvat/pull/1667>)
 - Wrong description on register view for the username field (<https://github.com/opencv/cvat/pull/1667>)
 - Wrong resolution for resizing a shape (<https://github.com/opencv/cvat/pull/1667>)
+- React warning because of not unique keys in labels viewer (<https://github.com/opencv/cvat/pull/1727>)
+
 
 ### Security
 - SQL injection in Django `CVE-2020-9402` (<https://github.com/opencv/cvat/pull/1657>)
@@ -86,6 +90,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A problem with mask to polygons conversion when polygons are too small (<https://github.com/opencv/cvat/pull/1581>)
 - Unable to upload video with uneven size (<https://github.com/opencv/cvat/pull/1594>)
 - Fixed an issue with `z_order` having no effect on segmentations (<https://github.com/opencv/cvat/pull/1589>)
+
+### Security
 - Permission group whitelist check for analytics view (<https://github.com/opencv/cvat/pull/1608>)
 
 ## [1.0.0-beta.2] - 2020-04-30

diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json
diff --git a/cvat-ui/package.json b/cvat-ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

diff --git a/cvat-ui/src/components/labels-editor/common.ts b/cvat-ui/src/components/labels-editor/common.ts
@@ -18,6 +18,59 @@ export interface Label {
 
 let id = 0;
 
+function validateParsedAttribute(attr: Attribute): void {
+    if (typeof (attr.name) !== 'string') {
+        throw new Error(`Type of attribute name must be a string. Got value ${attr.name}`);
+    }
+
+    if (!['number', 'undefined'].includes(typeof (attr.id))) {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Type of attribute id must be a number or undefined. Got value ${attr.id}`);
+    }
+
+    if (!['checkbox', 'number', 'text', 'radio', 'select'].includes((attr.input_type || '').toLowerCase())) {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Unknown input type: ${attr.input_type}`);
+    }
+
+    if (typeof (attr.mutable) !== 'boolean') {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Mutable flag must be a boolean value. Got value ${attr.mutable}`);
+    }
+
+    if (!Array.isArray(attr.values)) {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Attribute values must be an array. Got type ${typeof (attr.values)}`);
+    }
+
+    for (const value of attr.values) {
+        if (typeof (value) !== 'string') {
+            throw new Error(`Attribute: "${attr.name}". `
+            + `Each value must be a string. Got value ${value}`);
+        }
+    }
+}
+
+export function validateParsedLabel(label: Label): void {
+    if (typeof (label.name) !== 'string') {
+        throw new Error(`Type of label name must be a string. Got value ${label.name}`);
+    }
+
+    if (!['number', 'undefined'].includes(typeof (label.id))) {
+        throw new Error(`Label "${label.name}". `
+        + `Type of label id must be only a number or undefined. Got value ${label.id}`);
+    }
+
+    if (!Array.isArray(label.attributes)) {
+        throw new Error(`Label "${label.name}". `
+        + `attributes must be an array. Got type ${typeof (label.attributes)}`);
+    }
+
+    for (const attr of label.attributes) {
+        validateParsedAttribute(attr);
+    }
+}
+
 export function idGenerator(): number {
     return --id;
 }

diff --git a/cvat-ui/src/components/labels-editor/raw-viewer.tsx b/cvat-ui/src/components/labels-editor/raw-viewer.tsx
@@ -12,6 +12,8 @@ import Form, { FormComponentProps } from 'antd/lib/form/Form';
 import {
     Label,
     Attribute,
+    validateParsedLabel,
+    idGenerator,
 } from './common';
 
 type Props = FormComponentProps & {
@@ -22,7 +24,18 @@ type Props = FormComponentProps & {
 class RawViewer extends React.PureComponent<Props> {
     private validateLabels = (_: any, value: string, callback: any): void => {
         try {
-            JSON.parse(value);
+            const parsed = JSON.parse(value);
+            if (!Array.isArray(parsed)) {
+                callback('Field is expected to be a JSON array');
+            }
+
+            for (const label of parsed) {
+                try {
+                    validateParsedLabel(label);
+                } catch (error) {
+                    callback(error.toString());
+                }
+            }
         } catch (error) {
             callback(error.toString());
         }
@@ -39,7 +52,14 @@ class RawViewer extends React.PureComponent<Props> {
         e.preventDefault();
         form.validateFields((error, values): void => {
             if (!error) {
-                onSubmit(JSON.parse(values.labels));
+                const parsed = JSON.parse(values.labels);
+                for (const label of parsed) {
+                    label.id = label.id || idGenerator();
+                    for (const attr of label.attributes) {
+                        attr.id = attr.id || idGenerator();
+                    }
+                }
+                onSubmit(parsed);
             }
         });
     };

diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py
@@ -97,16 +97,21 @@ def _get_mutable_attribute_id(self, label_id, attribute_name):
     def _get_immutable_attribute_id(self, label_id, attribute_name):
         return self._get_attribute_id(label_id, attribute_name, 'immutable')
 
+    def abs_frame_id(self, relative_id):
+        return relative_id * self._frame_step + self._db_task.data.start_frame
+
+    def rel_frame_id(self, absolute_id):
+        return (absolute_id - self._db_task.data.start_frame) // self._frame_step
+
     def _init_frame_info(self):
         if hasattr(self._db_task.data, 'video'):
             self._frame_info = {frame: {
-                "path": "frame_{:06d}".format(
-                    self._db_task.data.start_frame + frame * self._frame_step),
+                "path": "frame_{:06d}".format(self.abs_frame_id(frame)),
                 "width": self._db_task.data.video.width,
                 "height": self._db_task.data.video.height,
             } for frame in range(self._db_task.data.size)}
         else:
-            self._frame_info = {db_image.frame: {
+            self._frame_info = {self.rel_frame_id(db_image.frame): {
                 "path": db_image.path,
                 "width": db_image.width,
                 "height": db_image.height,
@@ -193,8 +198,7 @@ def _export_attributes(self, attributes):
     def _export_tracked_shape(self, shape):
         return TaskData.TrackedShape(
             type=shape["type"],
-            frame=self._db_task.data.start_frame +
-                shape["frame"] * self._frame_step,
+            frame=self.abs_frame_id(shape["frame"]),
             label=self._get_label_name(shape["label_id"]),
             points=shape["points"],
             occluded=shape["occluded"],
@@ -210,8 +214,7 @@ def _export_labeled_shape(self, shape):
         return TaskData.LabeledShape(
             type=shape["type"],
             label=self._get_label_name(shape["label_id"]),
-            frame=self._db_task.data.start_frame +
-                shape["frame"] * self._frame_step,
+            frame=self.abs_frame_id(shape["frame"]),
             points=shape["points"],
             occluded=shape["occluded"],
             z_order=shape.get("z_order", 0),
@@ -221,8 +224,7 @@ def _export_labeled_shape(self, shape):
 
     def _export_tag(self, tag):
         return TaskData.Tag(
-            frame=self._db_task.data.start_frame +
-                tag["frame"] * self._frame_step,
+            frame=self.abs_frame_id(tag["frame"]),
             label=self._get_label_name(tag["label_id"]),
             group=tag.get("group", 0),
             attributes=self._export_attributes(tag["attributes"]),
@@ -232,7 +234,7 @@ def group_by_frame(self, include_empty=False):
         frames = {}
         def get_frame(idx):
             frame_info = self._frame_info[idx]
-            frame = self._db_task.data.start_frame + idx * self._frame_step
+            frame = self.abs_frame_id(idx)
             if frame not in frames:
                 frames[frame] = TaskData.Frame(
                     idx=idx,
@@ -299,8 +301,7 @@ def meta(self):
     def _import_tag(self, tag):
         _tag = tag._asdict()
         label_id = self._get_label_id(_tag.pop('label'))
-        _tag['frame'] = (int(_tag['frame']) -
-            self._db_task.data.start_frame) // self._frame_step
+        _tag['frame'] = self.rel_frame_id(int(_tag['frame']))
         _tag['label_id'] = label_id
         _tag['attributes'] = [self._import_attribute(label_id, attrib)
             for attrib in _tag['attributes']
@@ -316,8 +317,7 @@ def _import_attribute(self, label_id, attribute):
     def _import_shape(self, shape):
         _shape = shape._asdict()
         label_id = self._get_label_id(_shape.pop('label'))
-        _shape['frame'] = (int(_shape['frame']) -
-            self._db_task.data.start_frame) // self._frame_step
+        _shape['frame'] = self.rel_frame_id(int(_shape['frame']))
         _shape['label_id'] = label_id
         _shape['attributes'] = [self._import_attribute(label_id, attrib)
             for attrib in _shape['attributes']
@@ -327,14 +327,13 @@ def _import_shape(self, shape):
     def _import_track(self, track):
         _track = track._asdict()
         label_id = self._get_label_id(_track.pop('label'))
-        _track['frame'] = (min(int(shape.frame) for shape in _track['shapes']) -
-            self._db_task.data.start_frame) // self._frame_step
+        _track['frame'] = self.rel_frame_id(
+            min(int(shape.frame) for shape in _track['shapes']))
         _track['label_id'] = label_id
         _track['attributes'] = []
         _track['shapes'] = [shape._asdict() for shape in _track['shapes']]
         for shape in _track['shapes']:
-            shape['frame'] = (int(shape['frame']) - \
-                self._db_task.data.start_frame) // self._frame_step
+            shape['frame'] = self.rel_frame_id(int(shape['frame']))
             _track['attributes'] = [self._import_attribute(label_id, attrib)
                 for attrib in shape['attributes']
                 if self._get_immutable_attribute_id(label_id, attrib.name)]
@@ -567,7 +566,7 @@ def import_dm_annotations(dm_dataset, task_data):
     label_cat = dm_dataset.categories()[datumaro.AnnotationType.label]
 
     for item in dm_dataset:
-        frame_number = match_frame(item, task_data)
+        frame_number = task_data.abs_frame_id(match_frame(item, task_data))
 
         # do not store one-item groups
         group_map = {0: 0}

diff --git a/cvat/apps/dataset_manager/formats/mot.py b/cvat/apps/dataset_manager/formats/mot.py
@@ -40,7 +40,7 @@ def _import(src_file, task_data):
 
         for item in dataset:
             item = item.wrap(id=int(item.id) - 1) # NOTE: MOT frames start from 1
-            frame_id = match_frame(item, task_data)
+            frame_number = task_data.abs_frame_id(match_frame(item, task_data))
 
             for ann in item.annotations:
                 if ann.type != datumaro.AnnotationType.bbox:
@@ -57,7 +57,7 @@ def _import(src_file, task_data):
                     outside=False,
                     keyframe=False,
                     z_order=ann.z_order,
-                    frame=frame_id,
+                    frame=frame_number,
                     attributes=[],
                 )
 

diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt
@@ -1,22 +1,22 @@
 click==6.7
 Django==2.2.13
-django-appconf==1.0.2
+django-appconf==1.0.4
 django-auth-ldap==1.4.0
-django-cacheops==4.0.6
-django-compressor==2.2
+django-cacheops==5.0
+django-compressor==2.4
 django-rq==2.0.0
 EasyProcess==0.2.3
 Pillow==7.1.2
 numpy==1.16.2
-python-ldap==3.0.0
+python-ldap==3.2.0
 pytz==2018.3
 pyunpack==0.1.2
 rcssmin==1.0.6
 redis==3.2.0
 requests==2.23.0
 rjsmin==1.0.12
 rq==1.0.0
-rq-scheduler==0.9.1
+rq-scheduler==0.10.0
 scipy==1.4.1
 sqlparse==0.2.4
 django-sendfile==0.3.11
@@ -28,10 +28,10 @@ GitPython==3.1.3
 coreapi==2.3.3
 django-filter==2.0.0
 Markdown==3.2.2
-djangorestframework==3.9.3
+djangorestframework==3.11.0
 Pygments==2.6.1
 drf-yasg==1.17.0
-Shapely==1.6.4.post2
+Shapely==1.7.0
 pdf2image==1.6.0
 pascal_voc_writer==0.1.4
 django-rest-auth[with_social]==0.9.5
@@ -43,7 +43,7 @@ keras==2.3.1
 opencv-python==4.1.0.25
 h5py==2.9.0
 imgaug==0.2.9
-django-cors-headers==3.2.0
+django-cors-headers==3.3.0
 furl==2.0.0
 av==6.2.0
 # The package is used by pyunpack as a command line tool to support multiple

diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt
@@ -1,13 +1,13 @@
 -r base.txt
-astroid==2.2.5
+astroid==2.4.2
 isort==4.3.21
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
 pylint==2.3.1
-pylint-django==0.9.4
+pylint-django==2.0.15
 pylint-plugin-utils==0.2.6
 rope==0.11
 wrapt==1.12.1
 django-extensions==2.0.6
 Werkzeug==0.15.3
-snakeviz==0.4.2
+snakeviz==2.1.0