From 32ef3fcf7ada106306ab3ac956cc6e9d38987f06 Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Fri, 24 Sep 2021 10:31:53 +0300 Subject: [PATCH 01/10] Add Cityscapes format --- .../dataset_manager/formats/cityscapes.py | 52 +++++++++++++++++++ cvat/apps/dataset_manager/formats/registry.py | 1 + 2 files changed, 53 insertions(+) create mode 100644 cvat/apps/dataset_manager/formats/cityscapes.py diff --git a/cvat/apps/dataset_manager/formats/cityscapes.py b/cvat/apps/dataset_manager/formats/cityscapes.py new file mode 100644 index 00000000000..b5c393a96fe --- /dev/null +++ b/cvat/apps/dataset_manager/formats/cityscapes.py @@ -0,0 +1,52 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os.path as osp +from tempfile import TemporaryDirectory + +from datumaro.components.dataset import Dataset +from pyunpack import Archive + +from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, + ProjectData, import_dm_annotations) +from cvat.apps.dataset_manager.util import make_zip_archive + +from .registry import dm_env, exporter, importer + + +@exporter(name='Cityscapes', ext='ZIP', version='1.0') +def _export(dst_file, instance_data, save_images=False): + dataset = Dataset.from_extractors(GetCVATDataExtractor( + instance_data, include_images=save_images), env=dm_env) + with TemporaryDirectory() as temp_dir: + dataset.transform('polygons_to_masks') + dataset.transform('boxes_to_masks') + dataset.export(temp_dir, 'cityscapes', save_images=save_images, + label_map='source') + make_zip_archive(temp_dir, dst_file) + +@importer(name='Cityscapes', ext='ZIP', version='1.0') +def _import(src_file, instance_data): + with TemporaryDirectory() as tmp_dir: + Archive(src_file.name).extractall(tmp_dir) + + labelmap_file = osp.join(tmp_dir, 'label_colors.txt') + if not osp.isfile(labelmap_file): + labels_meta = instance_data.meta['project']['labels'] \ + if isinstance(instance_data, ProjectData) \ + else instance_data.meta['task']['labels'] + + labels = ('%s %s %s %s' % ( + int(label['color'][1:3], base=16), + int(label['color'][3:5], base=16), + int(label['color'][5:7], base=16), + label['name']) + for _, label in labels_meta + ) + with open(labelmap_file, 'w') as f: + f.write('\n'.join(labels)) + + dataset = Dataset.import_from(tmp_dir, 'cityscapes', env=dm_env) + dataset.transform('masks_to_polygons') + import_dm_annotations(dataset, instance_data) \ No newline at end of file diff --git a/cvat/apps/dataset_manager/formats/registry.py b/cvat/apps/dataset_manager/formats/registry.py index 959127ca017..592daccbaf9 100644 --- a/cvat/apps/dataset_manager/formats/registry.py +++ b/cvat/apps/dataset_manager/formats/registry.py @@ -109,4 +109,5 @@ def make_exporter(name): import cvat.apps.dataset_manager.formats.icdar import cvat.apps.dataset_manager.formats.velodynepoint import cvat.apps.dataset_manager.formats.pointcloud +import cvat.apps.dataset_manager.formats.cityscapes From ff9914c46d7322b86d0e9b111a21d9dd6d6759f9 Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Fri, 24 Sep 2021 10:32:17 +0300 Subject: [PATCH 02/10] Add documentation --- .../advanced/formats/format-cityscapes.md | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 site/content/en/docs/manual/advanced/formats/format-cityscapes.md diff --git a/site/content/en/docs/manual/advanced/formats/format-cityscapes.md b/site/content/en/docs/manual/advanced/formats/format-cityscapes.md new file mode 100644 index 00000000000..e8b70aba03e --- /dev/null +++ b/site/content/en/docs/manual/advanced/formats/format-cityscapes.md @@ -0,0 +1,142 @@ +--- +linkTitle: 'Cityscapes' +weight: 16 +--- + +# [Cityscapes](https://www.cityscapes-dataset.com/login/) + +- [Format specification](https://github.com/mcordts/cityscapesScripts#the-cityscapes-dataset) + +- Supported annotations + + - Polygons (segmentation task) + +- Supported attributes + - 'is_crowd' (boolean, should be defined for labels as `checkbox` -es) + Specifies if the annotation label can distinguish between different instances. + If False, the annotation id field encodes the instance id. + +# Cityscapes export + +Downloaded file: a zip archive of the following structure: + +``` +. +├── label_color.txt +├── gtFine +│ ├── +│ │ └── +│ │ ├── image_0_gtFine_instanceIds.png +│ │ ├── image_0_gtFine_color.png +│ │ ├── image_0_gtFine_labelIds.png +│ │ ├── image_1_gtFine_instanceIds.png +│ │ ├── image_1_gtFine_color.png +│ │ ├── image_1_gtFine_labelIds.png +│ │ ├── ... +└── imgsFine # if saving images was requested + └── leftImg8bit + ├── + │ └── + │ ├── image_0_leftImg8bit.png + │ ├── image_1_leftImg8bit.png + │ ├── ... +``` + +- `label_color.txt` a file that describes the color for each label + +``` +# label_color.txt example +# r g b label_name +0 0 0 background +0 255 0 tree +... +``` + +- `*_gtFine_color.png` class labels encoded by its color. +- `*_gtFine_labelIds.png` class labels are encoded by its index. +- `*_gtFine_instanceIds.png` class and instance labels encoded + by an instance ID. The pixel values encode class and the individual instance: + the integer part of a division by 1000 of each ID provides class ID, + the remainder is the instance ID. If a certain annotation describes multiple + instances, then the pixels have the regular ID of that class + +# Cityscapes annotations import + +Uploaded file: a zip archive with the following structure: + +``` +. +├── label_color.txt # optional +└── gtFine + └── + ├── image_0_gtFine_instanceIds.png + ├── image_1_gtFine_instanceIds.png + ├── ... +``` + +# Creating task with Cityscapes dataset + +Create a task with the labels you need +or you can use the labels and colors of the original dataset. +To work with the Cityscapes format, you must have a black color label +for the background. + +Original Cityscapes color map: + +
+ +```JSON +[ + {"name": "unlabeled", "color": "#000000", "attributes": []}, + {"name": "egovehicle", "color": "#000000", "attributes": []}, + {"name": "rectificationborder", "color": "#000000", "attributes": []}, + {"name": "outofroi", "color": "#000000", "attributes": []}, + {"name": "static", "color": "#000000", "attributes": []}, + {"name": "dynamic", "color": "#6f4a00", "attributes": []}, + {"name": "ground", "color": "#510051", "attributes": []}, + {"name": "road", "color": "#804080", "attributes": []}, + {"name": "sidewalk", "color": "#f423e8", "attributes": []}, + {"name": "parking", "color": "#faaaa0", "attributes": []}, + {"name": "railtrack", "color": "#e6968c", "attributes": []}, + {"name": "building", "color": "#464646", "attributes": []}, + {"name": "wall", "color": "#66669c", "attributes": []}, + {"name": "fence", "color": "#be9999", "attributes": []}, + {"name": "guardrail", "color": "#b4a5b4", "attributes": []}, + {"name": "bridge", "color": "#966464", "attributes": []}, + {"name": "tunnel", "color": "#96785a", "attributes": []}, + {"name": "pole", "color": "#999999", "attributes": []}, + {"name": "polegroup", "color": "#999999", "attributes": []}, + {"name": "trafficlight", "color": "#faaa1e", "attributes": []}, + {"name": "trafficsign", "color": "#dcdc00", "attributes": []}, + {"name": "vegetation", "color": "#6b8e23", "attributes": []}, + {"name": "terrain", "color": "#98fb98", "attributes": []}, + {"name": "sky", "color": "#4682b4", "attributes": []}, + {"name": "person", "color": "#dc143c", "attributes": []}, + {"name": "rider", "color": "#ff0000", "attributes": []}, + {"name": "car", "color": "#00008e", "attributes": []}, + {"name": "truck", "color": "#000046", "attributes": []}, + {"name": "bus", "color": "#003c64", "attributes": []}, + {"name": "caravan", "color": "#00005a", "attributes": []}, + {"name": "trailer", "color": "#00006e", "attributes": []}, + {"name": "train", "color": "#005064", "attributes": []}, + {"name": "motorcycle", "color": "#0000e6", "attributes": []}, + {"name": "bicycle", "color": "#770b20", "attributes": []}, + {"name": "licenseplate", "color": "#00000e", "attributes": []} +] + +``` + +
+ +Upload images when creating a task: + +``` +images.zip/ + ├── image_0.jpg + ├── image_1.jpg + ├── ... + +``` + +After creating the task, upload the Cityscapes annotations as described +in the previous section. From 51be456f23ec89eaf1667fe48b8e9e4163aa5f91 Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Wed, 29 Sep 2021 16:19:35 +0300 Subject: [PATCH 03/10] Handle colormap more carefuly --- .../dataset_manager/formats/cityscapes.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/cityscapes.py b/cvat/apps/dataset_manager/formats/cityscapes.py index b5c393a96fe..607f849718c 100644 --- a/cvat/apps/dataset_manager/formats/cityscapes.py +++ b/cvat/apps/dataset_manager/formats/cityscapes.py @@ -6,24 +6,29 @@ from tempfile import TemporaryDirectory from datumaro.components.dataset import Dataset +from datumaro.plugins.cityscapes_format import write_label_map from pyunpack import Archive from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, - ProjectData, import_dm_annotations) + import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer +from .utils import make_colormap @exporter(name='Cityscapes', ext='ZIP', version='1.0') def _export(dst_file, instance_data, save_images=False): dataset = Dataset.from_extractors(GetCVATDataExtractor( instance_data, include_images=save_images), env=dm_env) + dataset.transform('polygons_to_masks') + dataset.transform('boxes_to_masks') + dataset.transform('merge_instance_segments') with TemporaryDirectory() as temp_dir: - dataset.transform('polygons_to_masks') - dataset.transform('boxes_to_masks') dataset.export(temp_dir, 'cityscapes', save_images=save_images, - label_map='source') + apply_colormap=True, label_map={label: info[0] + for label, info in make_colormap(instance_data).items()}) + make_zip_archive(temp_dir, dst_file) @importer(name='Cityscapes', ext='ZIP', version='1.0') @@ -33,20 +38,10 @@ def _import(src_file, instance_data): labelmap_file = osp.join(tmp_dir, 'label_colors.txt') if not osp.isfile(labelmap_file): - labels_meta = instance_data.meta['project']['labels'] \ - if isinstance(instance_data, ProjectData) \ - else instance_data.meta['task']['labels'] - - labels = ('%s %s %s %s' % ( - int(label['color'][1:3], base=16), - int(label['color'][3:5], base=16), - int(label['color'][5:7], base=16), - label['name']) - for _, label in labels_meta - ) - with open(labelmap_file, 'w') as f: - f.write('\n'.join(labels)) + colormap = {label: info[0] + for label, info in make_colormap(instance_data).items()} + write_label_map(labelmap_file, colormap) dataset = Dataset.import_from(tmp_dir, 'cityscapes', env=dm_env) dataset.transform('masks_to_polygons') - import_dm_annotations(dataset, instance_data) \ No newline at end of file + import_dm_annotations(dataset, instance_data) From de0a9e5f1cdf4d0ad3bc8f9ac9ea8adeb30daf7a Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Wed, 29 Sep 2021 16:23:50 +0300 Subject: [PATCH 04/10] Add tests --- cvat/apps/dataset_manager/formats/utils.py | 2 +- .../tests/assets/annotations.json | 28 ++++- .../dataset_manager/tests/assets/tasks.json | 58 +++++++++- .../dataset_manager/tests/test_formats.py | 22 ++-- .../tests/test_rest_api_formats.py | 101 +++++++++--------- 5 files changed, 144 insertions(+), 67 deletions(-) diff --git a/cvat/apps/dataset_manager/formats/utils.py b/cvat/apps/dataset_manager/formats/utils.py index 0d545e46583..df3de855ed6 100644 --- a/cvat/apps/dataset_manager/formats/utils.py +++ b/cvat/apps/dataset_manager/formats/utils.py @@ -5,7 +5,7 @@ import os.path as osp from hashlib import blake2s -from datumaro.cli.util import make_file_name +from datumaro.util.os_util import make_file_name def get_color_from_index(index): diff --git a/cvat/apps/dataset_manager/tests/assets/annotations.json b/cvat/apps/dataset_manager/tests/assets/annotations.json index 2a7a0e212f8..fb0cc6190b2 100644 --- a/cvat/apps/dataset_manager/tests/assets/annotations.json +++ b/cvat/apps/dataset_manager/tests/assets/annotations.json @@ -244,7 +244,7 @@ "type": "polygon", "occluded": false, "z_order": 0, - "points": [35.0, 22.5, 53.32, 30.63, 22.34, 29.45, 47.43, 38.21], + "points": [35.0, 22.5, 53.32, 30.63, 22.34, 29.45, 47.43, 38.21], "frame": 0, "label_id": null, "group": 0, @@ -920,7 +920,7 @@ } ] }, - "CVAT for images 1.1 tag": { + "CVAT for images 1.1 tag": { "version": 0, "tags": [ { @@ -982,7 +982,7 @@ "shapes": [], "tracks": [] }, - "CVAT for images 1.1 different types": { + "CVAT for images 1.1 different types": { "version": 0, "tags": [], "shapes": [ @@ -1028,7 +1028,7 @@ } ] }, - "CVAT for video 1.1 polygon": { + "CVAT for video 1.1 polygon": { "version": 0, "tags": [], "shapes": [], @@ -1069,7 +1069,7 @@ ], "attributes": [] }, - { + { "frame": 0, "label_id": null, "group": 1, @@ -1196,5 +1196,23 @@ } ], "tracks": [] + }, + "Cityscapes 1.0": { + "version": 0, + "tags": [], + "shapes": [ + { + "type": "polygon", + "occluded": false, + "z_order": 0, + "points": [11.9, 21.5, 35.2, 21.9, 33.6, 31.9, 12.4, 30.47], + "frame": 0, + "label_id": null, + "group": 0, + "source": "manual", + "attributes": [] + } + ], + "tracks": [] } } diff --git a/cvat/apps/dataset_manager/tests/assets/tasks.json b/cvat/apps/dataset_manager/tests/assets/tasks.json index 23ea55e2ae7..f2a168599ae 100644 --- a/cvat/apps/dataset_manager/tests/assets/tasks.json +++ b/cvat/apps/dataset_manager/tests/assets/tasks.json @@ -54,7 +54,7 @@ } ] }, - "icdar_localization_and_recognition": { + "ICDAR Localization 1.0": { "name": "icdar localization/recogntion task", "overlap": 0, "segment_size": 100, @@ -74,7 +74,27 @@ } ] }, - "icdar_segmentation": { + "ICDAR Recognition 1.0": { + "name": "icdar localization/recogntion task", + "overlap": 0, + "segment_size": 100, + "owner_id": 1, + "assignee_id": 2, + "labels": [ + { + "name": "icdar", + "attributes": [ + { + "name": "text", + "mutable": false, + "input_type": "text", + "values": ["word_1", "word_2", "word_3"] + } + ] + } + ] + }, + "ICDAR Segmentation 1.0": { "name": "icdar segmentation task", "overlap": 0, "segment_size": 100, @@ -112,7 +132,7 @@ } ] }, - "market1501": { + "Market-1501 1.0": { "name": "market1501 task", "overlap": 0, "segment_size": 100, @@ -144,6 +164,38 @@ } ] }, + "Cityscapes 1.0": { + "name": "cityscapes task", + "overlap": 0, + "segment_size": 100, + "owner_id": 1, + "assignee_id": 2, + "labels": [ + { + "name": "car", + "color": "#2080c0", + "attributes": [ + { + "name": "is_crowd", + "mutable": false, + "input_type": "checkbox", + "default_value": "false", + "values": ["false", "true"] + } + ] + }, + { + "name": "person", + "color": "#c06060", + "attributes": [] + }, + { + "name": "background", + "color": "#000000", + "attributes": [] + } + ] + }, "wrong_checkbox_value": { "name": "wrong checkbox value task", "overlap": 0, diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index 69c7e34797e..ce7dee7e5e6 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -27,7 +27,7 @@ from cvat.apps.engine.models import Task -def generate_image_file(filename, size=(100, 50)): +def generate_image_file(filename, size=(100, 100)): f = BytesIO() image = Image.new('RGB', size=size) image.save(f, 'jpeg') @@ -294,8 +294,8 @@ def test_export_formats_query(self): 'ICDAR Localization 1.0', 'ICDAR Segmentation 1.0', 'Kitti Raw Format 1.0', - 'Sly Point Cloud Format 1.0' - + 'Sly Point Cloud Format 1.0', + 'Cityscapes 1.0' }) def test_import_formats_query(self): @@ -321,7 +321,8 @@ def test_import_formats_query(self): 'ICDAR Localization 1.0', 'ICDAR Segmentation 1.0', 'Kitti Raw Format 1.0', - 'Sly Point Cloud Format 1.0' + 'Sly Point Cloud Format 1.0', + 'Cityscapes 1.0' }) def test_exports(self): @@ -368,6 +369,7 @@ def test_empty_images_are_exported(self): ('ICDAR Recognition 1.0', 'icdar_word_recognition'), ('ICDAR Localization 1.0', 'icdar_text_localization'), ('ICDAR Segmentation 1.0', 'icdar_text_segmentation'), + # ('Cityscapes 1.0', 'cityscapes'), does not support, empty annotations ]: with self.subTest(format=format_name): if not dm.formats.registry.EXPORT_FORMATS[format_name].ENABLED: @@ -698,6 +700,10 @@ def _generate_task(self, images, annotation_format, **overrides): } ] }, + { + "name": "background", + "attributes": [], + }, {"name": "person"} ] @@ -834,7 +840,7 @@ def _generate_annotations(self, task, annotation_format): { "frame": 0, "attributes": [], - "points": [1.0, 2.1, 10.6, 53.22, 100, 300.222], + "points": [1.0, 2.1, 10.6, 53.22, 90, 90.222], "type": "polygon", "occluded": False, "outside": False @@ -875,10 +881,10 @@ def _generate_annotations(self, task, annotation_format): elif annotation_format == "MOTS PNG 1.0": tracks = [track_wo_attrs] else: - shapes = [rectangle_shape_wo_attrs, + shapes = [rectangle_shape_wo_attrs, \ rectangle_shape_with_attrs] - tags = tag_wo_attrs - tracks = track_wo_attrs + tags = [tag_wo_attrs] + tracks = [track_wo_attrs] annotations = { "version": 0, diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index f20bb02f00a..e8ebd6b2eb6 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -355,12 +355,12 @@ def test_api_v1_dump_and_upload_annotations_with_objects_type_is_shape(self): with self.subTest(format=dump_format_name): images = self._generate_task_images(3) # create task with annotations - if dump_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], images) - elif dump_format_name in ["ICDAR Localization 1.0", "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], images) - elif dump_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], images) + if dump_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[dump_format_name], images) else: task = self._create_task(tasks["main"], images) task_id = task["id"] @@ -368,7 +368,7 @@ def test_api_v1_dump_and_upload_annotations_with_objects_type_is_shape(self): "MOT 1.1", "MOTS PNG 1.0", \ "PASCAL VOC 1.1", "Segmentation mask 1.1", \ "TFRecord 1.0", "YOLO 1.1", "ImageNet 1.0", \ - "WiderFace 1.0", "VGGFace2 1.0", \ + "WiderFace 1.0", "VGGFace2 1.0", "Cityscapes 1.0"\ ]: self._create_annotations(task, dump_format_name, "default") else: @@ -417,12 +417,12 @@ def test_api_v1_dump_and_upload_annotations_with_objects_type_is_shape(self): for user, edata in list(expected.items()): # remove all annotations from task (create new task without annotation) images = self._generate_task_images(3) - if upload_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], images) - elif upload_format_name in ["ICDAR Localization 1.0", "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], images) - elif upload_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], images) + if upload_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[upload_format_name], images) else: task = self._create_task(tasks["main"], images) task_id = task["id"] @@ -460,12 +460,12 @@ def test_api_v1_dump_annotations_with_objects_type_is_track(self): with self.subTest(format=dump_format_name): # create task with annotations video = self._generate_task_videos(1) - if dump_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], video) - elif dump_format_name in ["ICDAR Localization 1.0", "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], video) - elif dump_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], video) + if dump_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[dump_format_name], video) else: task = self._create_task(tasks["main"], video) task_id = task["id"] @@ -474,7 +474,7 @@ def test_api_v1_dump_annotations_with_objects_type_is_track(self): "MOT 1.1", "MOTS PNG 1.0", \ "PASCAL VOC 1.1", "Segmentation mask 1.1", \ "TFRecord 1.0", "YOLO 1.1", "ImageNet 1.0", \ - "WiderFace 1.0", "VGGFace2 1.0", \ + "WiderFace 1.0", "VGGFace2 1.0", "Cityscapes 1.0" \ ]: self._create_annotations(task, dump_format_name, "default") else: @@ -521,12 +521,12 @@ def test_api_v1_dump_annotations_with_objects_type_is_track(self): for user, edata in list(expected.items()): # remove all annotations from task (create new task without annotation) video = self._generate_task_videos(1) - if upload_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], video) - elif upload_format_name in ["ICDAR Localization 1.0", "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], video) - elif upload_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], video) + if upload_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[upload_format_name], video) else: task = self._create_task(tasks["main"], video) task_id = task["id"] @@ -802,12 +802,12 @@ def test_api_v1_export_dataset(self): with self.subTest(format=dump_format_name): images = self._generate_task_images(3) # create task with annotations - if dump_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], images) - elif dump_format_name in ["ICDAR Localization 1.0", "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], images) - elif dump_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], images) + if dump_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[dump_format_name], images) else: task = self._create_task(tasks["main"], images) task_id = task["id"] @@ -871,6 +871,7 @@ def test_api_v1_dump_empty_frames(self): with self.subTest(format=upload_format_name): if upload_format_name in [ "MOTS PNG 1.0", # issue #2925 and changed points values + "Cityscapes 1.0" # formats doesn't support empty annotations ]: self.skipTest("Format is fail") images = self._generate_task_images(3) @@ -898,18 +899,18 @@ def test_api_v1_rewriting_annotations(self): if dump_format_name in [ "MOTS PNG 1.0", # issue #2925 and changed points values "Datumaro 1.0", # Datumaro 1.0 is not in the list of import format - 'Kitti Raw Format 1.0', - 'Sly Point Cloud Format 1.0' - + "Kitti Raw Format 1.0", + "Sly Point Cloud Format 1.0", + "Cityscapes 1.0" # expanding annotations due to background mask ]: self.skipTest("Format is fail") images = self._generate_task_images(3) - if dump_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], images) - elif dump_format_name in ["ICDAR Localization 1.0", "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], images) - elif dump_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], images) + if dump_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[dump_format_name], images) else: task = self._create_task(tasks["main"], images) task_id = task["id"] @@ -917,7 +918,7 @@ def test_api_v1_rewriting_annotations(self): "MOT 1.1", "MOTS PNG 1.0", \ "PASCAL VOC 1.1", "Segmentation mask 1.1", \ "TFRecord 1.0", "YOLO 1.1", "ImageNet 1.0", \ - "WiderFace 1.0", "VGGFace2 1.0", \ + "WiderFace 1.0", "VGGFace2 1.0", "Cityscapes 1.0"\ ]: self._create_annotations(task, dump_format_name, "default") else: @@ -1008,19 +1009,19 @@ def test_api_v1_tasks_annotations_dump_and_upload_with_datumaro(self): "Segmentation mask 1.1", # changed points values "ICDAR Segmentation 1.0", # changed points values 'Kitti Raw Format 1.0', - 'Sly Point Cloud Format 1.0' + 'Sly Point Cloud Format 1.0', + 'Cityscapes 1.0' # changed points value ]: self.skipTest("Format is fail") # create task images = self._generate_task_images(3) - if dump_format_name == "Market-1501 1.0": - task = self._create_task(tasks["market1501"], images) - elif dump_format_name in ["ICDAR Localization 1.0", - "ICDAR Recognition 1.0"]: - task = self._create_task(tasks["icdar_localization_and_recognition"], images) - elif dump_format_name == "ICDAR Segmentation 1.0": - task = self._create_task(tasks["icdar_segmentation"], images) + if dump_format_name in [ + "Market-1501 1.0", "Cityscapes 1.0", \ + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ + "ICDAR Segmentation 1.0" + ]: + task = self._create_task(tasks[dump_format_name], images) else: task = self._create_task(tasks["main"], images) From a57adef1c687912c541d5d048b402d72a838a9ba Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Wed, 29 Sep 2021 16:24:31 +0300 Subject: [PATCH 05/10] Correct import --- cvat/apps/dataset_manager/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index 4f51c69a91e..12708242909 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -8,7 +8,7 @@ from datetime import timedelta import django_rq -from datumaro.cli.util import make_file_name +from datumaro.util.os_util import make_file_name from datumaro.util import to_snake_case from django.utils import timezone From 2c6d67a9f1c9f8aad20a0f9f10ca844f90c01d67 Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Fri, 29 Oct 2021 16:34:19 +0300 Subject: [PATCH 06/10] Fix tests --- cvat/apps/dataset_manager/tests/test_formats.py | 2 +- cvat/apps/dataset_manager/tests/test_rest_api_formats.py | 2 +- cvat/apps/engine/tests/test_rest_api.py | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index 60c7d6e5a46..94eb2a87209 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -323,7 +323,7 @@ def test_import_formats_query(self): 'ICDAR Segmentation 1.0', 'Kitti Raw Format 1.0', 'Sly Point Cloud Format 1.0', - 'Cityscapes 1.0' + 'Cityscapes 1.0', 'Datumaro 1.0', 'Datumaro 3D 1.0' }) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 8e17a9e203c..b3a946bef9d 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -1011,7 +1011,7 @@ def test_api_v1_tasks_annotations_dump_and_upload_with_datumaro(self): "ICDAR Segmentation 1.0", # changed points values 'Kitti Raw Format 1.0', 'Sly Point Cloud Format 1.0', - 'Cityscapes 1.0' # changed points value + 'Cityscapes 1.0', # changed points value 'Datumaro 3D 1.0' ]: self.skipTest("Format is fail") diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 6d4769d5bcd..9b4adac4cb6 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -4811,6 +4811,10 @@ def _get_initial_annotation(annotation_format): annotations["shapes"] = points_wo_attrs \ + rectangle_shapes_wo_attrs + elif annotation_format == "Cityscapes 1.0": + annotations["shapes"] = points_wo_attrs \ + + rectangle_shapes_wo_attrs + elif annotation_format == "Market-1501 1.0": tags_with_attrs = [{ "frame": 1, From 403c5f3246109311f72646b90f85a4a195b73893 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 2 Nov 2021 13:05:24 +0300 Subject: [PATCH 07/10] Fix cypress test --- ...ue_2473_import_annotations_frames_dots_in_name.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js index c82494d46de..9bc53a3d939 100644 --- a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js +++ b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js @@ -70,9 +70,14 @@ context('Import annotations for frames with dots in name.', { browser: '!firefox cy.get('.cvat-modal-export-task').find('.cvat-modal-export-select').click(); cy.get('.ant-select-dropdown') .not('.ant-select-dropdown-hidden') - .trigger('wheel', {deltaY: 700}) - .contains('.cvat-modal-export-option-item', dumpType) - .click(); + .within(() => { + cy.get('.rc-virtual-list-holder') + .trigger('wheel', { deltaY: 1000 }) + .trigger('wheel', { deltaY: 1000 }) + .contains('.cvat-modal-export-option-item', dumpType) + .should('be.visible') + .click(); + }); cy.get('.cvat-modal-export-select').should('contain.text', dumpType); cy.get('.cvat-modal-export-task').contains('button', 'OK').click(); cy.wait('@dumpAnnotations', { timeout: 5000 }).its('response.statusCode').should('equal', 202); @@ -92,6 +97,7 @@ context('Import annotations for frames with dots in name.', { browser: '!firefox it('Upload annotation with YOLO format to job.', () => { cy.interactMenu('Upload annotations'); cy.contains('.cvat-menu-load-submenu-item', dumpType.split(' ')[0]) + .scrollIntoView() .should('be.visible') .within(() => { cy.get('.cvat-menu-load-submenu-item-button') From 20cd3385affc4d8ba6afce3a54ecd20fcc866d7f Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 2 Nov 2021 13:10:42 +0300 Subject: [PATCH 08/10] Fixed linter issues --- .../issue_2473_import_annotations_frames_dots_in_name.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js index 9bc53a3d939..2d12e325bb0 100644 --- a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js +++ b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js @@ -25,7 +25,7 @@ context('Import annotations for frames with dots in name.', { browser: '!firefox const createRectangleShape2Points = { points: 'By 2 Points', type: 'Shape', - labelName: labelName, + labelName, firstX: 250, firstY: 350, secondX: 350, From 1663ee33dfe1bc8bce75857e972925c2271b8599 Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Wed, 3 Nov 2021 11:07:41 +0300 Subject: [PATCH 09/10] Update CHAGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b453705f91e..0e52a2d34dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a tutorial on attaching cloud storage AWS-S3 () and Azure Blob Container () - The feature to remove annotations in a specified range of frames () +- Add Cityscapes format () ### Changed From d6d2d28d0b3587608a44486713df9b8eebd3b6c8 Mon Sep 17 00:00:00 2001 From: "kirill.sizov" Date: Mon, 8 Nov 2021 12:15:09 +0300 Subject: [PATCH 10/10] Delete invalid code --- cvat/apps/dataset_manager/tests/test_formats.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index 94eb2a87209..10225065c23 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -383,16 +383,6 @@ def test_empty_images_are_exported(self): def check(file_path): def load_dataset(src): - if importer_name == 'datumaro_project': - project = datumaro.components.project. \ - Project.load(src) - - # NOTE: can't import cvat.utils.cli - # for whatever reason, so remove the dependency - # - project.config.remove('sources') - - return project.make_dataset() return datumaro.components.dataset. \ Dataset.import_from(src, importer_name, env=dm_env)