Add Open Images format

cvat/apps/dataset_manager/formats/openimages.py

@@ -0,0 +1,83 @@
+# Copyright (C) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import glob
+import os.path as osp
+from tempfile import TemporaryDirectory
+
+from datumaro.components.dataset import Dataset, DatasetItem
+from datumaro.plugins.open_images_format import OpenImagesPath
+from datumaro.util.image import DEFAULT_IMAGE_META_FILE_NAME
+from pyunpack import Archive
+
+from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
+    find_dataset_root, import_dm_annotations, match_dm_item)
+from cvat.apps.dataset_manager.util import make_zip_archive
+
+from .registry import dm_env, exporter, importer
+
+
+def find_item_ids(path):
+    image_desc_patterns = (
+        OpenImagesPath.FULL_IMAGE_DESCRIPTION_FILE_NAME,
+        *OpenImagesPath.SUBSET_IMAGE_DESCRIPTION_FILE_PATTERNS
+    )
+
+    image_desc_patterns = (
+        osp.join(path, OpenImagesPath.ANNOTATIONS_DIR, pattern)
+        for pattern in image_desc_patterns
+    )
+
+    for pattern in image_desc_patterns:
+        for path in glob.glob(pattern):
+            with open(path, 'r') as desc:
+                next(desc)
+                for row in desc:
+                    yield row.split(',')[0]
+
+@exporter(name='Open Images', ext='ZIP', version='1.0')
+def _export(dst_file, task_data, save_images=False):
+    dataset = Dataset.from_extractors(GetCVATDataExtractor(
+        task_data, include_images=save_images), env=dm_env)
+    dataset.transform('polygons_to_masks')
+    dataset.transform('merge_instance_segments')
+
+    with TemporaryDirectory() as temp_dir:
+        dataset.export(temp_dir, 'open_images', save_images=save_images)
+
+        make_zip_archive(temp_dir, dst_file)
+
+@importer(name='Open Images', ext='ZIP', version='1.0')
+def _import(src_file, task_data):
+    with TemporaryDirectory() as tmp_dir:
+        Archive(src_file.name).extractall(tmp_dir)
+
+        image_meta_path = osp.join(tmp_dir, OpenImagesPath.ANNOTATIONS_DIR,
+            DEFAULT_IMAGE_META_FILE_NAME)
+        image_meta = None
+
+        if not osp.isfile(image_meta_path):
+            image_meta = {}
+            item_ids = list(find_item_ids(tmp_dir))
+
+            root_hint = find_dataset_root(
+                [DatasetItem(id=item_id) for item_id in item_ids], task_data)
+
+            for item_id in item_ids:
+                frame_info = None
+                try:
+                    frame_id = match_dm_item(DatasetItem(id=item_id),
+                        task_data, root_hint)
+                    frame_info = task_data.frame_info[frame_id]
+                except Exception: # nosec
+                    pass
+                if frame_info is not None:
+                    image_meta[item_id] = (frame_info['height'], frame_info['width'])
+
+        dataset = Dataset.import_from(tmp_dir, 'open_images',
+            image_meta=image_meta, env=dm_env)
+        dataset.transform('masks_to_polygons')
+        import_dm_annotations(dataset, task_data)
+
+

cvat/apps/dataset_manager/formats/registry.py

@@ -121,4 +121,5 @@ def make_exporter(name):
 import cvat.apps.dataset_manager.formats.icdar
 import cvat.apps.dataset_manager.formats.velodynepoint
 import cvat.apps.dataset_manager.formats.pointcloud
+import cvat.apps.dataset_manager.formats.openimages
 

cvat/apps/dataset_manager/tests/assets/annotations.json

@@ -474,6 +474,35 @@
       }
     ]
   },
+  "Open Images 1.0": {
+    "version": 0,
+    "tags": [],
+    "shapes": [
+      {
+        "type": "rectangle",
+        "occluded": false,
+        "z_order": 0,
+        "points": [1.0, 2.0, 3.0, 4.0],
+        "frame": 0,
+        "label_id": null,
+        "group": 0,
+        "source": "manual",
+        "attributes": []
+      },
+      {
+        "type": "polygon",
+        "occluded": false,
+        "z_order": 0,
+        "points": [1.0, 1.0, 1.0, 20.0, 20.0, 1.0, 20.0, 1.0, 1.0, 1.0],
+        "frame": 0,
+        "label_id": null,
+        "group": 0,
+        "source": "manual",
+        "attributes": []
+      }
+    ],
+    "tracks": []
+  },
   "PASCAL VOC 1.1": {
     "version": 0,
     "tags": [

cvat/apps/dataset_manager/tests/test_formats.py

@@ -295,8 +295,8 @@ def test_export_formats_query(self):
             'ICDAR Localization 1.0',
             'ICDAR Segmentation 1.0',
             'Kitti Raw Format 1.0',
-            'Sly Point Cloud Format 1.0'
-
+            'Sly Point Cloud Format 1.0',
+            'Open Images 1.0'
         })
 
     def test_import_formats_query(self):
@@ -323,6 +323,7 @@ def test_import_formats_query(self):
             'ICDAR Segmentation 1.0',
             'Kitti Raw Format 1.0',
             'Sly Point Cloud Format 1.0',
+            'Open Images 1.0',
             'Datumaro 1.0',
             'Datumaro 3D 1.0'
         })

cvat/apps/dataset_manager/tests/test_rest_api_formats.py

@@ -916,7 +916,8 @@ def test_api_v1_rewriting_annotations(self):
                         "MOT 1.1", "MOTS PNG 1.0", \
                         "PASCAL VOC 1.1", "Segmentation mask 1.1", \
                         "TFRecord 1.0", "YOLO 1.1", "ImageNet 1.0", \
-                        "WiderFace 1.0", "VGGFace2 1.0", "Datumaro 1.0" \
+                        "WiderFace 1.0", "VGGFace2 1.0", "Datumaro 1.0",\
+                        "Open Images 1.0" \
                     ]:
                         self._create_annotations(task, dump_format_name, "default")
                     else:
@@ -1005,6 +1006,7 @@ def test_api_v1_tasks_annotations_dump_and_upload_with_datumaro(self):
                         "MOTS PNG 1.0", # changed points values
                         "Segmentation mask 1.1", # changed points values
                         "ICDAR Segmentation 1.0", # changed points values
+                        "Open Images 1.0", # changed points values
                         'Kitti Raw Format 1.0',
                         'Sly Point Cloud Format 1.0',
                         'Datumaro 3D 1.0'
@@ -1028,7 +1030,8 @@ def test_api_v1_tasks_annotations_dump_and_upload_with_datumaro(self):
                         "MOT 1.1", "MOTS PNG 1.0", \
                         "PASCAL VOC 1.1", "Segmentation mask 1.1", \
                         "TFRecord 1.0", "YOLO 1.1", "ImageNet 1.0", \
-                        "WiderFace 1.0", "VGGFace2 1.0", "Datumaro 1.0", \
+                        "WiderFace 1.0", "VGGFace2 1.0", "Open Images 1.0", \
+                        "Datumaro 1.0", \
                     ]:
                         self._create_annotations(task, dump_format_name, "default")
                     else:

cvat/apps/engine/tests/test_rest_api.py

@@ -4811,6 +4811,11 @@ def _get_initial_annotation(annotation_format):
                 annotations["shapes"] = points_wo_attrs \
                                       + rectangle_shapes_wo_attrs
 
+            elif annotation_format == "Open Images 1.0":
+                annotations["tags"] = tags_wo_attrs
+                annotations["shapes"] = rectangle_shapes_wo_attrs \
+                                      + polygon_shapes_wo_attrs
+
             elif annotation_format == "Market-1501 1.0":
                 tags_with_attrs = [{
                     "frame": 1,

site/content/en/docs/manual/advanced/formats/format-openimages.md

@@ -0,0 +1,108 @@
+---
+linkTitle: 'Open Images'
+weight: 15
+---
+
+# [Open Images](https://storage.googleapis.com/openimages/web/index.html)
+
+- [Format specification](https://storage.googleapis.com/openimages/web/download.html)
+
+- Supported annotations:
+
+  - Rectangles (detection task)
+  - Tags (classification task)
+  - Polygons (segmentation task)
+
+- Supported attributes:
+
+  - Labels
+
+    - `score` (should be defined for labels as `text` or `number`).
+      The confidence level from 0 to 1.
+
+  - Bounding boxes
+
+    - `score` (should be defined for labels as `text` or `number`).
+      The confidence level from 0 to 1.
+    - `occluded` (both UI option and a separate attribute).
+      Whether the object is occluded by another object.
+    - `truncated` (should be defined for labels as `checkbox` -es).
+      Whether the object extends beyond the boundary of the image.
+    - `is_group_of` (should be defined for labels as `checkbox` -es).
+      Whether the object represents a group of objects of the same class.
+    - `is_depiction` (should be defined for labels as `checkbox` -es).
+      Whether the object is a depiction (such as a drawing)
+      rather than a real object.
+    - `is_inside` (should be defined for labels as `checkbox` -es).
+      Whether the object is seen from the inside.
+
+  - Masks
+    - `box_id` (should be defined for labels as `text`).
+      An identifier for the bounding box associated with the mask.
+    - `predicted_iou` (should be defined for labels as `text` or `number`).
+      Predicted IoU value with respect to the ground truth.
+
+## Open Images export
+
+Downloaded file: a zip archive of the following structure:
+
+```
+└─ taskname.zip/
+    ├── annotations/
+    │   ├── bbox_labels_600_hierarchy.json
+    │   ├── class-descriptions.csv
+    |   ├── images.meta  # additional file with information about image sizes
+    │   ├── <subset_name>-image_ids_and_rotation.csv
+    │   ├── <subset_name>-annotations-bbox.csv
+    │   ├── <subset_name>-annotations-human-imagelabels.csv
+    │   └── <subset_name>-annotations-object-segmentation.csv
+    ├── images/
+    │   ├── subset1/
+    │   │   ├── <image_name101.jpg>
+    │   │   ├── <image_name102.jpg>
+    │   │   └── ...
+    │   ├── subset2/
+    │   │   ├── <image_name201.jpg>
+    │   │   ├── <image_name202.jpg>
+    │   │   └── ...
+    |   ├── ...
+    └── masks/
+        ├── subset1/
+        │   ├── <mask_name101.png>
+        │   ├── <mask_name102.png>
+        │   └── ...
+        ├── subset2/
+        │   ├── <mask_name201.png>
+        │   ├── <mask_name202.png>
+        │   └── ...
+        ├── ...
+```
+
+## Open Images import
+
+Uploaded file: a zip archive of the following structure:
+
+```
+└─ upload.zip/
+    ├── annotations/
+    │   ├── bbox_labels_600_hierarchy.json
+    │   ├── class-descriptions.csv
+    |   ├── images.meta  # optional, file with information about image sizes
+    │   ├── <subset_name>-image_ids_and_rotation.csv
+    │   ├── <subset_name>-annotations-bbox.csv
+    │   ├── <subset_name>-annotations-human-imagelabels.csv
+    │   └── <subset_name>-annotations-object-segmentation.csv
+    └── masks/
+        ├── subset1/
+        │   ├── <mask_name101.png>
+        │   ├── <mask_name102.png>
+        │   └── ...
+        ├── subset2/
+        │   ├── <mask_name201.png>
+        │   ├── <mask_name202.png>
+        │   └── ...
+        ├── ...
+```
+
+Image ids in the `<subset_name>-image_ids_and_rotation.csv` should match with
+image names in the task.