(WIP) Tfdsv4 carla mot

armory/datasets/adversarial/carla_multi_object_tracking_dev/__init__.py

@@ -0,0 +1,3 @@
+"""carla_multi_object_tracking_dev dataset."""
+
+from .carla_multi_object_tracking_dev import CarlaMultiObjectTrackingDev

armory/data/adversarial/carla_mot_dev.py → armory/datasets/adversarial/carla_multi_object_tracking_dev/carla_multi_object_tracking_dev.py

@@ -1,11 +1,11 @@
-"""carla_mot_dev dataset."""
+"""carla_multi_object_tracking_dev dataset."""
 
 import glob
 import os
 import re
 
 import numpy as np
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
 import tensorflow_datasets as tfds
 
 _DESCRIPTION = """
@@ -22,7 +22,7 @@
 }
 """
 
-_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz"
+_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz"
 
 
 class CarlaMOTDev(tfds.core.GeneratorBasedBuilder):
@@ -70,14 +70,9 @@ def _info(self) -> tfds.core.DatasetInfo:
 
     def _split_generators(self, dl_manager: tfds.download.DownloadManager):
         """Returns SplitGenerators."""
-        path = dl_manager.download_and_extract(_URLS)
+        path = dl_manager.download_and_extract(_URL)
 
-        return [
-            tfds.core.SplitGenerator(
-                name="dev",
-                gen_kwargs={"path": os.path.join(path, "dev")},
-            )
-        ]
+        return {"dev": self._generate_examples(path / "dev")}
 
     def _generate_examples(self, path):
         """Yields examples."""

armory/datasets/adversarial/carla_multi_object_tracking_dev/checksums.tsv

@@ -0,0 +1 @@
+https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_mot_dev_1.0.0.tar.gz	704303119	cdd4be9cd3bcb5c2f94a6628350f106deec6fdc7b6a9c05711309b2bcc814f3d	carla_mot_dev_1.0.0.tar.gz

armory/datasets/cached_datasets.json

@@ -27,6 +27,13 @@
         "url": null,
         "version": "1.0.0"
     },
+    "carla_over_obj_det_train": {
+        "sha256": "1fa2626726df4de8c0878f97ededdde90d4c55a4a845c8cee8db26a9797a2c6f",
+        "size": 14396331191,
+        "subdir": "carla_over_obj_det_train/1.0.0",
+        "url": null,
+        "version": "1.0.0"
+    },
     "carla_video_tracking_dev": {
         "sha256": "958d470dcd394928050f4123a7af05b0e389ceeec6fa0a3261df55a65e553b69",
         "size": 1281628036,
@@ -40,12 +47,6 @@
         "subdir": "carla_video_tracking_test/2.0.0",
         "url": null,
         "version": "2.0.0"
-    "carla_over_obj_det_train": {
-        "sha256": "1fa2626726df4de8c0878f97ededdde90d4c55a4a845c8cee8db26a9797a2c6f",
-        "size": 14396331191,
-        "subdir": "carla_over_obj_det_train/1.0.0",
-        "url": null,
-        "version": "1.0.0"
     },
     "cifar10": {
         "sha256": "05bf2d90665560e08ca6cf6e896b503ed4b83cd0e332d3a2e5970c9329acbaee",

armory/datasets/preprocessing.py

@@ -98,25 +98,35 @@ def carla_over_obj_det_dev(element, modality="rgb"):
     )
 
 
-def carla_video_tracking_preprocess(x, max_frames=None):
-    # Clip
+def clip(batch, max_frames=None):
     if max_frames:
         max_frames = int(max_frames)
         if max_frames <= 0:
             raise ValueError(f"max_frames {max_frames} must be > 0")
-        x = x[:max_frames, :]
-    x = tf.cast(x, tf.float32) / 255.0
-    return x
+        batch = batch[:max_frames, :]
+    return batch
 
 
-def carla_video_tracking_preprocess_labels(y, y_patch_metadata, max_frames=None):
-    # Clip
+def clip_labels(boxes, patch_metadata_dict, max_frames=None):
     if max_frames:
         max_frames = int(max_frames)
         if max_frames <= 0:
             raise ValueError(f"max_frames {max_frames} must be > 0")
-        y = y[:max_frames, :]
-        y_patch_metadata = {k: v[:max_frames, :] for (k, v) in y_patch_metadata.items()}
+        boxes = boxes[:max_frames, :]
+        patch_metadata_dict = {
+            k: v[:max_frames, :] for (k, v) in patch_metadata_dict.items()
+        }
+    return boxes, patch_metadata_dict
+
+
+def carla_video_tracking_preprocess(x, max_frames=None):
+    x = clip(x, max_frames=max_frames)
+    x = tf.cast(x, tf.float32) / 255.0
+    return x
+
+
+def carla_video_tracking_preprocess_labels(y, y_patch_metadata, max_frames=None):
+    y, y_patch_metadata = clip_labels(y, y_patch_metadata, max_frames=max_frames)
     # Update labels
     y = {"boxes": y}
     y_patch_metadata = {
@@ -146,6 +156,96 @@ def carla_over_obj_det_train(element, modality="rgb"):
     ), convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"])
 
 
+def mot_zero_index(y, y_patch_metadata):
+    if tf.rank(y) == 2:
+        y = tf.tensor_scatter_nd_update(y, [[0, 0]], [y[0, 0] - 1])
+    else:
+        for i in tf.range(tf.shape(y)[0]):
+            y_i = tf.expand_dims(y[i], axis=0)  # Add an extra dimension
+            y_i = tf.tensor_scatter_nd_update(y_i, [[0, 0]], [y_i[0, 0] - 1])
+            y_i = tf.squeeze(y_i, axis=0)  # Remove the extra dimension
+            y = tf.tensor_scatter_nd_update(y, [[i]], [y_i])
+    return y, y_patch_metadata
+
+
+def mot_array_to_coco(batch):
+    """
+    Map from 3D array (batch_size x detections x 9) to extended coco format
+        of dimension (batch_size x frames x detections_per_frame)
+    NOTE: 'image_id' is given as the frame of a video, so is not unique
+    """
+    if len(batch.shape) == 2:
+        not_batch = True
+        batch = tf.expand_dims(batch, axis=0)
+    elif len(batch.shape) == 3:
+        not_batch = False
+    else:
+        raise ValueError(f"batch.ndim {len(batch.shape)} is not in (2, 3)")
+
+    # output = tf.TensorArray(dtype=tf.float32, size=batch.shape[0], dynamic_size=False)
+    output = []
+    for i in range(batch.shape[0]):
+        array = batch[i]
+        # if not tf.math.greater(tf.shape(array)[0], 0):
+        if array.shape[0] == 0:
+            # no object detections
+            # output = output.write(i, [])
+            output.append([])
+            continue
+
+        # frames = tf.TensorArray(dtype=tf.float32, size=tf.shape(array)[0], dynamic_size=False)
+        frames = []
+        for detection in array:
+            frame = tf.lookup.StaticHashTable(
+                {
+                    # TODO: should image_id include video number as well?
+                    "image_id": tf.cast(tf.math.round(detection[0]), tf.int32),
+                    "category_id": tf.cast(tf.math.round(detection[7]), tf.int32),
+                    "bbox": tf.cast(detection[2:6], float),
+                    "score": tf.cast(detection[6], float),
+                    # The following are extended fields
+                    "object_id": tf.cast(
+                        tf.math.round(detection[1]), tf.int32
+                    ),  # for a specific object across frames
+                    "visibility": tf.cast(detection[8], float),
+                }
+            )
+            frames.append(frame)
+            # frames = frames.write(frames.size(), frame)
+        # output = output.write(i, frames)
+        output.append(frames)
+
+    if not_batch:
+        output = output[0]
+
+    raise NotImplementedError("This does not work yet")
+    return output
+
+
+def carla_mot_label_preprocessing(
+    y, y_patch_metadata, coco_format=False, max_frames=None
+):
+    y, y_patch_metadata = mot_zero_index(y, y_patch_metadata)
+    y, y_patch_metadata = clip_labels(y, y_patch_metadata, max_frames=max_frames)
+    if coco_format:
+        y = mot_array_to_coco(y)
+    y_patch_metadata = {k: tf.squeeze(v, axis=0) for k, v in y_patch_metadata.items()}
+    return y, y_patch_metadata
+
+
+@register
+def carla_multi_object_tracking_dev(element, coco_format=False, max_frames=None):
+    return carla_video_tracking_preprocess(
+        element["video"],
+        max_frames=max_frames,
+    ), carla_mot_label_preprocessing(
+        element["annotations"],
+        element["patch_metadata"],
+        coco_format=coco_format,
+        max_frames=max_frames,
+    )
+
+
 @register
 def xview(element):
     return image_to_canon(element["image"]), convert_tf_obj_det_label_to_pytorch(

armory/utils/config_loading.py

@@ -31,6 +31,7 @@
 from armory.art_experimental.attacks import patch
 from armory.art_experimental.attacks.sweep import SweepAttack
 from armory.datasets.generator import ArmoryDataGenerator
+from armory.data.datasets import EvalGenerator  # TODO: Remove before PR merge
 from armory.data.utils import maybe_download_weights_from_s3
 from armory.utils import labels
 

scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json

@@ -16,8 +16,8 @@
     "dataset": {
         "test": {
             "batch_size": 1,
-            "split": "dev",
-            "name": "carla_video_tracking_dev"
+            "name": "carla_video_tracking_dev",
+            "split": "dev"
         }
     },
     "defense": {

scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json

@@ -16,8 +16,8 @@
     "dataset": {
         "test": {
             "batch_size": 1,
-            "split": "dev",
-            "name": "carla_video_tracking_dev"
+            "name": "carla_video_tracking_dev",
+            "split": "dev"
         }
     },
     "defense": null,

scenario_configs/eval6/carla_mot/carla_mot_adversarialpatch_undefended.json

@@ -17,12 +17,14 @@
         "use_label": true
     },
     "dataset": {
-        "batch_size": 1,
-        "coco_format": true,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_multi_object_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_multi_object_tracking_dev",
+            "preprocessor_kwargs": {
+                "coco_format": true
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {

scenario_configs/eval6/carla_mot/carla_mot_dpatch_defended.json

@@ -14,12 +14,14 @@
         "use_label": false
     },
     "dataset": {
-        "batch_size": 1,
-        "coco_format": true,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_multi_object_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_multi_object_tracking_dev",
+            "preprocessor_kwargs": {
+                "coco_format": true
+            },
+            "split": "dev"
+        }
     },
     "defense": {
         "kwargs": {

scenario_configs/eval6/carla_mot/carla_mot_dpatch_undefended.json

@@ -14,12 +14,14 @@
         "use_label": false
     },
     "dataset": {
-        "batch_size": 1,
-        "coco_format": true,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_multi_object_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_multi_object_tracking_dev",
+            "preprocessor_kwargs": {
+                "coco_format": true
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {