carla OD dev set + attack integration

armory/baseline_models/pytorch/carla_single_modality_object_detection_frcnn.py

@@ -19,19 +19,14 @@ def get_art_model(
 ) -> PyTorchFasterRCNN:
 
     if weights_path:
-        assert (
-            model_kwargs.get("num_classes", None) == 4
-        ), "model trained on CARLA data outputs predictions for 4 classes"
-        assert not model_kwargs.get(
-            "pretrained", False
-        ), "model trained on CARLA data should not use COCO-pretrained weights"
-    else:
-        assert (
-            model_kwargs.get("num_classes", None) == 91
-        ), "model without predefined weights should use COCO classes"
-        assert model_kwargs.get(
-            "pretrained", False
-        ), "model without predefined weights should use COCO-pretrained weights"
+        assert model_kwargs.get("num_classes", None) == 4, (
+            "model trained on CARLA data outputs predictions for 4 classes, "
+            "set model_kwargs['num_classes'] to 4."
+        )
+        assert not model_kwargs.get("pretrained", False), (
+            "model trained on CARLA data should not use COCO-pretrained weights, set "
+            "model_kwargs['pretrained'] to False."
+        )
 
     model = models.detection.fasterrcnn_resnet50_fpn(**model_kwargs)
     model.to(DEVICE)

armory/data/adversarial/carla_obj_det_dev.py

@@ -25,7 +25,9 @@
 }
 """
 
-_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_obj_det_dev.tar.gz"
+# fmt: off
+_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev.tar.gz"
+# fmt: on
 
 
 class CarlaObjDetDev(tfds.core.GeneratorBasedBuilder):

armory/data/adversarial_datasets.py

@@ -38,7 +38,8 @@
 ucf101_adversarial_context = datasets.ImageContext(x_shape=(None, 112, 112, 3))
 apricot_adversarial_context = datasets.ImageContext(x_shape=(None, None, 3))
 dapricot_adversarial_context = datasets.ImageContext(x_shape=(3, None, None, 3))
-carla_obj_det_dev_context = datasets.ImageContext(x_shape=(2, 600, 800, 3))
+carla_obj_det_dev_single_modal_context = datasets.ImageContext(x_shape=(600, 800, 3))
+carla_obj_det_dev_multimodal_context = datasets.ImageContext(x_shape=(600, 800, 6))
 carla_video_tracking_context = datasets.VideoContext(
     x_shape=(None, 600, 800, 3), frame_rate=None
 )
@@ -75,10 +76,6 @@ def dapricot_canonical_preprocessing(batch):
     )
 
 
-def carla_obj_det_dev_canonical_preprocessing(batch):
-    return datasets.canonical_image_preprocess(carla_obj_det_dev_context, batch)
-
-
 def carla_video_tracking_dev_canonical_preprocessing(batch):
     return datasets.canonical_variable_image_preprocess(
         carla_video_tracking_context, batch
@@ -569,9 +566,23 @@ def carla_obj_det_dev_label_preprocessing(x, y):
     y_object, y_patch_metadata = y
     y_object = {k: np.squeeze(v, axis=0) for k, v in y_object.items()}
     y_patch_metadata = {k: np.squeeze(v, axis=0) for k, v in y_patch_metadata.items()}
+
+    # convert TF format to PyTorch format of [x1, y1, x2, y2]
+    height, width = x.shape[2:4]
+    converted_boxes = y_object["boxes"][:, [1, 0, 3, 2]]
+    converted_boxes *= [width, height, width, height]
+    y_object["boxes"] = converted_boxes
     return (y_object, y_patch_metadata)
 
 
+def carla_obj_det_dev_canonical_preprocessing(batch):
+    if batch.shape[-1] == 6:
+        context = carla_obj_det_dev_multimodal_context
+    else:
+        context = carla_obj_det_dev_single_modal_context
+    return datasets.canonical_image_preprocess(context, batch)
+
+
 def carla_obj_det_dev(
     split: str = "dev",
     epochs: int = 1,
@@ -581,7 +592,7 @@ def carla_obj_det_dev(
     label_preprocessing_fn=carla_obj_det_dev_label_preprocessing,
     cache_dataset: bool = True,
     framework: str = "numpy",
-    shuffle_files: bool = True,
+    shuffle_files: bool = False,
     **kwargs,
 ):
     """
@@ -591,6 +602,33 @@ def carla_obj_det_dev(
     if batch_size != 1:
         raise ValueError("carla_obj_det_dev batch size must be set to 1")
 
+    modality = kwargs.pop("modality", "rgb")
+    if modality not in ["rgb", "depth", "both"]:
+        raise ValueError(
+            'Unknown modality: {}.  Must be one of "rgb", "depth", or "both"'.format(
+                modality
+            )
+        )
+
+    def rgb_fn(batch):
+        return batch[:, 0]
+
+    def depth_fn(batch):
+        return batch[:, 1]
+
+    def both_fn(batch):
+        return np.concatenate((batch[:, 0], batch[:, 1]), axis=-1)
+
+    func_dict = {"rgb": rgb_fn, "depth": depth_fn, "both": both_fn}
+    mode_split_fn = func_dict[modality]
+    preprocessing_fn = datasets.preprocessing_chain(mode_split_fn, preprocessing_fn)
+
+    context = (
+        carla_obj_det_dev_multimodal_context
+        if modality == "both"
+        else carla_obj_det_dev_single_modal_context
+    )
+
     return datasets._generator_from_tfds(
         "carla_obj_det_dev:1.0.1",
         split=split,
@@ -602,7 +640,7 @@ def carla_obj_det_dev(
         cache_dataset=cache_dataset,
         framework=framework,
         shuffle_files=shuffle_files,
-        context=carla_obj_det_dev_context,
+        context=context,
         as_supervised=False,
         supervised_xy_keys=("image", ("objects", "patch_metadata")),
         **kwargs,

armory/data/cached_s3_checksums/carla_obj_det_dev.txt

@@ -1 +1 @@
-armory-public-data carla/carla_obj_det_dev_cached_1.0.0.tar.gz 2974113 858b99c3e740f7c2a6b30fbf6a9787bf607f0bd5f39083a15097e8e125a64aba
+armory-public-data carla/carla_od_dev_cached_1.0.1.tar.gz 88301496 0076fb770fa24a4ea3df5c1ccd8f91bed58f38aa6f2c8db697004f1b4706a348

armory/data/url_checksums/carla_obj_det_dev.txt

@@ -1 +1 @@
-https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_obj_det_dev.tar.gz 3068275 6b9bc96b8c3c09fead6dc9f4f9d51803073ab33760c3bfdd60e1a3fb9c300005
+https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev.tar.gz 90382428 0497a1c8f63ed1b56a11b8bc3fa6927894b2eb810fcc5be8e381cd10a013f4b0

armory/scenarios/carla_object_detection.py

@@ -0,0 +1,84 @@
+"""
+CARLA object detection
+
+Scenario Contributor: MITRE Corporation
+"""
+
+import logging
+
+from armory.scenarios.scenario import Scenario
+from armory.utils import metrics
+
+logger = logging.getLogger(__name__)
+
+
+class CarlaObjectDetectionTask(Scenario):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.skip_misclassified:
+            raise ValueError(
+                "skip_misclassified shouldn't be set for carla_video_tracking scenario"
+            )
+
+    def load_dataset(self):
+        if self.config["dataset"]["batch_size"] != 1:
+            raise ValueError("batch_size must be 1 for evaluation.")
+        super().load_dataset(eval_split_default="dev")
+
+    def run_benign(self):
+        x, y = self.x, self.y
+        y_object, y_patch_metadata = y
+
+        # convert dict to List[dict] to comply with ART format
+        y_object = [y_object]
+
+        x.flags.writeable = False
+
+        with metrics.resource_context(name="Inference", **self.profiler_kwargs):
+            y_pred = self.model.predict(x, **self.predict_kwargs)
+        self.metrics_logger.update_task(y_object, y_pred)
+        self.y_pred = y_pred
+
+    def run_attack(self):
+        x, y = self.x, self.y
+        y_object, y_patch_metadata = y
+
+        # convert dict to List[dict] to comply with ART format
+        y_object = [y_object]
+
+        with metrics.resource_context(name="Attack", **self.profiler_kwargs):
+            if self.use_label:
+                y_target = [y_object]
+            elif self.targeted:
+                y_target = self.label_targeter.generate(y_object)
+            elif self.skip_benign:
+                y_target = None  # most attacks will call self.model.predict(x)
+            else:
+                y_target = None
+
+            x_adv = self.attack.generate(
+                x=x,
+                y=y_target,
+                y_patch_metadata=[y_patch_metadata],
+                **self.generate_kwargs
+            )
+
+        # Ensure that input sample isn't overwritten by model
+        x_adv.flags.writeable = False
+        y_pred_adv = self.model.predict(x_adv, **self.predict_kwargs)
+        self.metrics_logger.update_task(y_object, y_pred_adv, adversarial=True)
+        if self.targeted:
+            self.metrics_logger.update_task(
+                y_target, y_pred_adv, adversarial=True, targeted=True
+            )
+        self.metrics_logger.update_perturbation(x, x_adv)
+
+        # If using multimodal input, add a warning if depth channels are perturbed
+        if x.shape[-1] == 6:
+            if (x[..., 3:] != x_adv[..., 3:]).sum() > 0:
+                logger.warning("Adversarial attack perturbed depth channels")
+
+        if self.sample_exporter is not None:
+            self.sample_exporter.export(x, x_adv, y, y_pred_adv)
+
+        self.x_adv, self.y_target, self.y_pred_adv = x_adv, y_target, y_pred_adv

armory/utils/config_loading.py

@@ -51,7 +51,6 @@ def load_fn(sub_config):
 def load_dataset(dataset_config, *args, num_batches=None, **kwargs):
     """
     Loads a dataset from configuration file
-
     If num_batches is None, this function will return a generator that iterates
     over the entire dataset.
     """

armory/utils/config_schema.json

@@ -242,7 +242,16 @@
                 "max_image_circle_patch_diameter",
                 "word_error_rate",
                 "object_detection_AP_per_class",
-                "object_detection_mAP"
+                "object_detection_mAP",
+                "object_detection_disappearance_rate",
+                "object_detection_hallucinations_per_image",
+                "object_detection_misclassification_rate",
+                "object_detection_true_positive_rate",
+                "carla_od_AP_per_class",
+                "carla_od_disappearance_rate",
+                "carla_od_hallucinations_per_image",
+                "carla_od_misclassification_rate",
+                "carla_od_true_positive_rate"
             ]
         },
         "sysconfig": {

armory/utils/export.py

@@ -90,6 +90,17 @@ def _export_images(self, x, x_adv):
                 mode = "RGB"
                 x_i_mode = x_i
                 x_adv_i_mode = x_adv_i
+            elif x_i.shape[-1] == 6:
+                mode = "RGB"
+                x_i_mode = x_i[..., :3]
+                x_adv_i_mode = x_adv_i[..., :3]
+                x_i_depth = x_i[..., 3:]
+                depth_image = Image.fromarray(
+                    np.uint8(np.clip(x_i_depth, 0.0, 1.0) * 255.0), mode
+                )
+                depth_image.save(
+                    os.path.join(self.output_dir, f"{self.saved_samples}_depth.png")
+                )
             else:
                 raise ValueError(f"Expected 1 or 3 channels, found {x_i.shape[-1]}")