Skip to content

Commit

Permalink
Eval7 - CARLA Overhead Object Detection Dev Dataset (#1912)
Browse files Browse the repository at this point in the history 
* Eval7 CARLA overhead object detection dev dataset and associated config changes

* updated checksum files

* restoring imports removed by isort pr 1789

* update url

* baseline results for carla overhead od dev set 2.0.0

* update doc link to to configs

---------

Co-authored-by: Sterling <sterling.suggs@twosixtech.com>
  • Loading branch information
@yusong-tan @swsuggs
yusong-tan and swsuggs authored May 9, 2023
1 parent 14006ec commit 81675d9
Show file tree
Hide file tree
Showing 12 changed files with 505 additions and 16 deletions.
18 changes: 16 additions & 2 deletions armory/data/adversarial/carla_over_obj_det_dev.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,17 @@
"""

# fmt: off
_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz"
_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz"
# fmt: on


class CarlaOverObjDetDev(tfds.core.GeneratorBasedBuilder):
"""DatasetBuilder for carla_obj_det_dev dataset."""

VERSION = tfds.core.Version("1.0.0")
VERSION = tfds.core.Version("2.0.0")
RELEASE_NOTES = {
"1.0.0": "Eval6 update from CarlaObjDetDev with images collected from overhead perspectives",
"2.0.0": "Eval7 images collected from overhead perspectives, where patches on the sidewalk/street are constrained to +-0.03m depth perturbation and patches located elsewhere are constrained to +-3m",
}

def _info(self) -> tfds.core.DatasetInfo:
Expand Down Expand Up @@ -86,6 +87,9 @@ def _info(self) -> tfds.core.DatasetInfo:
# mask[x,y] == 1 indicates patch pixel; 0 otherwise
"mask": tfds.features.Image(shape=(960, 1280, 3)),
"avg_patch_depth": tfds.features.Tensor(shape=(), dtype=tf.float64),
"max_depth_perturb_meters": tfds.features.Tensor(
shape=(), dtype=tf.float64
),
}
),
}
Expand Down Expand Up @@ -143,6 +147,15 @@ def _generate_examples(self, path):
image_depth = deepcopy(image_rgb)
image_depth["file_name"] = fpath_depth

# Set depth perturbation bound based on image filename
if (
fname[0] == "1"
): # all images that starts with 1#######.png have patch located off off sidewalk/street
max_depth_perturb_meters = 3.0

else:
max_depth_perturb_meters = 0.03

# get object annotations for each image
annotations = cocoanno.get_annotations(image_rgb["id"])

Expand Down Expand Up @@ -188,6 +201,7 @@ def build_bbox(x, y, width, height):
)
),
"mask": os.path.join(path, foreground_mask_folder, fname),
"max_depth_perturb_meters": max_depth_perturb_meters,
},
}

Expand Down
24 changes: 23 additions & 1 deletion armory/data/adversarial_datasets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,30 @@
import tensorflow as tf

from armory.data import datasets
from armory.data.adversarial import ( # noqa: F401
dapricot_dev,
dapricot_test,
gtsrb_bh_poison_micronnet,
)
from armory.data.adversarial import ( # noqa: F401
resisc45_densenet121_univpatch_and_univperturbation_adversarial_224x224,
ucf101_mars_perturbation_and_patch_adversarial_112x112,
)
from armory.data.adversarial import carla_mot_dev as cmotd # noqa: F401
from armory.data.adversarial import apricot_dev, apricot_test # noqa: F401
from armory.data.adversarial import carla_mot_test as cmott # noqa: F401
from armory.data.adversarial import carla_obj_det_dev as codd # noqa: F401
from armory.data.adversarial import carla_obj_det_test as codt # noqa: F401
from armory.data.adversarial import carla_over_obj_det_dev as coodd # noqa: F401
from armory.data.adversarial import carla_over_obj_det_test as coodt # noqa: F401
from armory.data.adversarial import carla_video_tracking_dev as cvtd # noqa: F401
from armory.data.adversarial import carla_video_tracking_test as cvtt # noqa: F401
from armory.data.adversarial import imagenet_adversarial as IA # noqa: F401
from armory.data.adversarial import librispeech_adversarial as LA # noqa: F401
from armory.data.adversarial.apricot_metadata import ADV_PATCH_MAGIC_NUMBER_LABEL_ID

# Although these imports are unreferenced in this file, they are required for tfds to know they exist.

imagenet_adversarial_context = datasets.ImageContext(x_shape=(224, 224, 3))
librispeech_adversarial_context = datasets.AudioContext(
x_shape=(None,), sample_rate=16000
Expand Down Expand Up @@ -699,7 +721,7 @@ def both_fn(batch):
)

return datasets._generator_from_tfds(
"carla_over_obj_det_dev:1.0.0",
"carla_over_obj_det_dev:2.0.0",
split=split,
batch_size=batch_size,
epochs=epochs,
Expand Down
2 changes: 1 addition & 1 deletion armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
armory-public-data carla/carla_over_od_dev_cached_1.0.0.tar.gz 59760901 251db1715fdcb7ff68f99b155b49cff2ceb4097e9f4e8b1e835d502579de249a
armory-public-data carla/carla_over_od_dev_cached_2.0.0.tar.gz 62773923 fa991a146afd771f806e050fdcfdf6953667a86acbbc05a3cd9c7d7a51ba92a6
2 changes: 1 addition & 1 deletion armory/data/url_checksums/carla_over_obj_det_dev.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz 61751299 ee6fbd0f00cf88aaef3ae097e9d661d52b0d97e5688098d342dad4780f931c8a
https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz 64846940 248a04c61c8dc96f8bfd62e4a0977973b77962fdd64685ee7e4327e82c80140f
22 changes: 11 additions & 11 deletions docs/baseline_results/carla_od_results.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,28 +30,28 @@ Find reference baseline configurations [here](https://github.com/twosixlabs/armo

## CARLA Overhead OD Dataset

Dev data results obtained using Armory 0.16.0, Test data results obtained using Armory 0.16.1
Dev data results obtained using Armory 0.16.6, Test data results obtained using Armory 0.16.1

Single Modality (RGB) Object Detection

| Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size |
|------|----------|-------------------|------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------|
| Dev | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.16 | 0.64 | 31.0 | 0.009 | 0.35 | 20 |
| Dev | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.42 | 0.43 | 12.0 | 0.009 | 0.56 | 20 |
| Dev | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.51 | 0.44 | 3.1 | 0.02 | 0.54 | 0.41 | 0.52 | 9.8 | 0.01 | 0.47 | 20 |
| Test | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 |
| Dev 2.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.05 | 0.80 | 56.1 | 0.01 | 0.19 | 20 |
| Dev 2.0.0 | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.43 | 0.40 | 16.9 | 0.03 | 0.57 | 20 |
| Dev 2.0.0 | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.59 | 0.43 | 1.7 | 0.03 | 0.54 | 0.40 | 0.52 | 9.0 | 0.03 | 0.45 | 20 |
| Test 1.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 |

Multimodality (RGB+depth) Object Detection

| Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size |
|------|----------|-------------------|-----------------------------------------------------------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------|
| Dev | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.39 | 0.53 | 5.0 | 0.02 | 0.45 | 20 |
| Dev | yes | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.56 | 0.48 | 1.1 | 0.02 | 0.50 | 20 |
| Dev | no | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.54 | 0.42 | 0.65 | 0.02 | 0.56 | 20 |
| Dev | yes | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.65 | 0.38 | 0.9 | 0.02 | 0.60 | 20 |
| Test | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 |
| Dev 2.0.0 | no | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.14 | 0.56 | 29.5 | 0.02 | 0.41 | 20 |
| Dev 2.0.0 | yes | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.16 | 0.51 | 25.7 | 0.02 | 0.47 | 20 |
| Dev 2.0.0 | no | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.59 | 0.37 | 3.3 | 0.03 | 0.60 | 20 |
| Dev 2.0.0 | yes | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.61 | 0.37 | 1.7 | 0.03 | 0.60 | 20 |
| Test 1.0.0 | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 |


Defended results not available for Adversarial Patch attack against single modality because JPEG Compression defense is not implemented in PyTorch and so is not fully differentiable

Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval6/carla_overhead_object_detection)
Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval7/carla_overhead_object_detection)
64 changes: 64 additions & 0 deletions ...figs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
{
"_description": "CARLA single modality object detection, contributed by MITRE Corporation",
"adhoc": null,
"attack": {
"knowledge": "white",
"kwargs": {
"batch_size": 1,
"learning_rate": 0.003,
"max_iter": 1000,
"optimizer": "pgd",
"targeted": false,
"verbose": true
},
"module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch",
"name": "CARLAAdversarialPatchPyTorch",
"use_label": true
},
"dataset": {
"batch_size": 1,
"eval_split": "dev",
"framework": "numpy",
"modality": "rgb",
"module": "armory.data.adversarial_datasets",
"name": "carla_over_obj_det_dev"
},
"defense": null,
"metric": {
"means": true,
"perturbation": "l0",
"record_metric_per_sample": false,
"task": [
"carla_od_AP_per_class",
"carla_od_disappearance_rate",
"carla_od_hallucinations_per_image",
"carla_od_misclassification_rate",
"carla_od_true_positive_rate",
"object_detection_mAP_tide"
]
},
"model": {
"fit": false,
"fit_kwargs": {},
"model_kwargs": {
"num_classes": 3
},
"module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",
"name": "get_art_model",
"weights_file": "carla_rgb_weights_eval7and8.pt",
"wrapper_kwargs": {}
},
"scenario": {
"kwargs": {},
"module": "armory.scenarios.carla_object_detection",
"name": "CarlaObjectDetectionTask"
},
"sysconfig": {
"docker_image": "twosixarmory/armory",
"external_github_repo": null,
"gpus": "all",
"output_dir": null,
"output_filename": null,
"use_gpu": false
}
}
75 changes: 75 additions & 0 deletions scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{
"_description": "CARLA single modality object detection, contributed by MITRE Corporation",
"adhoc": null,
"attack": {
"knowledge": "white",
"kwargs": {
"batch_size": 1,
"learning_rate": 0.002,
"max_iter": 2000,
"verbose": true
},
"module": "armory.art_experimental.attacks.carla_obj_det_patch",
"name": "CARLADapricotPatch",
"use_label": false
},
"dataset": {
"batch_size": 1,
"eval_split": "dev",
"framework": "numpy",
"modality": "rgb",
"module": "armory.data.adversarial_datasets",
"name": "carla_over_obj_det_dev"
},
"defense": {
"kwargs": {
"apply_fit": false,
"apply_predict": true,
"clip_values": [
0.0,
1.0
],
"quality": 50
},
"module": "art.defences.preprocessor",
"name": "JpegCompression",
"type": "Preprocessor"
},
"metric": {
"means": true,
"perturbation": "l0",
"record_metric_per_sample": false,
"task": [
"carla_od_AP_per_class",
"carla_od_disappearance_rate",
"carla_od_hallucinations_per_image",
"carla_od_misclassification_rate",
"carla_od_true_positive_rate",
"object_detection_mAP_tide"
]
},
"model": {
"fit": false,
"fit_kwargs": {},
"model_kwargs": {
"num_classes": 3
},
"module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",
"name": "get_art_model",
"weights_file": "carla_rgb_weights_eval7and8.pt",
"wrapper_kwargs": {}
},
"scenario": {
"kwargs": {},
"module": "armory.scenarios.carla_object_detection",
"name": "CarlaObjectDetectionTask"
},
"sysconfig": {
"docker_image": "twosixarmory/armory",
"external_github_repo": "colour-science/colour@v0.3.16",
"gpus": "all",
"output_dir": null,
"output_filename": null,
"use_gpu": false
}
}
Loading

0 comments on commit 81675d9

Please sign in to comment.