Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Eval7 - CARLA Overhead Object Detection Dev Dataset #1912

Merged
merged 6 commits into from
May 9, 2023
Merged

Eval7 - CARLA Overhead Object Detection Dev Dataset #1912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5829560
Eval7 CARLA overhead object detection dev dataset and associated conf…
yusong-tan Apr 11, 2023
70c5b9a
updated checksum files
swsuggs May 5, 2023
02a0a77
restoring imports removed by isort pr 1789
swsuggs May 5, 2023
b8b80a1
update url
swsuggs May 5, 2023
8d917de
baseline results for carla overhead od dev set 2.0.0
swsuggs May 9, 2023
f6fbf25
update doc link to to configs
swsuggs May 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions armory/data/adversarial/carla_over_obj_det_dev.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,17 @@
"""

# fmt: off
_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz"
_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz"
# fmt: on


class CarlaOverObjDetDev(tfds.core.GeneratorBasedBuilder):
"""DatasetBuilder for carla_obj_det_dev dataset."""

VERSION = tfds.core.Version("1.0.0")
VERSION = tfds.core.Version("2.0.0")
RELEASE_NOTES = {
"1.0.0": "Eval6 update from CarlaObjDetDev with images collected from overhead perspectives",
"2.0.0": "Eval7 images collected from overhead perspectives, where patches on the sidewalk/street are constrained to +-0.03m depth perturbation and patches located elsewhere are constrained to +-3m",
}

def _info(self) -> tfds.core.DatasetInfo:
Expand Down Expand Up @@ -86,6 +87,9 @@ def _info(self) -> tfds.core.DatasetInfo:
# mask[x,y] == 1 indicates patch pixel; 0 otherwise
"mask": tfds.features.Image(shape=(960, 1280, 3)),
"avg_patch_depth": tfds.features.Tensor(shape=(), dtype=tf.float64),
"max_depth_perturb_meters": tfds.features.Tensor(
shape=(), dtype=tf.float64
),
}
),
}
Expand Down Expand Up @@ -143,6 +147,15 @@ def _generate_examples(self, path):
image_depth = deepcopy(image_rgb)
image_depth["file_name"] = fpath_depth

# Set depth perturbation bound based on image filename
if (
fname[0] == "1"
): # all images that starts with 1#######.png have patch located off off sidewalk/street
max_depth_perturb_meters = 3.0

else:
max_depth_perturb_meters = 0.03

# get object annotations for each image
annotations = cocoanno.get_annotations(image_rgb["id"])

Expand Down Expand Up @@ -188,6 +201,7 @@ def build_bbox(x, y, width, height):
)
),
"mask": os.path.join(path, foreground_mask_folder, fname),
"max_depth_perturb_meters": max_depth_perturb_meters,
},
}

Expand Down
24 changes: 23 additions & 1 deletion armory/data/adversarial_datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,30 @@
import tensorflow as tf

from armory.data import datasets
from armory.data.adversarial import ( # noqa: F401
dapricot_dev,
dapricot_test,
gtsrb_bh_poison_micronnet,
)
from armory.data.adversarial import ( # noqa: F401
resisc45_densenet121_univpatch_and_univperturbation_adversarial_224x224,
ucf101_mars_perturbation_and_patch_adversarial_112x112,
)
from armory.data.adversarial import carla_mot_dev as cmotd # noqa: F401
from armory.data.adversarial import apricot_dev, apricot_test # noqa: F401
from armory.data.adversarial import carla_mot_test as cmott # noqa: F401
from armory.data.adversarial import carla_obj_det_dev as codd # noqa: F401
from armory.data.adversarial import carla_obj_det_test as codt # noqa: F401
from armory.data.adversarial import carla_over_obj_det_dev as coodd # noqa: F401
from armory.data.adversarial import carla_over_obj_det_test as coodt # noqa: F401
from armory.data.adversarial import carla_video_tracking_dev as cvtd # noqa: F401
from armory.data.adversarial import carla_video_tracking_test as cvtt # noqa: F401
from armory.data.adversarial import imagenet_adversarial as IA # noqa: F401
from armory.data.adversarial import librispeech_adversarial as LA # noqa: F401
from armory.data.adversarial.apricot_metadata import ADV_PATCH_MAGIC_NUMBER_LABEL_ID

# Although these imports are unreferenced in this file, they are required for tfds to know they exist.

imagenet_adversarial_context = datasets.ImageContext(x_shape=(224, 224, 3))
librispeech_adversarial_context = datasets.AudioContext(
x_shape=(None,), sample_rate=16000
Expand Down Expand Up @@ -699,7 +721,7 @@ def both_fn(batch):
)

return datasets._generator_from_tfds(
"carla_over_obj_det_dev:1.0.0",
"carla_over_obj_det_dev:2.0.0",
split=split,
batch_size=batch_size,
epochs=epochs,
Expand Down
2 changes: 1 addition & 1 deletion armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
armory-public-data carla/carla_over_od_dev_cached_1.0.0.tar.gz 59760901 251db1715fdcb7ff68f99b155b49cff2ceb4097e9f4e8b1e835d502579de249a
armory-public-data carla/carla_over_od_dev_cached_2.0.0.tar.gz 62773923 fa991a146afd771f806e050fdcfdf6953667a86acbbc05a3cd9c7d7a51ba92a6
2 changes: 1 addition & 1 deletion armory/data/url_checksums/carla_over_obj_det_dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz 61751299 ee6fbd0f00cf88aaef3ae097e9d661d52b0d97e5688098d342dad4780f931c8a
https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz 64846940 248a04c61c8dc96f8bfd62e4a0977973b77962fdd64685ee7e4327e82c80140f
22 changes: 11 additions & 11 deletions docs/baseline_results/carla_od_results.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,28 +30,28 @@ Find reference baseline configurations [here](https://github.com/twosixlabs/armo

## CARLA Overhead OD Dataset

Dev data results obtained using Armory 0.16.0, Test data results obtained using Armory 0.16.1
Dev data results obtained using Armory 0.16.6, Test data results obtained using Armory 0.16.1

Single Modality (RGB) Object Detection

| Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size |
|------|----------|-------------------|------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------|
| Dev | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.16 | 0.64 | 31.0 | 0.009 | 0.35 | 20 |
| Dev | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.42 | 0.43 | 12.0 | 0.009 | 0.56 | 20 |
| Dev | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.51 | 0.44 | 3.1 | 0.02 | 0.54 | 0.41 | 0.52 | 9.8 | 0.01 | 0.47 | 20 |
| Test | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 |
| Dev 2.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.05 | 0.80 | 56.1 | 0.01 | 0.19 | 20 |
| Dev 2.0.0 | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.43 | 0.40 | 16.9 | 0.03 | 0.57 | 20 |
| Dev 2.0.0 | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.59 | 0.43 | 1.7 | 0.03 | 0.54 | 0.40 | 0.52 | 9.0 | 0.03 | 0.45 | 20 |
| Test 1.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 |

Multimodality (RGB+depth) Object Detection

| Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size |
|------|----------|-------------------|-----------------------------------------------------------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------|
| Dev | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.39 | 0.53 | 5.0 | 0.02 | 0.45 | 20 |
| Dev | yes | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.56 | 0.48 | 1.1 | 0.02 | 0.50 | 20 |
| Dev | no | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.54 | 0.42 | 0.65 | 0.02 | 0.56 | 20 |
| Dev | yes | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.65 | 0.38 | 0.9 | 0.02 | 0.60 | 20 |
| Test | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 |
| Dev 2.0.0 | no | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.14 | 0.56 | 29.5 | 0.02 | 0.41 | 20 |
| Dev 2.0.0 | yes | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.16 | 0.51 | 25.7 | 0.02 | 0.47 | 20 |
| Dev 2.0.0 | no | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.59 | 0.37 | 3.3 | 0.03 | 0.60 | 20 |
| Dev 2.0.0 | yes | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.61 | 0.37 | 1.7 | 0.03 | 0.60 | 20 |
| Test 1.0.0 | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 |


Defended results not available for Adversarial Patch attack against single modality because JPEG Compression defense is not implemented in PyTorch and so is not fully differentiable

Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval6/carla_overhead_object_detection)
Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval7/carla_overhead_object_detection)
64 changes: 64 additions & 0 deletions ...figs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
{
"_description": "CARLA single modality object detection, contributed by MITRE Corporation",
"adhoc": null,
"attack": {
"knowledge": "white",
"kwargs": {
"batch_size": 1,
"learning_rate": 0.003,
"max_iter": 1000,
"optimizer": "pgd",
"targeted": false,
"verbose": true
},
"module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch",
"name": "CARLAAdversarialPatchPyTorch",
"use_label": true
},
"dataset": {
"batch_size": 1,
"eval_split": "dev",
"framework": "numpy",
"modality": "rgb",
"module": "armory.data.adversarial_datasets",
"name": "carla_over_obj_det_dev"
},
"defense": null,
"metric": {
"means": true,
"perturbation": "l0",
"record_metric_per_sample": false,
"task": [
"carla_od_AP_per_class",
"carla_od_disappearance_rate",
"carla_od_hallucinations_per_image",
"carla_od_misclassification_rate",
"carla_od_true_positive_rate",
"object_detection_mAP_tide"
]
},
"model": {
"fit": false,
"fit_kwargs": {},
"model_kwargs": {
"num_classes": 3
},
"module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",
"name": "get_art_model",
"weights_file": "carla_rgb_weights_eval7and8.pt",
"wrapper_kwargs": {}
},
"scenario": {
"kwargs": {},
"module": "armory.scenarios.carla_object_detection",
"name": "CarlaObjectDetectionTask"
},
"sysconfig": {
"docker_image": "twosixarmory/armory",
"external_github_repo": null,
"gpus": "all",
"output_dir": null,
"output_filename": null,
"use_gpu": false
}
}
75 changes: 75 additions & 0 deletions scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{
"_description": "CARLA single modality object detection, contributed by MITRE Corporation",
"adhoc": null,
"attack": {
"knowledge": "white",
"kwargs": {
"batch_size": 1,
"learning_rate": 0.002,
"max_iter": 2000,
"verbose": true
},
"module": "armory.art_experimental.attacks.carla_obj_det_patch",
"name": "CARLADapricotPatch",
"use_label": false
},
"dataset": {
"batch_size": 1,
"eval_split": "dev",
"framework": "numpy",
"modality": "rgb",
"module": "armory.data.adversarial_datasets",
"name": "carla_over_obj_det_dev"
},
"defense": {
"kwargs": {
"apply_fit": false,
"apply_predict": true,
"clip_values": [
0.0,
1.0
],
"quality": 50
},
"module": "art.defences.preprocessor",
"name": "JpegCompression",
"type": "Preprocessor"
},
"metric": {
"means": true,
"perturbation": "l0",
"record_metric_per_sample": false,
"task": [
"carla_od_AP_per_class",
"carla_od_disappearance_rate",
"carla_od_hallucinations_per_image",
"carla_od_misclassification_rate",
"carla_od_true_positive_rate",
"object_detection_mAP_tide"
]
},
"model": {
"fit": false,
"fit_kwargs": {},
"model_kwargs": {
"num_classes": 3
},
"module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",
"name": "get_art_model",
"weights_file": "carla_rgb_weights_eval7and8.pt",
"wrapper_kwargs": {}
},
"scenario": {
"kwargs": {},
"module": "armory.scenarios.carla_object_detection",
"name": "CarlaObjectDetectionTask"
},
"sysconfig": {
"docker_image": "twosixarmory/armory",
"external_github_repo": "colour-science/colour@v0.3.16",
"gpus": "all",
"output_dir": null,
"output_filename": null,
"use_gpu": false
}
}
Loading