From 81675d9daa710d487ea074a4e9ccd0dc8e51b300 Mon Sep 17 00:00:00 2001 From: yusong-tan <59029053+yusong-tan@users.noreply.github.com> Date: Tue, 9 May 2023 08:38:47 -0700 Subject: [PATCH] Eval7 - CARLA Overhead Object Detection Dev Dataset (#1912) * Eval7 CARLA overhead object detection dev dataset and associated config changes * updated checksum files * restoring imports removed by isort pr 1789 * update url * baseline results for carla overhead od dev set 2.0.0 * update doc link to to configs --------- Co-authored-by: Sterling --- .../adversarial/carla_over_obj_det_dev.py | 18 ++++- armory/data/adversarial_datasets.py | 24 +++++- .../carla_over_obj_det_dev.txt | 2 +- .../url_checksums/carla_over_obj_det_dev.txt | 2 +- docs/baseline_results/carla_od_results.md | 22 +++--- ...a_obj_det_adversarialpatch_undefended.json | 64 ++++++++++++++++ .../carla_obj_det_dpatch_defended.json | 75 +++++++++++++++++++ .../carla_obj_det_dpatch_undefended.json | 62 +++++++++++++++ ..._multimodal_adversarialpatch_defended.json | 64 ++++++++++++++++ ...ultimodal_adversarialpatch_undefended.json | 64 ++++++++++++++++ ...la_obj_det_multimodal_dpatch_defended.json | 62 +++++++++++++++ ..._obj_det_multimodal_dpatch_undefended.json | 62 +++++++++++++++ 12 files changed, 505 insertions(+), 16 deletions(-) create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json diff --git a/armory/data/adversarial/carla_over_obj_det_dev.py b/armory/data/adversarial/carla_over_obj_det_dev.py index 4bc849c87..54c857131 100644 --- a/armory/data/adversarial/carla_over_obj_det_dev.py +++ b/armory/data/adversarial/carla_over_obj_det_dev.py @@ -24,16 +24,17 @@ """ # fmt: off -_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz" +_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz" # fmt: on class CarlaOverObjDetDev(tfds.core.GeneratorBasedBuilder): """DatasetBuilder for carla_obj_det_dev dataset.""" - VERSION = tfds.core.Version("1.0.0") + VERSION = tfds.core.Version("2.0.0") RELEASE_NOTES = { "1.0.0": "Eval6 update from CarlaObjDetDev with images collected from overhead perspectives", + "2.0.0": "Eval7 images collected from overhead perspectives, where patches on the sidewalk/street are constrained to +-0.03m depth perturbation and patches located elsewhere are constrained to +-3m", } def _info(self) -> tfds.core.DatasetInfo: @@ -86,6 +87,9 @@ def _info(self) -> tfds.core.DatasetInfo: # mask[x,y] == 1 indicates patch pixel; 0 otherwise "mask": tfds.features.Image(shape=(960, 1280, 3)), "avg_patch_depth": tfds.features.Tensor(shape=(), dtype=tf.float64), + "max_depth_perturb_meters": tfds.features.Tensor( + shape=(), dtype=tf.float64 + ), } ), } @@ -143,6 +147,15 @@ def _generate_examples(self, path): image_depth = deepcopy(image_rgb) image_depth["file_name"] = fpath_depth + # Set depth perturbation bound based on image filename + if ( + fname[0] == "1" + ): # all images that starts with 1#######.png have patch located off off sidewalk/street + max_depth_perturb_meters = 3.0 + + else: + max_depth_perturb_meters = 0.03 + # get object annotations for each image annotations = cocoanno.get_annotations(image_rgb["id"]) @@ -188,6 +201,7 @@ def build_bbox(x, y, width, height): ) ), "mask": os.path.join(path, foreground_mask_folder, fname), + "max_depth_perturb_meters": max_depth_perturb_meters, }, } diff --git a/armory/data/adversarial_datasets.py b/armory/data/adversarial_datasets.py index 2e4f6ba98..ebf5f8d0f 100644 --- a/armory/data/adversarial_datasets.py +++ b/armory/data/adversarial_datasets.py @@ -8,8 +8,30 @@ import tensorflow as tf from armory.data import datasets +from armory.data.adversarial import ( # noqa: F401 + dapricot_dev, + dapricot_test, + gtsrb_bh_poison_micronnet, +) +from armory.data.adversarial import ( # noqa: F401 + resisc45_densenet121_univpatch_and_univperturbation_adversarial_224x224, + ucf101_mars_perturbation_and_patch_adversarial_112x112, +) +from armory.data.adversarial import carla_mot_dev as cmotd # noqa: F401 +from armory.data.adversarial import apricot_dev, apricot_test # noqa: F401 +from armory.data.adversarial import carla_mot_test as cmott # noqa: F401 +from armory.data.adversarial import carla_obj_det_dev as codd # noqa: F401 +from armory.data.adversarial import carla_obj_det_test as codt # noqa: F401 +from armory.data.adversarial import carla_over_obj_det_dev as coodd # noqa: F401 +from armory.data.adversarial import carla_over_obj_det_test as coodt # noqa: F401 +from armory.data.adversarial import carla_video_tracking_dev as cvtd # noqa: F401 +from armory.data.adversarial import carla_video_tracking_test as cvtt # noqa: F401 +from armory.data.adversarial import imagenet_adversarial as IA # noqa: F401 +from armory.data.adversarial import librispeech_adversarial as LA # noqa: F401 from armory.data.adversarial.apricot_metadata import ADV_PATCH_MAGIC_NUMBER_LABEL_ID +# Although these imports are unreferenced in this file, they are required for tfds to know they exist. + imagenet_adversarial_context = datasets.ImageContext(x_shape=(224, 224, 3)) librispeech_adversarial_context = datasets.AudioContext( x_shape=(None,), sample_rate=16000 @@ -699,7 +721,7 @@ def both_fn(batch): ) return datasets._generator_from_tfds( - "carla_over_obj_det_dev:1.0.0", + "carla_over_obj_det_dev:2.0.0", split=split, batch_size=batch_size, epochs=epochs, diff --git a/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt b/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt index e4efeded9..89d50199c 100644 --- a/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt +++ b/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt @@ -1 +1 @@ -armory-public-data carla/carla_over_od_dev_cached_1.0.0.tar.gz 59760901 251db1715fdcb7ff68f99b155b49cff2ceb4097e9f4e8b1e835d502579de249a \ No newline at end of file +armory-public-data carla/carla_over_od_dev_cached_2.0.0.tar.gz 62773923 fa991a146afd771f806e050fdcfdf6953667a86acbbc05a3cd9c7d7a51ba92a6 \ No newline at end of file diff --git a/armory/data/url_checksums/carla_over_obj_det_dev.txt b/armory/data/url_checksums/carla_over_obj_det_dev.txt index 78bcd4d95..db10ac71e 100644 --- a/armory/data/url_checksums/carla_over_obj_det_dev.txt +++ b/armory/data/url_checksums/carla_over_obj_det_dev.txt @@ -1 +1 @@ -https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz 61751299 ee6fbd0f00cf88aaef3ae097e9d661d52b0d97e5688098d342dad4780f931c8a \ No newline at end of file +https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz 64846940 248a04c61c8dc96f8bfd62e4a0977973b77962fdd64685ee7e4327e82c80140f \ No newline at end of file diff --git a/docs/baseline_results/carla_od_results.md b/docs/baseline_results/carla_od_results.md index f43070e70..00b1db45b 100644 --- a/docs/baseline_results/carla_od_results.md +++ b/docs/baseline_results/carla_od_results.md @@ -30,28 +30,28 @@ Find reference baseline configurations [here](https://github.com/twosixlabs/armo ## CARLA Overhead OD Dataset -Dev data results obtained using Armory 0.16.0, Test data results obtained using Armory 0.16.1 +Dev data results obtained using Armory 0.16.6, Test data results obtained using Armory 0.16.1 Single Modality (RGB) Object Detection | Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size | |------|----------|-------------------|------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------| -| Dev | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.16 | 0.64 | 31.0 | 0.009 | 0.35 | 20 | -| Dev | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.42 | 0.43 | 12.0 | 0.009 | 0.56 | 20 | -| Dev | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.51 | 0.44 | 3.1 | 0.02 | 0.54 | 0.41 | 0.52 | 9.8 | 0.01 | 0.47 | 20 | -| Test | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 | +| Dev 2.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.05 | 0.80 | 56.1 | 0.01 | 0.19 | 20 | +| Dev 2.0.0 | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.43 | 0.40 | 16.9 | 0.03 | 0.57 | 20 | +| Dev 2.0.0 | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.59 | 0.43 | 1.7 | 0.03 | 0.54 | 0.40 | 0.52 | 9.0 | 0.03 | 0.45 | 20 | +| Test 1.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 | Multimodality (RGB+depth) Object Detection | Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size | |------|----------|-------------------|-----------------------------------------------------------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------| -| Dev | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.39 | 0.53 | 5.0 | 0.02 | 0.45 | 20 | -| Dev | yes | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.56 | 0.48 | 1.1 | 0.02 | 0.50 | 20 | -| Dev | no | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.54 | 0.42 | 0.65 | 0.02 | 0.56 | 20 | -| Dev | yes | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.65 | 0.38 | 0.9 | 0.02 | 0.60 | 20 | -| Test | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 | +| Dev 2.0.0 | no | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.14 | 0.56 | 29.5 | 0.02 | 0.41 | 20 | +| Dev 2.0.0 | yes | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.16 | 0.51 | 25.7 | 0.02 | 0.47 | 20 | +| Dev 2.0.0 | no | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.59 | 0.37 | 3.3 | 0.03 | 0.60 | 20 | +| Dev 2.0.0 | yes | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.61 | 0.37 | 1.7 | 0.03 | 0.60 | 20 | +| Test 1.0.0 | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 | Defended results not available for Adversarial Patch attack against single modality because JPEG Compression defense is not implemented in PyTorch and so is not fully differentiable -Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval6/carla_overhead_object_detection) \ No newline at end of file +Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval7/carla_overhead_object_detection) \ No newline at end of file diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json new file mode 100644 index 000000000..f8ff17f0f --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json @@ -0,0 +1,64 @@ +{ + "_description": "CARLA single modality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "learning_rate": 0.003, + "max_iter": 1000, + "optimizer": "pgd", + "targeted": false, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch", + "name": "CARLAAdversarialPatchPyTorch", + "use_label": true + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "rgb", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": { + "num_classes": 3 + }, + "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", + "name": "get_art_model", + "weights_file": "carla_rgb_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json new file mode 100644 index 000000000..528924b8d --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json @@ -0,0 +1,75 @@ +{ + "_description": "CARLA single modality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "learning_rate": 0.002, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "rgb", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": { + "kwargs": { + "apply_fit": false, + "apply_predict": true, + "clip_values": [ + 0.0, + 1.0 + ], + "quality": 50 + }, + "module": "art.defences.preprocessor", + "name": "JpegCompression", + "type": "Preprocessor" + }, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": { + "num_classes": 3 + }, + "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", + "name": "get_art_model", + "weights_file": "carla_rgb_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json new file mode 100644 index 000000000..5a921899c --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json @@ -0,0 +1,62 @@ +{ + "_description": "CARLA single modality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "learning_rate": 0.002, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "rgb", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": { + "num_classes": 3 + }, + "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", + "name": "get_art_model", + "weights_file": "carla_rgb_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json new file mode 100644 index 000000000..0daaeb35e --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json @@ -0,0 +1,64 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.003, + "learning_rate_depth": 0.005, + "max_iter": 1000, + "optimizer": "pgd", + "targeted": false, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch", + "name": "CARLAAdversarialPatchPyTorch", + "use_label": true + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", + "name": "get_art_model_mm_robust", + "weights_file": "carla_multimodal_robust_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json new file mode 100644 index 000000000..1696d01e7 --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json @@ -0,0 +1,64 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.003, + "learning_rate_depth": 0.005, + "max_iter": 1000, + "optimizer": "pgd", + "targeted": false, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch", + "name": "CARLAAdversarialPatchPyTorch", + "use_label": true + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", + "name": "get_art_model_mm", + "weights_file": "carla_multimodal_naive_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json new file mode 100644 index 000000000..fa3961833 --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json @@ -0,0 +1,62 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.002, + "learning_rate_depth": 0.003, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", + "name": "get_art_model_mm_robust", + "weights_file": "carla_multimodal_robust_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json new file mode 100644 index 000000000..0312da8f3 --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json @@ -0,0 +1,62 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.002, + "learning_rate_depth": 0.003, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", + "name": "get_art_model_mm", + "weights_file": "carla_multimodal_naive_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +}