From 5829560262a7faa2fb6e6ddefc1f005b9bd5acc8 Mon Sep 17 00:00:00 2001 From: Yusong Date: Tue, 11 Apr 2023 23:09:57 +0000 Subject: [PATCH 1/6] Eval7 CARLA overhead object detection dev dataset and associated config changes --- .../adversarial/carla_over_obj_det_dev.py | 18 ++++- armory/data/adversarial_datasets.py | 2 +- ...a_obj_det_adversarialpatch_undefended.json | 64 ++++++++++++++++ .../carla_obj_det_dpatch_defended.json | 75 +++++++++++++++++++ .../carla_obj_det_dpatch_undefended.json | 62 +++++++++++++++ ..._multimodal_adversarialpatch_defended.json | 64 ++++++++++++++++ ...ultimodal_adversarialpatch_undefended.json | 64 ++++++++++++++++ ...la_obj_det_multimodal_dpatch_defended.json | 62 +++++++++++++++ ..._obj_det_multimodal_dpatch_undefended.json | 62 +++++++++++++++ 9 files changed, 470 insertions(+), 3 deletions(-) create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json create mode 100644 scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json diff --git a/armory/data/adversarial/carla_over_obj_det_dev.py b/armory/data/adversarial/carla_over_obj_det_dev.py index 4bc849c87..f3953e287 100644 --- a/armory/data/adversarial/carla_over_obj_det_dev.py +++ b/armory/data/adversarial/carla_over_obj_det_dev.py @@ -24,16 +24,17 @@ """ # fmt: off -_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz" +_URLS = "carla_over_od_dev_2.0.0.tar.gz" # fmt: on class CarlaOverObjDetDev(tfds.core.GeneratorBasedBuilder): """DatasetBuilder for carla_obj_det_dev dataset.""" - VERSION = tfds.core.Version("1.0.0") + VERSION = tfds.core.Version("2.0.0") RELEASE_NOTES = { "1.0.0": "Eval6 update from CarlaObjDetDev with images collected from overhead perspectives", + "2.0.0": "Eval7 images collected from overhead perspectives, where patches on the sidewalk/street are constrained to +-0.03m depth perturbation and patches located elsewhere are constrained to +-3m", } def _info(self) -> tfds.core.DatasetInfo: @@ -86,6 +87,9 @@ def _info(self) -> tfds.core.DatasetInfo: # mask[x,y] == 1 indicates patch pixel; 0 otherwise "mask": tfds.features.Image(shape=(960, 1280, 3)), "avg_patch_depth": tfds.features.Tensor(shape=(), dtype=tf.float64), + "max_depth_perturb_meters": tfds.features.Tensor( + shape=(), dtype=tf.float64 + ), } ), } @@ -143,6 +147,15 @@ def _generate_examples(self, path): image_depth = deepcopy(image_rgb) image_depth["file_name"] = fpath_depth + # Set depth perturbation bound based on image filename + if ( + fname[0] == "1" + ): # all images that starts with 1#######.png have patch located off off sidewalk/street + max_depth_perturb_meters = 3.0 + + else: + max_depth_perturb_meters = 0.03 + # get object annotations for each image annotations = cocoanno.get_annotations(image_rgb["id"]) @@ -188,6 +201,7 @@ def build_bbox(x, y, width, height): ) ), "mask": os.path.join(path, foreground_mask_folder, fname), + "max_depth_perturb_meters": max_depth_perturb_meters, }, } diff --git a/armory/data/adversarial_datasets.py b/armory/data/adversarial_datasets.py index 2e4f6ba98..58c08ee46 100644 --- a/armory/data/adversarial_datasets.py +++ b/armory/data/adversarial_datasets.py @@ -699,7 +699,7 @@ def both_fn(batch): ) return datasets._generator_from_tfds( - "carla_over_obj_det_dev:1.0.0", + "carla_over_obj_det_dev:2.0.0", split=split, batch_size=batch_size, epochs=epochs, diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json new file mode 100644 index 000000000..f8ff17f0f --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json @@ -0,0 +1,64 @@ +{ + "_description": "CARLA single modality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "learning_rate": 0.003, + "max_iter": 1000, + "optimizer": "pgd", + "targeted": false, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch", + "name": "CARLAAdversarialPatchPyTorch", + "use_label": true + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "rgb", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": { + "num_classes": 3 + }, + "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", + "name": "get_art_model", + "weights_file": "carla_rgb_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json new file mode 100644 index 000000000..528924b8d --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json @@ -0,0 +1,75 @@ +{ + "_description": "CARLA single modality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "learning_rate": 0.002, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "rgb", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": { + "kwargs": { + "apply_fit": false, + "apply_predict": true, + "clip_values": [ + 0.0, + 1.0 + ], + "quality": 50 + }, + "module": "art.defences.preprocessor", + "name": "JpegCompression", + "type": "Preprocessor" + }, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": { + "num_classes": 3 + }, + "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", + "name": "get_art_model", + "weights_file": "carla_rgb_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json new file mode 100644 index 000000000..5a921899c --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json @@ -0,0 +1,62 @@ +{ + "_description": "CARLA single modality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "learning_rate": 0.002, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "rgb", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": { + "num_classes": 3 + }, + "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn", + "name": "get_art_model", + "weights_file": "carla_rgb_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json new file mode 100644 index 000000000..0daaeb35e --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json @@ -0,0 +1,64 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.003, + "learning_rate_depth": 0.005, + "max_iter": 1000, + "optimizer": "pgd", + "targeted": false, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch", + "name": "CARLAAdversarialPatchPyTorch", + "use_label": true + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", + "name": "get_art_model_mm_robust", + "weights_file": "carla_multimodal_robust_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json new file mode 100644 index 000000000..1696d01e7 --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json @@ -0,0 +1,64 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.003, + "learning_rate_depth": 0.005, + "max_iter": 1000, + "optimizer": "pgd", + "targeted": false, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_adversarial_patch", + "name": "CARLAAdversarialPatchPyTorch", + "use_label": true + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", + "name": "get_art_model_mm", + "weights_file": "carla_multimodal_naive_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json new file mode 100644 index 000000000..fa3961833 --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json @@ -0,0 +1,62 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.002, + "learning_rate_depth": 0.003, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion", + "name": "get_art_model_mm_robust", + "weights_file": "carla_multimodal_robust_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} diff --git a/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json new file mode 100644 index 000000000..0312da8f3 --- /dev/null +++ b/scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json @@ -0,0 +1,62 @@ +{ + "_description": "CARLA multimodality object detection, contributed by MITRE Corporation", + "adhoc": null, + "attack": { + "knowledge": "white", + "kwargs": { + "batch_size": 1, + "depth_delta_meters": 3, + "learning_rate": 0.002, + "learning_rate_depth": 0.003, + "max_iter": 2000, + "verbose": true + }, + "module": "armory.art_experimental.attacks.carla_obj_det_patch", + "name": "CARLADapricotPatch", + "use_label": false + }, + "dataset": { + "batch_size": 1, + "eval_split": "dev", + "framework": "numpy", + "modality": "both", + "module": "armory.data.adversarial_datasets", + "name": "carla_over_obj_det_dev" + }, + "defense": null, + "metric": { + "means": true, + "perturbation": "l0", + "record_metric_per_sample": false, + "task": [ + "carla_od_AP_per_class", + "carla_od_disappearance_rate", + "carla_od_hallucinations_per_image", + "carla_od_misclassification_rate", + "carla_od_true_positive_rate", + "object_detection_mAP_tide" + ] + }, + "model": { + "fit": false, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn", + "name": "get_art_model_mm", + "weights_file": "carla_multimodal_naive_weights_eval7and8.pt", + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.carla_object_detection", + "name": "CarlaObjectDetectionTask" + }, + "sysconfig": { + "docker_image": "twosixarmory/armory", + "external_github_repo": "colour-science/colour@v0.3.16", + "gpus": "all", + "output_dir": null, + "output_filename": null, + "use_gpu": false + } +} From 70c5b9afb0c750a08eb0ad23c89d4697aa9e23f3 Mon Sep 17 00:00:00 2001 From: Sterling Date: Fri, 5 May 2023 16:07:04 +0000 Subject: [PATCH 2/6] updated checksum files --- armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt | 2 +- armory/data/url_checksums/carla_over_obj_det_dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt b/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt index e4efeded9..89d50199c 100644 --- a/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt +++ b/armory/data/cached_s3_checksums/carla_over_obj_det_dev.txt @@ -1 +1 @@ -armory-public-data carla/carla_over_od_dev_cached_1.0.0.tar.gz 59760901 251db1715fdcb7ff68f99b155b49cff2ceb4097e9f4e8b1e835d502579de249a \ No newline at end of file +armory-public-data carla/carla_over_od_dev_cached_2.0.0.tar.gz 62773923 fa991a146afd771f806e050fdcfdf6953667a86acbbc05a3cd9c7d7a51ba92a6 \ No newline at end of file diff --git a/armory/data/url_checksums/carla_over_obj_det_dev.txt b/armory/data/url_checksums/carla_over_obj_det_dev.txt index 78bcd4d95..db10ac71e 100644 --- a/armory/data/url_checksums/carla_over_obj_det_dev.txt +++ b/armory/data/url_checksums/carla_over_obj_det_dev.txt @@ -1 +1 @@ -https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz 61751299 ee6fbd0f00cf88aaef3ae097e9d661d52b0d97e5688098d342dad4780f931c8a \ No newline at end of file +https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz 64846940 248a04c61c8dc96f8bfd62e4a0977973b77962fdd64685ee7e4327e82c80140f \ No newline at end of file From 02a0a77b498159a1147aab8679d4fd3d341cdce2 Mon Sep 17 00:00:00 2001 From: Sterling Date: Fri, 5 May 2023 16:16:38 +0000 Subject: [PATCH 3/6] restoring imports removed by isort pr 1789 --- armory/data/adversarial_datasets.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/armory/data/adversarial_datasets.py b/armory/data/adversarial_datasets.py index 58c08ee46..ebf5f8d0f 100644 --- a/armory/data/adversarial_datasets.py +++ b/armory/data/adversarial_datasets.py @@ -8,8 +8,30 @@ import tensorflow as tf from armory.data import datasets +from armory.data.adversarial import ( # noqa: F401 + dapricot_dev, + dapricot_test, + gtsrb_bh_poison_micronnet, +) +from armory.data.adversarial import ( # noqa: F401 + resisc45_densenet121_univpatch_and_univperturbation_adversarial_224x224, + ucf101_mars_perturbation_and_patch_adversarial_112x112, +) +from armory.data.adversarial import carla_mot_dev as cmotd # noqa: F401 +from armory.data.adversarial import apricot_dev, apricot_test # noqa: F401 +from armory.data.adversarial import carla_mot_test as cmott # noqa: F401 +from armory.data.adversarial import carla_obj_det_dev as codd # noqa: F401 +from armory.data.adversarial import carla_obj_det_test as codt # noqa: F401 +from armory.data.adversarial import carla_over_obj_det_dev as coodd # noqa: F401 +from armory.data.adversarial import carla_over_obj_det_test as coodt # noqa: F401 +from armory.data.adversarial import carla_video_tracking_dev as cvtd # noqa: F401 +from armory.data.adversarial import carla_video_tracking_test as cvtt # noqa: F401 +from armory.data.adversarial import imagenet_adversarial as IA # noqa: F401 +from armory.data.adversarial import librispeech_adversarial as LA # noqa: F401 from armory.data.adversarial.apricot_metadata import ADV_PATCH_MAGIC_NUMBER_LABEL_ID +# Although these imports are unreferenced in this file, they are required for tfds to know they exist. + imagenet_adversarial_context = datasets.ImageContext(x_shape=(224, 224, 3)) librispeech_adversarial_context = datasets.AudioContext( x_shape=(None,), sample_rate=16000 From b8b80a16adebada6535f60614d942d88d6e5e9c3 Mon Sep 17 00:00:00 2001 From: Sterling Date: Fri, 5 May 2023 16:17:26 +0000 Subject: [PATCH 4/6] update url --- armory/data/adversarial/carla_over_obj_det_dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/armory/data/adversarial/carla_over_obj_det_dev.py b/armory/data/adversarial/carla_over_obj_det_dev.py index f3953e287..54c857131 100644 --- a/armory/data/adversarial/carla_over_obj_det_dev.py +++ b/armory/data/adversarial/carla_over_obj_det_dev.py @@ -24,7 +24,7 @@ """ # fmt: off -_URLS = "carla_over_od_dev_2.0.0.tar.gz" +_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz" # fmt: on From 8d917ded2c49dddbed86c647cff141a3b09b847c Mon Sep 17 00:00:00 2001 From: Sterling Date: Tue, 9 May 2023 12:21:39 +0000 Subject: [PATCH 5/6] baseline results for carla overhead od dev set 2.0.0 --- docs/baseline_results/carla_od_results.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/baseline_results/carla_od_results.md b/docs/baseline_results/carla_od_results.md index f43070e70..118ff6d71 100644 --- a/docs/baseline_results/carla_od_results.md +++ b/docs/baseline_results/carla_od_results.md @@ -30,26 +30,26 @@ Find reference baseline configurations [here](https://github.com/twosixlabs/armo ## CARLA Overhead OD Dataset -Dev data results obtained using Armory 0.16.0, Test data results obtained using Armory 0.16.1 +Dev data results obtained using Armory 0.16.6, Test data results obtained using Armory 0.16.1 Single Modality (RGB) Object Detection | Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size | |------|----------|-------------------|------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------| -| Dev | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.16 | 0.64 | 31.0 | 0.009 | 0.35 | 20 | -| Dev | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.64 | 0.36 | 1.9 | 0.02 | 0.62 | 0.42 | 0.43 | 12.0 | 0.009 | 0.56 | 20 | -| Dev | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.51 | 0.44 | 3.1 | 0.02 | 0.54 | 0.41 | 0.52 | 9.8 | 0.01 | 0.47 | 20 | -| Test | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 | +| Dev 2.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.05 | 0.80 | 56.1 | 0.01 | 0.19 | 20 | +| Dev 2.0.0 | no | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.65 | 0.29 | 3.1 | 0.03 | 0.68 | 0.43 | 0.40 | 16.9 | 0.03 | 0.57 | 20 | +| Dev 2.0.0 | yes | Robust DPatch | learning_rate=0.002, max_iter=2000 | 0.59 | 0.43 | 1.7 | 0.03 | 0.54 | 0.40 | 0.52 | 9.0 | 0.03 | 0.45 | 20 | +| Test 1.0.0 | no | Adversarial Patch | learning_rate=0.003, max_iter=1000 | 0.60 | 0.42 | 3.6 | 0.03 | 0.55 | 0.04 | 0.81 | 54.1 | 0.0 | 0.19 | 15 | Multimodality (RGB+depth) Object Detection | Data | Defended | Attack | Attack Parameters | Benign mAP | Benign Disappearance Rate | Benign Hallucination per Image | Benign Misclassification Rate | Benign True Positive Rate | Adversarial mAP | Adversarial Disappearance Rate | Adversarial Hallucination per Image | Adversarial Misclassification Rate | Adversarial True Positive Rate | Test Size | |------|----------|-------------------|-----------------------------------------------------------------------------------------|-------------|-----------------------------|----------------------------------|---------------------------------|-----------------------------|------------------|----------------------------------|--------------------------------------|-------------------------------------|---------------------------------|-----------| -| Dev | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.39 | 0.53 | 5.0 | 0.02 | 0.45 | 20 | -| Dev | yes | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.56 | 0.48 | 1.1 | 0.02 | 0.50 | 20 | -| Dev | no | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.63 | 0.38 | 0.7 | 0.02 | 0.61 | 0.54 | 0.42 | 0.65 | 0.02 | 0.56 | 20 | -| Dev | yes | Robust DPatch | depth_delta_meters=0.03, learning_rate=0.002, learning_rate_depth=0.0001, max_iter=2000 | 0.67 | 0.34 | 0.9 | 0.02 | 0.64 | 0.65 | 0.38 | 0.9 | 0.02 | 0.60 | 20 | -| Test | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 | +| Dev 2.0.0 | no | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.14 | 0.56 | 29.5 | 0.02 | 0.41 | 20 | +| Dev 2.0.0 | yes | Adversarial Patch | depth_delta_meters=3, learning_rate=0.003, learning_rate_depth=0.005, max_iter=1000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.16 | 0.51 | 25.7 | 0.02 | 0.47 | 20 | +| Dev 2.0.0 | no | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.66 | 0.29 | 2.9 | 0.03 | 0.68 | 0.59 | 0.37 | 3.3 | 0.03 | 0.60 | 20 | +| Dev 2.0.0 | yes | Robust DPatch | depth_delta_meters=3, learning_rate=0.002, learning_rate_depth=0.003, max_iter=2000 | 0.70 | 0.28 | 1.9 | 0.03 | 0.69 | 0.61 | 0.37 | 1.7 | 0.03 | 0.60 | 20 | +| Test 1.0.0 | no | Adversarial Patch | depth_delta_meters=0.03, learning_rate=0.003, learning_rate_depth=0.0001, max_iter=1000 | 0.58 | 0.39 | 0.8 | 0.03 | 0.58 | 0.19 | 0.72 | 15.8 | 0.01 | 0.23 | 15 | Defended results not available for Adversarial Patch attack against single modality because JPEG Compression defense is not implemented in PyTorch and so is not fully differentiable From f6fbf25ea9354307a49f204518a42a9941fcafc3 Mon Sep 17 00:00:00 2001 From: Sterling Date: Tue, 9 May 2023 13:49:33 +0000 Subject: [PATCH 6/6] update doc link to to configs --- docs/baseline_results/carla_od_results.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/baseline_results/carla_od_results.md b/docs/baseline_results/carla_od_results.md index 118ff6d71..00b1db45b 100644 --- a/docs/baseline_results/carla_od_results.md +++ b/docs/baseline_results/carla_od_results.md @@ -54,4 +54,4 @@ Multimodality (RGB+depth) Object Detection Defended results not available for Adversarial Patch attack against single modality because JPEG Compression defense is not implemented in PyTorch and so is not fully differentiable -Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval6/carla_overhead_object_detection) \ No newline at end of file +Find reference baseline configurations [here](https://github.com/twosixlabs/armory/tree/master/scenario_configs/eval7/carla_overhead_object_detection) \ No newline at end of file