Armory v0.18.0 release candidate

.github/workflows/release.yml

@@ -35,7 +35,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          ref: ${{ github.event.client_payload.branch }}
 
       - name: 🔨 Build wheel
         run: |
@@ -86,7 +85,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          ref: ${{ github.event.client_payload.branch }}
 
       - name: 🌎 Setup Build Environment
         run: |

armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn.py

@@ -108,6 +108,11 @@ def get_art_model_mm(
 ) -> PyTorchFasterRCNN:
 
     num_classes = model_kwargs.pop("num_classes", 3)
+    frcnn_kwargs = {
+        arg: model_kwargs.pop(arg)
+        for arg in ["min_size", "max_size"]
+        if arg in model_kwargs
+    }
 
     backbone = MultimodalNaive(**model_kwargs)
 
@@ -116,6 +121,7 @@ def get_art_model_mm(
         num_classes=num_classes,
         image_mean=[0.485, 0.456, 0.406, 0.0, 0.0, 0.0],
         image_std=[0.229, 0.224, 0.225, 1.0, 1.0, 1.0],
+        **frcnn_kwargs,
     )
     model.to(DEVICE)
 

armory/baseline_models/pytorch/carla_multimodality_object_detection_frcnn_robust_fusion.py

@@ -182,6 +182,11 @@ def get_art_model_mm_robust(
 ) -> PyTorchFasterRCNN:
 
     num_classes = model_kwargs.pop("num_classes", 3)
+    frcnn_kwargs = {
+        arg: model_kwargs.pop(arg)
+        for arg in ["min_size", "max_size"]
+        if arg in model_kwargs
+    }
 
     backbone = MultimodalRobust(**model_kwargs)
 
@@ -190,6 +195,7 @@ def get_art_model_mm_robust(
         num_classes=num_classes,
         image_mean=[0.485, 0.456, 0.406, 0.0, 0.0, 0.0],
         image_std=[0.229, 0.224, 0.225, 1.0, 1.0, 1.0],
+        **frcnn_kwargs,
     )
     model.to(DEVICE)
 

armory/data/utils.py

@@ -248,7 +248,7 @@ def move_merge(source, dest):
 
 def download_verify_dataset_cache(dataset_dir, checksum_file, name):
     found_checksum_flag = False
-    log.info("Attempting download_verigy_dataset_cache with dataset_dir")
+    log.info("Attempting download_verify_dataset_cache with dataset_dir")
     for checksum_dir in CHECKSUMS_DIRS:
         checksum_file_full_path = os.path.join(checksum_dir, checksum_file)
         if os.path.exists(checksum_file_full_path):

docs/baseline_results/object_detection_poisoning_results.md

@@ -0,0 +1,186 @@
+# Object Detection Poisoning Baseline Evaluation
+
+Containing results for Global Misclassification and Object Generation attacks.
+
+**All tables are the mean of 3 runs.**  Results obtained with Armory 0.17.2 June 2023.
+
+Relevant parameters:
+```
+"target_class": 1
+"score_threshold": 0.05,
+```
+(Source class is N/A for these two attacks)
+
+See [the paper](https://arxiv.org/pdf/2205.14497.pdf) for a detailed description of each metric recorded.
+
+# Global Misclassification Attack
+
+## Globe trigger
+
+### Undefended
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.527 | 0.460 | - | - | - | - | - |
+| 01 |  0.499 | 0.467 | 0.500 | 0.470 | 0.081 | 0.243 | 0.016 |
+| 05 |  0.514 | 0.493 | 0.169 | 0.313 | 0.149 | 0.447 | 0.620 |
+| 10 |  0.487 | 0.463 | 0.173 | 0.323 | 0.152 | 0.457 | 0.635 |
+| 20 |  0.510 | 0.467 | 0.166 | 0.317 | 0.164 | 0.493 | 0.701 |
+| 30 |  0.472 | 0.453 | 0.156 | 0.327 | 0.162 | 0.487 | 0.688 |
+
+
+### Random Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.432 | 0.403 | - | - | - | - | - |
+| 01 |  0.448 | 0.430 | 0.447 | 0.427 | 0.074 | 0.223 | 0.014 |
+| 05 |  0.441 | 0.430 | 0.221 | 0.340 | 0.118 | 0.353 | 0.427 |
+| 10 |  0.448 | 0.450 | 0.160 | 0.300 | 0.144 | 0.433 | 0.587 |
+| 20 |  0.460 | 0.430 | 0.163 | 0.297 | 0.149 | 0.447 | 0.637 |
+| 30 |  0.424 | 0.423 | 0.126 | 0.277 | 0.152 | 0.457 | 0.672 |
+
+
+### Perfect Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.518 | 0.460 | - | - | - | - | - |
+| 01 |  0.549 | 0.473 | 0.549 | 0.473 | 0.082 | 0.247 | 0.024 |
+| 05 |  0.538 | 0.473 | 0.539 | 0.473 | 0.081 | 0.243 | 0.016 |
+| 10 |  0.516 | 0.457 | 0.517 | 0.457 | 0.079 | 0.237 | 0.020 |
+| 20 |  0.480 | 0.440 | 0.480 | 0.437 | 0.074 | 0.223 | 0.014 |
+| 30 |  0.481 | 0.423 | 0.480 | 0.427 | 0.073 | 0.220 | 0.012 |
+
+
+
+
+## Baby-on-board trigger
+
+### Undefended
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.518 | 0.457 | - | - | - | - | - |
+| 01 |  0.508 | 0.487 | 0.507 | 0.487 | 0.080 | 0.240 | 0.014 |
+| 05 |  0.488 | 0.473 | 0.379 | 0.433 | 0.106 | 0.317 | 0.219 |
+| 10 |  0.502 | 0.457 | 0.157 | 0.290 | 0.149 | 0.447 | 0.623 |
+| 20 |  0.524 | 0.483 | 0.173 | 0.343 | 0.173 | 0.520 | 0.704 |
+| 30 |  0.484 | 0.460 | 0.140 | 0.270 | 0.142 | 0.397 | 0.566 |
+
+
+### Random Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.454 | 0.430 | - | - | - | - | - |
+| 01 |  0.463 | 0.417 | 0.464 | 0.417 | 0.076 | 0.227 | 0.024 |
+| 05 |  0.427 | 0.417 | 0.350 | 0.390 | 0.089 | 0.267 | 0.186 |
+| 10 |  0.462 | 0.450 | 0.156 | 0.277 | 0.146 | 0.437 | 0.592 |
+| 20 |  0.461 | 0.443 | 0.146 | 0.273 | 0.160 | 0.480 | 0.690 |
+| 30 |  0.406 | 0.403 | 0.121 | 0.283 | 0.159 | 0.420 | 0.628 |
+
+
+### Perfect Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.519 | 0.463 | - | - | - | - | - |
+| 01 |  0.532 | 0.467 | 0.532 | 0.467 | 0.082 | 0.247 | 0.026 |
+| 05 |  0.521 | 0.460 | 0.520 | 0.460 | 0.079 | 0.237 | 0.017 |
+| 10 |  0.529 | 0.457 | 0.530 | 0.457 | 0.080 | 0.240 | 0.016 |
+| 20 |  0.518 | 0.447 | 0.519 | 0.450 | 0.079 | 0.237 | 0.014 |
+| 30 |  0.493 | 0.447 | 0.494 | 0.447 | 0.077 | 0.230 | 0.015 |
+
+
+
+
+
+# Object Generation Attack
+
+Generated box parameters:
+```
+"bbox_height": 70,
+"bbox_width": 50
+```
+
+## Globe Trigger
+
+### Undefended
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.529 | 0.463 | - | - | - | - | - |
+| 01 |  0.524 | 0.493 | - | - | 0.499 | 0.457 | 0.709 |
+| 05 |  0.514 | 0.473 | - | - | 0.519 | 0.533 | 0.915 |
+| 10 |  0.534 | 0.487 | - | - | 0.533 | 0.560 | 0.931 |
+| 20 |  0.529 | 0.477 | - | - | 0.542 | 0.553 | 0.925 |
+| 30 |  0.534 | 0.477 | - | - | 0.531 | 0.547 | 0.925 |
+
+
+### Random Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.459 | 0.430 | - | - | - | - | - |
+| 01 |  0.470 | 0.443 | - | - | 0.426 | 0.417 | 0.666 |
+| 05 |  0.487 | 0.447 | - | - | 0.466 | 0.513 | 0.933 |
+| 10 |  0.440 | 0.423 | - | - | 0.444 | 0.520 | 0.919 |
+| 20 |  0.430 | 0.397 | - | - | 0.450 | 0.490 | 0.942 |
+| 30 |  0.507 | 0.453 | - | - | 0.510 | 0.530 | 0.929 |
+
+
+### Perfect Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.530 | 0.470 | - | - | - | - | - |
+| 01 |  0.526 | 0.463 | - | - | 0.449 | 0.227 | 0.001 |
+| 05 |  0.516 | 0.450 | - | - | 0.437 | 0.220 | 0.002 |
+| 10 |  0.520 | 0.453 | - | - | 0.430 | 0.217 | 0.001 |
+| 20 |  0.492 | 0.447 | - | - | 0.409 | 0.213 | 0.000 |
+| 30 |  0.456 | 0.440 | - | - | 0.379 | 0.213 | 0.000 |
+
+
+
+
+## Baby-on-board trigger
+
+### Undefended
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.506 | 0.463 | - | - | - | - | - |
+| 01 |  0.524 | 0.457 | - | - | 0.496 | 0.500 | 0.840 |
+| 05 |  0.537 | 0.470 | - | - | 0.526 | 0.547 | 0.930 |
+| 10 |  0.527 | 0.467 | - | - | 0.537 | 0.553 | 0.940 |
+| 20 |  0.527 | 0.457 | - | - | 0.556 | 0.570 | 0.934 |
+| 30 |  0.548 | 0.487 | - | - | 0.559 | 0.573 | 0.935 |
+
+
+### Random Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.452 | 0.433 | - | - | - | - | - |
+| 01 |  0.440 | 0.423 | - | - | 0.413 | 0.427 | 0.856 |
+| 05 |  0.456 | 0.437 | - | - | 0.453 | 0.523 | 0.919 |
+| 10 |  0.494 | 0.423 | - | - | 0.474 | 0.507 | 0.932 |
+| 20 |  0.456 | 0.433 | - | - | 0.462 | 0.507 | 0.939 |
+| 30 |  0.499 | 0.457 | - | - | 0.516 | 0.530 | 0.930 |
+
+
+### Perfect Filter
+
+| Poison Percentage | Benign mAP | Benign AP target | Adv mAP - Clean labels | Adv AP target - Clean labels | Adv mAP - Adv labels | Adv AP target - Adv labels | Attack success rate |
+| ------- | ------- | ------- | ------- | ------- | ------- | ------- | ------- |
+| 00 |  0.532 | 0.480 | - | - | - | - | - |
+| 01 |  0.526 | 0.460 | - | - | 0.444 | 0.223 | 0.000 |
+| 05 |  0.529 | 0.460 | - | - | 0.446 | 0.220 | 0.001 |
+| 10 |  0.527 | 0.447 | - | - | 0.442 | 0.220 | 0.001 |
+| 20 |  0.503 | 0.443 | - | - | 0.418 | 0.213 | 0.000 |
+| 30 |  0.474 | 0.437 | - | - | 0.399 | 0.210 | 0.001 |
+
+
+
+

docs/configuration_files.md

@@ -70,7 +70,7 @@ All configuration files are verified against the jsonschema definition at run ti
     external_github_repo: [String or null or Object] External github repository(s) to download and place on PYTHONPATH within container
     external_github_repo_pythonpath: [String or null or Object] Relative path(s) in the repo directory to add to PYTHONPATH within container
     gpus: [String]: Which GPUs should the docker container have access to. "all" or comma sperated list (e.g. "1,3")
-    local_repo_path: [String or null or Object] Local github repository path(s) to place on PYTHONPATH within container
+    local_repo_path: [String or null or Object] Local github repository path(s) to place on PYTHONPATH within container (relative to the "local_git_dir" variable specified in ~/.armory/config.json)
     output_dir: [Optional String]:  Add an optional output directory prefix to the default output directory name.
     output_filename: [Optional String]: Optionally change the output filename prefix (from default of scenario name)  
     use_gpu: [Boolean]: Boolean to run container as nvidia-docker with GPU access

docs/poisoning.md

@@ -1,13 +1,13 @@
 # Poisoning
 
-Updated October 2022
+Updated June 2023
 
 Armory supports a handful of specific poisoning threat models and attacks.  This document will first describe these, providing enough background for newcomers to get up to speed on what these attacks do.  Then, the peculiarities of the poisoning configs will be addressed, including lots of helpful information about Witches' Brew.  Finally, we will describe the poisoning-specific metrics.
 
 
 ## Threat Models
 
-There are currently four threat models handled by Armory: dirty-label backdoor, clean-label backdoor, Witches' Brew (clean-label gradient matching), and Sleeper Agent.  In a backdoor attack, an adversary adds a small trigger, or backdoor, to a small portion of the train set in order to gain control of the the model at test time.
+Armory supports several poisoning threat models: dirty-label backdoor, clean-label backdoor, Witches' Brew and Sleeper Agent (clean-label gradient matching), and Object Detection (dirty-label backdoor).  In a backdoor attack, an adversary adds a small trigger, or backdoor, to a small portion of the train set in order to gain control of the the model at test time.
 The trigger is usually a small (but not imperceptible) image superposed on the data, and the adversary's goal is to force the model to misclassify test images that have the trigger applied.  Armory includes several trigger images under `utils/triggers/`.
 
 
@@ -20,7 +20,7 @@ In a [Dirty-label Backdoor (DLBD) Attack](https://arxiv.org/abs/1708.06733), tra
 
 #### Audio
 
-The DLBD attack for audio is similar to that of video. The difference is that instead of the trigger being an image that is placed over the existing image, the trigger is a short audio clip that is mixed with the existing audio. Example configs for speech are [here](../scenario_configs/eval6/poisoning)
+The DLBD attack for audio is similar to that of image. The difference is that instead of the trigger being an image that is placed over the existing image, the trigger is a short audio clip that is mixed with the existing audio. Example configs for speech are [here](../scenario_configs/eval6/poisoning)
 Current triggers include a whistle and clapping.
 
 ### Clean-label backdoor

docs/scenarios.md

@@ -546,7 +546,7 @@ For a complete overview, see the [poisoning doc](poisoning.md).
   * [Random Filter](https://github.com/twosixlabs/armory/blob/master/armory/art_experimental/poison_detection/random_filter.py)
   * [Perfect Filter](https://github.com/twosixlabs/armory/blob/1d6caa9166313c1409edbbc5f089d2bc774b5230/armory/scenarios/poison.py#L233-L235)
 * **Baseline Evaluations:**
-  * Minicoco Results to be added
+  * [MiniCoco results](baseline_results/object_detection_poisoning_results.md)
 
 
 

scenario_configs/eval7/carla_mot/carla_mot_adversarialpatch_undefended.json

@@ -39,6 +39,8 @@
         "fit": false,
         "fit_kwargs": {},
         "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960,
             "num_classes": 2
         },
         "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte",

scenario_configs/eval7/carla_mot/carla_mot_dpatch_defended.json

@@ -49,6 +49,8 @@
         "fit": false,
         "fit_kwargs": {},
         "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960,
             "num_classes": 2
         },
         "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte",

scenario_configs/eval7/carla_mot/carla_mot_dpatch_undefended.json

@@ -36,6 +36,8 @@
         "fit": false,
         "fit_kwargs": {},
         "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960,
             "num_classes": 2
         },
         "module": "armory.baseline_models.pytorch.carla_mot_frcnn_byte",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json

@@ -41,6 +41,8 @@
         "fit": false,
         "fit_kwargs": {},
         "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960,
             "num_classes": 3
         },
         "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json

@@ -52,6 +52,8 @@
         "fit": false,
         "fit_kwargs": {},
         "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960,
             "num_classes": 3
         },
         "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json

@@ -39,6 +39,8 @@
         "fit": false,
         "fit_kwargs": {},
         "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960,
             "num_classes": 3
         },
         "module": "armory.baseline_models.pytorch.carla_single_modality_object_detection_frcnn",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json

@@ -42,7 +42,10 @@
     "model": {
         "fit": false,
         "fit_kwargs": {},
-        "model_kwargs": {},
+        "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960
+        },
         "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion",
         "name": "get_art_model_mm_robust",
         "weights_file": "carla_multimodal_robust_weights_eval7and8.pt",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json

@@ -42,7 +42,10 @@
     "model": {
         "fit": false,
         "fit_kwargs": {},
-        "model_kwargs": {},
+        "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960
+        },
         "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn",
         "name": "get_art_model_mm",
         "weights_file": "carla_multimodal_naive_weights_eval7and8.pt",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json

@@ -40,7 +40,10 @@
     "model": {
         "fit": false,
         "fit_kwargs": {},
-        "model_kwargs": {},
+        "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960
+        },
         "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn_robust_fusion",
         "name": "get_art_model_mm_robust",
         "weights_file": "carla_multimodal_robust_weights_eval7and8.pt",

scenario_configs/eval7/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json

@@ -40,7 +40,10 @@
     "model": {
         "fit": false,
         "fit_kwargs": {},
-        "model_kwargs": {},
+        "model_kwargs": {
+            "max_size": 1280,
+            "min_size": 960
+        },
         "module": "armory.baseline_models.pytorch.carla_multimodality_object_detection_frcnn",
         "name": "get_art_model_mm",
         "weights_file": "carla_multimodal_naive_weights_eval7and8.pt",