armory 0.18.1 release candidate

.github/workflows/1-scan-lint-build.yml

@@ -80,17 +80,6 @@ jobs:
           fi
 
 
-      - name: 🦹‍♂️ Scanning with Bandit
-        run: |
-          bandit                                    \
-            -v                                      \
-            -f txt                                  \
-            -r ./armory                             \
-            -c "pyproject.toml"                     \
-            --output /tmp/artifacts/bandit_scan.txt \
-          || $( exit 0 ); echo $?
-
-
       - name: 🖋️ mypy Type Checking
         run: |
           python3 -m pip install mypy

.github/workflows/release.yml

@@ -64,6 +64,41 @@ jobs:
             dist/*.whl
 
 
+  release-base-image:
+    name: Build and Release Base Image
+    needs: [release-wheel]
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🐍 Setup Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: 📩 Checkout Armory w/ full depth(for tags and SCM)
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: 🌎 Setup Build Environment
+        run: |
+          pip install pip>=22.2.2
+          pip install .
+          armory configure --use-defaults
+
+      - name: ☁️ Login to DockerHub
+        if: ${{ env.RELEASE_DRY_RUN == 'false' }}
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: 🔨 Build and Push Base Image
+        run: |
+          echo "Building Base Image"
+          sed -i 's/\r$//' docker/build-base.sh
+          bash docker/build-base.sh
+
+
   release-docker:
     name: Build and Release Docker Images
     needs: [release-wheel]

README.md

@@ -135,7 +135,7 @@ Agency (DARPA).
 [python-url]: https://pypi.org/project/armory-testbed
 [license-badge]: https://img.shields.io/badge/License-MIT-yellow.svg
 [license-url]: https://opensource.org/licenses/MIT
-[docs-badge]: https://readthedocs.org/projects/armory/badge/
-[docs-url]: https://readthedocs.org/projects/armory/
+[docs-badge]: https://github.com/twosixlabs/armory/docs/assets/docs-badge.svg
+[docs-url]: https://github.com/twosixlabs/armory/docs
 [style-badge]: https://img.shields.io/badge/code%20style-black-000000.svg
 [style-url]: https://github.com/ambv/black

armory/__main__.py

@@ -19,7 +19,11 @@
 
 import armory
 from armory import arguments, paths
-from armory.cli.tools import log_current_branch, rgb_depth_convert
+from armory.cli.tools import (
+    log_current_branch,
+    plot_mAP_by_giou_with_patch_cli,
+    rgb_depth_convert,
+)
 from armory.configuration import load_global_config, save_config
 from armory.eval import Evaluator
 import armory.logs
@@ -722,6 +726,10 @@ def exec(command_args, prog, description):
 UTILS_COMMANDS = {
     "get-branch": (log_current_branch, "log the current git branch of armory"),
     "rgb-convert": (rgb_depth_convert, "converts rgb depth images to another format"),
+    "plot-mAP-by-giou": (
+        plot_mAP_by_giou_with_patch_cli,
+        "Visualize the output of the metric 'object_detection_AP_per_class_by_giou_from_patch.'",
+    ),
 }
 
 

armory/art_experimental/attacks/carla_mot_adversarial_patch.py

@@ -23,6 +23,45 @@ def __init__(self, estimator, coco_format=False, **kwargs):
 
         super().__init__(estimator=estimator, **kwargs)
 
+    # Set`loss.backward(retain_graph=False)` and zero out Adam optimizer gradients before each attack iteration
+    def _train_step(
+        self,
+        images: "torch.Tensor",
+        target: "torch.Tensor",
+        mask: Optional["torch.Tensor"] = None,
+    ) -> "torch.Tensor":
+
+        self.estimator.model.zero_grad()
+        # only zero gradients when there is a non-pgd optimizer; pgd optimizer appears to perform better when gradients accumulate
+        if self._optimizer_string == "Adam":
+            self._optimizer.zero_grad(set_to_none=True)
+        loss = self._loss(images, target, mask)
+        loss.backward(retain_graph=False)
+
+        if self._optimizer_string == "pgd":
+            if self._patch.grad is not None:
+                gradients = self._patch.grad.sign() * self.learning_rate
+            else:
+                raise ValueError("Gradient term in PyTorch model is `None`.")
+
+            with torch.no_grad():
+                self._patch[:] = torch.clamp(
+                    self._patch + gradients,
+                    min=self.estimator.clip_values[0],
+                    max=self.estimator.clip_values[1],
+                )
+        else:
+            self._optimizer.step()
+
+            with torch.no_grad():
+                self._patch[:] = torch.clamp(
+                    self._patch,
+                    min=self.estimator.clip_values[0],
+                    max=self.estimator.clip_values[1],
+                )
+
+        return loss
+
     def create_initial_image(self, size):
         """
         Create initial patch based on a user-defined image
@@ -400,7 +439,7 @@ def generate(self, x, y, y_patch_metadata):
             # Use this mask to embed patch into the background in the event of occlusion
             self.patch_masks_video = y_patch_metadata[i]["masks"]
 
-            # self._patch needs to be re-initialized with the correct shape
+            # self._patch and optimizer need to be re-initialized
             if self.patch_base_image is not None:
                 self.patch_base = self.create_initial_image(
                     (patch_width, patch_height),
@@ -412,6 +451,8 @@ def generate(self, x, y, y_patch_metadata):
             self._patch = torch.tensor(
                 patch_init, requires_grad=True, device=self.estimator.device
             )
+            if self._optimizer_string == "Adam":
+                self._optimizer = torch.optim.Adam([self._patch], lr=self.learning_rate)
 
             # Perform batch attack by attacking multiple frames from the same video
             for batch_i in range(0, x[i].shape[0], self.batch_frame_size):

armory/art_experimental/attacks/carla_obj_det_adversarial_patch.py

@@ -115,8 +115,13 @@ def _train_step(
         import torch  # lgtm [py/repeated-import]
 
         self.estimator.model.zero_grad()
+        # only zero gradients when there is a non-pgd optimizer; pgd optimizer appears to perform better when gradients accumulate
+        if self._optimizer_string == "Adam":
+            self._optimizer_rgb.zero_grad(set_to_none=True)
+            if images.shape[-1] == 6:
+                self._optimizer_depth.zero_grad(set_to_none=True)
         loss = self._loss(images, target, mask)
-        loss.backward(retain_graph=True)
+        loss.backward(retain_graph=False)
 
         if self._optimizer_string == "pgd":
             patch_grads = self._patch.grad
@@ -159,7 +164,6 @@ def _train_step(
                             min=self.min_depth,
                             max=self.max_depth,
                         )
-                        self.depth_perturbation[:] = perturbed_images - images_depth
                     else:
                         images_depth_linear = rgb_depth_to_linear(
                             images_depth[:, 0, :, :],
@@ -175,12 +179,49 @@ def _train_step(
                         perturbed_images = torch.stack(
                             [depth_r, depth_g, depth_b], dim=1
                         )
-                        self.depth_perturbation[:] = perturbed_images - images_depth
+                    self.depth_perturbation[:] = perturbed_images - images_depth
 
         else:
-            raise ValueError(
-                "Adam optimizer for CARLA Adversarial Patch not supported."
-            )
+            self._optimizer_rgb.step()
+            if images.shape[-1] == 6:
+                self._optimizer_depth.step()
+
+            with torch.no_grad():
+                self._patch[:] = torch.clamp(
+                    self._patch,
+                    min=self.estimator.clip_values[0],
+                    max=self.estimator.clip_values[1],
+                )
+
+                if images.shape[-1] == 6:
+                    images_depth = torch.permute(images[:, :, :, 3:], (0, 3, 1, 2))
+                    if self.depth_type == "log":
+                        perturbed_images = torch.clamp(
+                            images_depth + self.depth_perturbation,
+                            min=self.min_depth,
+                            max=self.max_depth,
+                        )
+                    else:
+                        images_depth_linear = rgb_depth_to_linear(
+                            images_depth[:, 0, :, :],
+                            images_depth[:, 1, :, :],
+                            images_depth[:, 2, :, :],
+                        )
+                        depth_linear = rgb_depth_to_linear(
+                            self.depth_perturbation[:, 0, :, :],
+                            self.depth_perturbation[:, 1, :, :],
+                            self.depth_perturbation[:, 2, :, :],
+                        )
+                        depth_linear = torch.clamp(
+                            images_depth_linear + depth_linear,
+                            min=self.min_depth,
+                            max=self.max_depth,
+                        )
+                        depth_r, depth_g, depth_b = linear_depth_to_rgb(depth_linear)
+                        perturbed_images = torch.stack(
+                            [depth_r, depth_g, depth_b], dim=1
+                        )
+                    self.depth_perturbation[:] = perturbed_images - images_depth
 
         return loss
 
@@ -348,6 +389,9 @@ def _random_overlay(
         ).to(self.estimator.device)
         foreground_mask = torch.permute(foreground_mask, (2, 0, 1))
         foreground_mask = torch.unsqueeze(foreground_mask, dim=0)
+        foreground_mask = ~(
+            ~foreground_mask * image_mask.bool()
+        )  # ensure area perturbed in depth is consistent with area perturbed in RGB
 
         # Adjust green screen brightness
         v_avg = (
@@ -501,6 +545,15 @@ def generate(self, x, y=None, y_patch_metadata=None):
                     device=self.estimator.device,
                 )
 
+            if self._optimizer_string == "Adam":
+                self._optimizer_rgb = torch.optim.Adam(
+                    [self._patch], lr=self.learning_rate
+                )
+                if x.shape[-1] == 6:
+                    self._optimizer_depth = torch.optim.Adam(
+                        [self.depth_perturbation], lr=self.learning_rate_depth
+                    )
+
             patch, _ = super().generate(np.expand_dims(x[i], axis=0), y=[y_gt])
 
             # Patch image

armory/art_experimental/attacks/carla_obj_det_patch.py

@@ -10,9 +10,9 @@
 
 from armory.art_experimental.attacks.carla_obj_det_utils import (
     linear_depth_to_rgb,
-    rgb_depth_to_linear,
     linear_to_log,
     log_to_linear,
+    rgb_depth_to_linear,
 )
 from armory.logs import log
 from armory.utils.external_repo import ExternalRepoImport
@@ -450,7 +450,9 @@ def inner_generate(  # type: ignore
                 if self.depth_type == "log":
                     depth_log = (
                         self.depth_perturbation
-                        + np.sign(depth_gradients) * self.learning_rate_depth
+                        + np.sign(depth_gradients)
+                        * (1 - 2 * int(self.targeted))
+                        * self.learning_rate_depth
                     )
                     perturbed_images = np.clip(
                         images_depth + depth_log, self.min_depth, self.max_depth
@@ -468,7 +470,10 @@ def inner_generate(  # type: ignore
                         self.depth_perturbation[:, :, :, 2],
                     ).astype("float32")
                     depth_linear = (
-                        depth_linear + np.sign(grads_linear) * self.learning_rate_depth
+                        depth_linear
+                        + np.sign(grads_linear)
+                        * (1 - 2 * int(self.targeted))
+                        * self.learning_rate_depth
                     )
 
                     images_depth_linear = rgb_depth_to_linear(
@@ -770,6 +775,23 @@ def generate(self, x, y=None, y_patch_metadata=None):
                 )  # (1,H,W,3)
                 self.foreground = np.all(self.binarized_patch_mask == 255, axis=-1)
                 self.foreground = np.expand_dims(self.foreground, (-1, 0))  # (1,H,W,1)
+                # ensure area perturbed in depth is consistent with area perturbed in RGB
+                h, _ = cv2.findHomography(
+                    np.array(
+                        [
+                            [0, 0],
+                            [patch_width - 1, 0],
+                            [patch_width - 1, patch_height - 1],
+                            [0, patch_height - 1],
+                        ]
+                    ),
+                    gs_coords,
+                )
+                rgb_mask = np.ones((patch_height, patch_width, 3), dtype=np.float32)
+                rgb_mask = cv2.warpPerspective(
+                    rgb_mask, h, (x.shape[2], x.shape[1]), cv2.INTER_CUBIC
+                )
+                self.foreground = self.foreground * rgb_mask[:, :, 0:1]
 
             if y is None:
                 patch = self.inner_generate(

armory/art_experimental/attacks/sweep.py

@@ -142,7 +142,7 @@ def _is_robust(self, y, y_pred):
             return metric_result > self.metric_threshold
 
     def _get_metric_result(self, y, y_pred):
-        if isinstance(y, np.ndarray) and y.dtype == np.object:
+        if isinstance(y, np.ndarray) and y.dtype == object:
             # convert np object array to list of dicts
             metric_result = self.metric_fn([y[0]], y_pred)
         else:

armory/baseline_models/model_configs/__init__.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from armory.data.utils import maybe_download_weights_from_s3
+from armory.logs import log
+
+CONFIGS_DIR = Path(__file__).parent
+
+
+def get_path(filename) -> str:
+    """
+    Get the absolute path of the provided config. Ordering priority is:
+        1) Check directly for provided filepath
+        2) Load from `model_configs` directory
+        3) Attempt to download from s3 as a weights file
+    """
+    filename = Path(filename)
+    if filename.is_file():
+        return str(filename)
+    cfgs_path = CONFIGS_DIR / filename
+    if cfgs_path.is_file():
+        return str(cfgs_path)
+
+    return maybe_download_weights_from_s3(filename)

armory/baseline_models/pytorch/yolov3.py

@@ -1,11 +1,11 @@
 from typing import Optional
 
-import torch
-
 from art.estimators.object_detection import PyTorchYolo
-from pytorchyolo.utils.loss import compute_loss
 from pytorchyolo.models import load_model
+from pytorchyolo.utils.loss import compute_loss
+import torch
 
+from armory.baseline_models import model_configs
 
 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -30,6 +30,7 @@ def get_art_model(
     model_kwargs: dict, wrapper_kwargs: dict, weights_path: Optional[str] = None
 ) -> PyTorchYolo:
 
+    model_kwargs["model_path"] = model_configs.get_path(model_kwargs["model_path"])
     model = load_model(weights_path=weights_path, **model_kwargs)
     model_wrapper = Yolo(model)