Generic Masked Patch Attack

.github/workflows/3-test-docker.yml

@@ -13,6 +13,14 @@ jobs:
     name: ☁️ Docker Armory Image Tests
     runs-on: ubuntu-latest
     steps:
+      - name: 💿 Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+
       - name: 🐄 checkout armory full depth with tags for scm
         uses: actions/checkout@v3
         with:
@@ -50,6 +58,14 @@ jobs:
     name: ☁️ Docker Deepspeech Image Tests
     runs-on: ubuntu-latest
     steps:
+      - name: 💿 Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+
       - name: 🐄 checkout armory full depth with tags for scm
         uses: actions/checkout@v3
         with:
@@ -96,6 +112,14 @@ jobs:
     name: ☁️ Docker Yolo Image Tests
     runs-on: ubuntu-latest
     steps:
+      - name: 💿 Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+
       - name: 🐄 checkout armory full depth with tags for scm
         uses: actions/checkout@v3
         with:
@@ -114,4 +138,4 @@ jobs:
 
       - name: 🚧 Build the Container
         run: |
-          python docker/build.py --framework yolo
+          python docker/build.py --framework yolo

.github/workflows/release.yml

@@ -69,6 +69,14 @@ jobs:
     needs: [release-wheel]
     runs-on: ubuntu-latest
     steps:
+      - name: 💿 Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+
       - name: 🐍 Setup Python 3.9
         uses: actions/setup-python@v4
         with:
@@ -111,6 +119,14 @@ jobs:
           - image: pytorch-deepspeech
           - image: yolo
     steps:
+      - name: 💿 Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+
       - name: 🐍 Setup Python 3.9
         uses: actions/setup-python@v4
         with:
@@ -174,4 +190,4 @@ jobs:
 
 # Workflow Test:
 # act --detect-event -j release-wheel
-# act workflow_dispatch -j release-docker --eventpath .github/workflows/tests/release-dry-run.json
+# act workflow_dispatch -j release-docker --eventpath .github/workflows/tests/release-dry-run.json

armory/__main__.py

@@ -19,11 +19,7 @@
 
 import armory
 from armory import arguments, paths
-from armory.cli.tools import (
-    log_current_branch,
-    plot_mAP_by_giou_with_patch_cli,
-    rgb_depth_convert,
-)
+from armory.cli import CLI_COMMANDS
 from armory.configuration import load_global_config, save_config
 from armory.eval import Evaluator
 import armory.logs
@@ -632,7 +628,6 @@ def configure(command_args, prog, description):
     print(resolved)
     save = None
     while save is None:
-
         if os.path.isfile(default_host_paths.armory_config):
             print("WARNING: this will overwrite existing configuration.")
             print("    Press Ctrl-C to abort.")
@@ -723,16 +718,6 @@ def exec(command_args, prog, description):
     sys.exit(exit_code)
 
 
-UTILS_COMMANDS = {
-    "get-branch": (log_current_branch, "log the current git branch of armory"),
-    "rgb-convert": (rgb_depth_convert, "converts rgb depth images to another format"),
-    "plot-mAP-by-giou": (
-        plot_mAP_by_giou_with_patch_cli,
-        "Visualize the output of the metric 'object_detection_AP_per_class_by_giou_from_patch.'",
-    ),
-}
-
-
 def utils_usage():
     lines = [
         f"{PROGRAM} <command>",
@@ -742,7 +727,7 @@ def utils_usage():
         "",
         "Commands:",
     ]
-    for name, (func, description) in UTILS_COMMANDS.items():
+    for name, (func, description) in CLI_COMMANDS.items():
         lines.append(f"    {name} - {description}")
     lines.extend(
         [
@@ -759,12 +744,12 @@ def utils(command_args, prog, description):
     parser = argparse.ArgumentParser(prog=prog, usage=utils_usage())
     parser.add_argument(
         "command",
-        choices=UTILS_COMMANDS.keys(),
+        choices=CLI_COMMANDS.keys(),
         help="utility command to run",
     )
     args = parser.parse_args(sys.argv[2:3])
 
-    func, description = UTILS_COMMANDS[args.command]
+    func, description = CLI_COMMANDS[args.command]
     prog = f"{PROGRAM} {args.command}"
     return func(sys.argv[3:], prog, description)
 

armory/art_experimental/attacks/carla_obj_det_adversarial_patch.py

@@ -1,41 +1,37 @@
-import os
-import sys
 from typing import Optional
 
 from art.attacks.evasion.adversarial_patch.adversarial_patch_pytorch import (
     AdversarialPatchPyTorch,
 )
 import cv2
 import numpy as np
-import requests
 import torch
 
-from armory import paths
 from armory.art_experimental.attacks.carla_obj_det_utils import (
+    PatchMask,
+    fetch_image_from_file_or_url,
     linear_depth_to_rgb,
     linear_to_log,
     log_to_linear,
     rgb_depth_to_linear,
 )
 from armory.logs import log
 
-VALID_IMAGE_TYPES = ["image/png", "image/jpeg", "image/jpg"]
-
 
 class CARLAAdversarialPatchPyTorch(AdversarialPatchPyTorch):
     """
     Apply patch attack to RGB channels and (optionally) masked PGD attack to depth channels.
     """
 
     def __init__(self, estimator, **kwargs):
-
         # Maximum depth perturbation from a flat patch
         self.depth_delta_meters = kwargs.pop("depth_delta_meters", 3)
         self.learning_rate_depth = kwargs.pop("learning_rate_depth", 0.0001)
         self.depth_perturbation = None
         self.min_depth = None
         self.max_depth = None
         self.patch_base_image = kwargs.pop("patch_base_image", None)
+        self.patch_mask = PatchMask.from_kwargs(kwargs.pop("patch_mask", None))
 
         # HSV bounds are user-defined to limit perturbation regions
         self.hsv_lower_bound = np.array(
@@ -52,67 +48,27 @@ def create_initial_image(self, size, hsv_lower_bound, hsv_upper_bound):
         Create initial patch based on a user-defined image and
         create perturbation mask based on HSV bounds
         """
-        module_path = globals()["__file__"]
-        module_folder = os.path.dirname(module_path)
-        # user-defined image is assumed to reside in the same location as the attack module
-        patch_base_image_path = os.path.abspath(
-            os.path.join(module_folder, self.patch_base_image)
-        )
-        # if the image does not exist iterate through path
-        if not os.path.exists(patch_base_image_path):
-            _path = sys.path.copy()
-            if (_cwd := paths.runtime_paths().cwd) not in _path:
-                _path.insert(0, _cwd)
-            for path in _path:
-                patch_base_image_path = os.path.abspath(
-                    os.path.join(path, self.patch_base_image)
-                )
-                if os.path.exists(patch_base_image_path):
-                    break
-            del _path, _cwd
-        # image not in cwd or module, check if it is a url to an image
-        if not os.path.exists(patch_base_image_path):
-            # Send a HEAD request
-            response = requests.head(self.patch_base_image, allow_redirects=True)
-
-            # Check the status code
-            if response.status_code != 200:
-                raise FileNotFoundError(
-                    f"Cannot find patch base image at {self.patch_base_image}. "
-                    f"Make sure it is in your cwd or {module_folder} or provide a valid url."
-                )
-            # If the status code is 200, check the content type
-            content_type = response.headers.get("content-type")
-            if content_type not in VALID_IMAGE_TYPES:
-                raise ValueError(
-                    f"Returned content at {self.patch_base_image} is not a valid image type. "
-                    f"Expected types are {VALID_IMAGE_TYPES}, but received {content_type}"
-                )
-
-            # If content type is valid, download the image
-            response = requests.get(self.patch_base_image, allow_redirects=True)
-            im = cv2.imdecode(
-                np.frombuffer(response.content, np.uint8), cv2.IMREAD_COLOR
+        if not isinstance(self.patch_base_image, str):
+            raise ValueError(
+                "patch_base_image must be a string path to an image or a url to an image"
             )
-        else:
-            im = cv2.imread(patch_base_image_path)
-
+        im = fetch_image_from_file_or_url(self.patch_base_image)
         im = cv2.resize(im, size)
         im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
 
         hsv = cv2.cvtColor(im, cv2.COLOR_RGB2HSV)
         # find the colors within the boundaries
-        mask = cv2.inRange(hsv, hsv_lower_bound, hsv_upper_bound)
-        mask = np.expand_dims(mask, 2)
+        color_mask = cv2.inRange(hsv, hsv_lower_bound, hsv_upper_bound)
+        color_mask = np.expand_dims(color_mask, 2)
         # cv2.imwrite(
-        #     "mask.png", mask
+        #     "color_mask.png", color_mask
         # )  # visualize perturbable regions. Comment out if not needed.
 
         patch_base = np.transpose(im, (2, 0, 1))
         patch_base = patch_base / 255.0
-        mask = np.transpose(mask, (2, 0, 1))
-        mask = mask / 255.0
-        return patch_base, mask
+        color_mask = np.transpose(color_mask, (2, 0, 1))
+        color_mask = color_mask / 255.0
+        return patch_base, color_mask
 
     def _train_step(
         self,
@@ -133,7 +89,9 @@ def _train_step(
 
         if self._optimizer_string == "pgd":
             patch_grads = self._patch.grad
-            patch_gradients = patch_grads.sign() * self.learning_rate * self.patch_mask
+            patch_gradients = (
+                patch_grads.sign() * self.learning_rate * self.patch_color_mask
+            )
 
             if images.shape[-1] == 6:
                 depth_grads = self.depth_perturbation.grad
@@ -331,7 +289,6 @@ def _random_overlay(
         padded_patch_list = []
 
         for i_sample in range(nb_samples):
-
             image_mask_i = image_mask[i_sample]
 
             height = padded_patch.shape[self.i_h + 1]
@@ -481,6 +438,15 @@ def generate(self, x, y=None, y_patch_metadata=None):
             # Use this mask to embed patch into the background in the event of occlusion
             self.binarized_patch_mask = y_patch_metadata[i]["mask"]
 
+            # Add patch mask to the image mask
+            if self.patch_mask is not None:
+                orig_patch_mask = self.binarized_patch_mask.copy()
+                projected_mask = self.patch_mask.project(
+                    self.binarized_patch_mask.shape, gs_coords, as_bool=True
+                )
+                # binarized_patch_mask already handled in loss function
+                self.binarized_patch_mask *= projected_mask
+
             # Eval7 contains a mixture of patch locations.
             # Patches that lie flat on the sidewalk or street are constrained to 0.03m depth perturbation, and they are best used to create disappearance errors.
             # Patches located elsewhere (i.e., that do not impede pedestrian/vehicle motion) are constrained to 3m depth perturbation, and they are best used to create hallucinations.
@@ -498,19 +464,21 @@ def generate(self, x, y=None, y_patch_metadata=None):
 
             # self._patch needs to be re-initialized with the correct shape
             if self.patch_base_image is not None:
-                patch_init, patch_mask = self.create_initial_image(
+                patch_init, patch_color_mask = self.create_initial_image(
                     (patch_width, patch_height),
                     self.hsv_lower_bound,
                     self.hsv_upper_bound,
                 )
             else:
                 patch_init = np.random.randint(0, 255, size=self.patch_shape) / 255
-                patch_mask = np.ones_like(patch_init)
+                patch_color_mask = np.ones_like(patch_init)
 
             self._patch = torch.tensor(
                 patch_init, requires_grad=True, device=self.estimator.device
             )
-            self.patch_mask = torch.Tensor(patch_mask).to(self.estimator.device)
+            self.patch_color_mask = torch.Tensor(patch_color_mask).to(
+                self.estimator.device
+            )
 
             # initialize depth variables
             if x.shape[-1] == 6:
@@ -583,6 +551,16 @@ def generate(self, x, y=None, y_patch_metadata=None):
                 np.all(self.binarized_patch_mask == 0, axis=-1)
             ]
 
+            # Embed patch mask fill into masked region
+            if self.patch_mask is not None:
+                patched_image = self.patch_mask.fill_masked_region(
+                    patched_image=patched_image,
+                    projected_mask=projected_mask,
+                    gs_coords=gs_coords,
+                    patch_init=patch_init,
+                    orig_patch_mask=orig_patch_mask,
+                )
+
             patched_image = np.clip(
                 patched_image,
                 self.estimator.clip_values[0],