diff --git a/MANIFEST.in b/MANIFEST.in index a897e1795..48a750416 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ recursive-include armory *.json recursive-include armory *.txt recursive-include armory *.ini +recursive-include armory *.png diff --git a/armory/__init__.py b/armory/__init__.py index b8ca631dd..d895da076 100644 --- a/armory/__init__.py +++ b/armory/__init__.py @@ -8,7 +8,7 @@ # Semantic Version -__version__ = "0.13.0" +__version__ = "0.13.1" # Submodule imports diff --git a/armory/art_experimental/attacks/dapricot_patch.py b/armory/art_experimental/attacks/dapricot_patch.py index a7d956e02..127d6f69e 100644 --- a/armory/art_experimental/attacks/dapricot_patch.py +++ b/armory/art_experimental/attacks/dapricot_patch.py @@ -1,110 +1,117 @@ +""" +Copyright 2021 The MITRE Corporation. All rights reserved +""" +import logging import numpy as np import cv2 - -from art.attacks.evasion import RobustDPatch, ProjectedGradientDescent -from armory.art_experimental.utils.dapricot_patch_utils import ( - insert_patch, - shape_coords, - create_mask, -) - - -class DApricotPatch(RobustDPatch): - """ - """ - - def __init__(self, estimator, **kwargs): - super().__init__(estimator=estimator, **kwargs) - - def generate(self, x, y_object=None, y_patch_metadata=None, **generate_kwargs): - - if "threat_model" not in generate_kwargs: - raise ValueError( - "'threat_model' kwarg must be defined in attack config's" - "'generate_kwargs' as one of ('physical', 'digital')" +import math + +from art.attacks.evasion import ProjectedGradientDescent + +logger = logging.getLogger(__name__) + + +def shape_coords(h, w, obj_shape): + if obj_shape == "octagon": + smallest_side = min(h, w) + pi = math.pi + h_init = 0 + w_init = smallest_side / 2 * math.sin(pi / 2) / math.sin(3 * pi / 8) + rads = np.linspace(0, -2 * pi, 8, endpoint=False) + 5 * pi / 8 + h_coords = [h_init * math.cos(r) - w_init * math.sin(r) + h / 2 for r in rads] + w_coords = [h_init * math.sin(r) + w_init * math.cos(r) + w / 2 for r in rads] + coords = [ + (int(round(wc)), int(round(hc))) for hc, wc in zip(h_coords, w_coords) + ] + elif obj_shape == "diamond": + if h <= w: + coords = np.array( + [ + [w // 2, 0], + [min(w - 1, int(round(w / 2 + h / 2))), h // 2], + [w // 2, h - 1], + [max(0, int(round(w / 2 - h / 2))), h // 2], + ] + ) + else: + coords = np.array( + [ + [w // 2, max(0, int(round(h / 2 - w / 2)))], + [w - 1, h // 2], + [w // 2, min(h - 1, int(round(h / 2 + w / 2)))], + [0, h // 2], + ] ) - elif generate_kwargs["threat_model"].lower() not in ("physical", "digital"): - raise ValueError( - f"'threat_model must be set to one of ('physical', 'digital'), not {generate_kwargs['threat_model']}." + elif obj_shape == "rect": + # 0.8 w/h aspect ratio + if h > w: + coords = np.array( + [ + [max(0, int(round(w / 2 - 0.4 * h))), 0], + [min(w - 1, int(round(w / 2 + 0.4 * h))), 0], + [min(w - 1, int(round(w / 2 + 0.4 * h))), h - 1], + [max(0, int(round(w / 2 - 0.4 * h))), h - 1], + ] ) - else: - threat_model = generate_kwargs["threat_model"].lower() + coords = np.array( + [ + [0, max(0, int(round(h / 2 - w / 1.6)))], + [w - 1, max(0, int(round(h / 2 - w / 1.6)))], + [w - 1, min(h - 1, int(round(h / 2 + w / 1.6)))], + [0, min(h - 1, int(round(h / 2 + w / 1.6)))], + ] + ) + else: + raise ValueError('obj_shape can only be {"rect", "diamond", "octagon"}') - num_imgs = x.shape[0] - attacked_images = [] + return np.array(coords) - if threat_model == "digital": - for i in range(num_imgs): - gs_coords = y_patch_metadata[i]["gs_coords"] - patch_x, patch_y = gs_coords[0] # upper left coordinates of patch - self.patch_location = (patch_x, patch_y) - patch_width = np.max(gs_coords[:, 0]) - np.min(gs_coords[:, 0]) - patch_height = np.max(gs_coords[:, 1]) - np.min(gs_coords[:, 1]) - self.patch_shape = (patch_height, patch_width, 3) - - # self._patch needs to be re-initialized with the correct shape - if self.estimator.clip_values is None: - self._patch = np.zeros(shape=self.patch_shape) - else: - self._patch = ( - np.random.randint(0, 255, size=self.patch_shape) - / 255 - * ( - self.estimator.clip_values[1] - - self.estimator.clip_values[0] - ) - + self.estimator.clip_values[0] - ) - - patch = super().generate(np.expand_dims(x[i], axis=0)) - - patch_geometric_shape = ( - y_patch_metadata[i]["shape"].tobytes().decode("utf-8") - ) - img_with_patch = insert_patch( - gs_coords, x[i], patch, patch_geometric_shape, - ) - attacked_images.append(img_with_patch) - else: - # generate patch using center image - gs_coords_center_img = y_patch_metadata[1]["gs_coords"] - patch_x, patch_y = gs_coords_center_img[ - 0 - ] # upper left coordinates of patch - self.patch_location = (patch_x, patch_y) - patch_width = np.max(gs_coords_center_img[:, 0]) - np.min( - gs_coords_center_img[:, 0] - ) - patch_height = np.max(gs_coords_center_img[:, 1]) - np.min( - gs_coords_center_img[:, 1] - ) - self.patch_shape = (patch_height, patch_width, 3) - - # self._patch needs to be re-initialized with the correct shape - if self.estimator.clip_values is None: - self._patch = np.zeros(shape=self.patch_shape) - else: - self._patch = ( - np.random.randint(0, 255, size=self.patch_shape) - / 255 - * (self.estimator.clip_values[1] - self.estimator.clip_values[0]) - + self.estimator.clip_values[0] - ) - patch = super().generate(np.expand_dims(x[1], axis=0)) +def in_polygon(x, y, vertices): + """ + Determine if a point (x,y) is inside a polygon with given vertices - for i in range(num_imgs): - gs_coords = y_patch_metadata[i]["gs_coords"] - patch_geometric_shape = ( - y_patch_metadata[i]["shape"].tobytes().decode("utf-8") - ) + Ref: https://www.eecs.umich.edu/courses/eecs380/HANDOUTS/PROJ2/InsidePoly.html + """ + n_pts = len(vertices) + i = 0 + j = n_pts - 1 + c = False + + while i < n_pts: + if ( + # y coordinate of the point has to be between the y coordinates of the i-th and j-th vertices + ((vertices[i][1] <= y) and (y < vertices[j][1])) + or ((vertices[j][1] <= y) and (y < vertices[i][1])) + ) and ( + # x coordinate of the point is to the left of the line connecting i-th and j-th vertices + x + < (vertices[j][0] - vertices[i][0]) + * (y - vertices[i][1]) + / (vertices[j][1] - vertices[i][1]) + + vertices[i][0] + ): + c = not c + j = i + i = i + 1 + return c + + +def create_mask(mask_type, h, w): + """ + create mask according to shape + """ - img_with_patch = insert_patch( - gs_coords, x[i], patch, patch_geometric_shape, - ) - attacked_images.append(img_with_patch) - return np.array(attacked_images) + mask = np.zeros((h, w, 3)) + coords = shape_coords(h, w, mask_type) + + for i in range(h): + for j in range(w): + if in_polygon(i, j, coords): + mask[i, j, :] = 1 + + return mask class DApricotMaskedPGD(ProjectedGradientDescent): @@ -114,21 +121,7 @@ class DApricotMaskedPGD(ProjectedGradientDescent): def __init__(self, estimator, **kwargs): super().__init__(estimator=estimator, **kwargs) - def generate(self, x, y_object=None, y_patch_metadata=None, **generate_kwargs): - - if "threat_model" not in generate_kwargs: - raise ValueError( - "'threat_model' kwarg must be defined in attack config's" - "'generate_kwargs' as one of ('physical', 'digital')" - ) - elif generate_kwargs["threat_model"].lower() not in ("physical", "digital"): - raise ValueError( - f"'threat_model must be set to one of ('physical', 'digital'), not {generate_kwargs['threat_model']}." - ) - - else: - threat_model = generate_kwargs["threat_model"].lower() - + def generate(self, x, y_object=None, y_patch_metadata=None, threat_model="digital"): num_imgs = x.shape[0] attacked_images = [] @@ -140,7 +133,7 @@ def generate(self, x, y_object=None, y_patch_metadata=None, **generate_kwargs): x[i], y_object[i]["area"], gs_coords, shape ) img_with_patch = super().generate( - np.expand_dims(x[i], axis=0), mask=img_mask + np.expand_dims(x[i], axis=0), y=[y_object[i]], mask=img_mask )[0] attacked_images.append(img_with_patch) else: @@ -153,7 +146,7 @@ def generate(self, x, y_object=None, y_patch_metadata=None, **generate_kwargs): def _compute_image_mask(self, x, gs_area, gs_coords, shape): gs_size = int(np.sqrt(gs_area)) patch_coords = shape_coords(gs_size, gs_size, shape) - h, status = cv2.findHomography(patch_coords, gs_coords) + h, _ = cv2.findHomography(patch_coords, gs_coords) inscribed_patch_mask = create_mask(shape, gs_size, gs_size) img_mask = cv2.warpPerspective( inscribed_patch_mask, h, (x.shape[1], x.shape[0]), cv2.INTER_CUBIC diff --git a/armory/art_experimental/attacks/poison_loader.py b/armory/art_experimental/attacks/poison_loader.py index 85fc27227..05e6e0610 100644 --- a/armory/art_experimental/attacks/poison_loader.py +++ b/armory/art_experimental/attacks/poison_loader.py @@ -1,9 +1,11 @@ """ This module enables loading of different perturbation functions in poisoning """ +import os from art.attacks.poisoning import PoisoningAttackBackdoor from art.attacks.poisoning import perturbations +import armory def poison_loader_GTSRB(**kwargs): @@ -24,13 +26,39 @@ def mod(x): raise ValueError( "poison_type 'image' requires 'backdoor_path' kwarg path to image" ) + backdoor_packaged_with_armory = kwargs.get( + "backdoor_packaged_with_armory", False + ) + if backdoor_packaged_with_armory: + backdoor_path = os.path.join( + # Get base directory where armory is pip installed + os.path.dirname(os.path.dirname(armory.__file__)), + backdoor_path, + ) size = kwargs.get("size") if size is None: raise ValueError("poison_type 'image' requires 'size' kwarg tuple") size = tuple(size) + mode = kwargs.get("mode", "RGB") + blend = kwargs.get("blend", 0.6) + base_img_size_x = kwargs.get("base_img_size_x", 48) + base_img_size_y = kwargs.get("base_img_size_y", 48) + channels_first = kwargs.get("channels_first", False) + x_shift = kwargs.get("x_shift", (base_img_size_x - size[0]) // 2) + y_shift = kwargs.get("y_shift", (base_img_size_y - size[1]) // 2) def mod(x): - return perturbations.insert_image(x, backdoor_path=backdoor_path, size=size) + return perturbations.insert_image( + x, + backdoor_path=backdoor_path, + size=size, + mode=mode, + x_shift=x_shift, + y_shift=y_shift, + channels_first=channels_first, + blend=blend, + random=False, + ) else: raise ValueError(f"Unknown poison_type {poison_type}") diff --git a/armory/art_experimental/defences/jpeg_compression_normalized.py b/armory/art_experimental/defences/jpeg_compression_normalized.py index 6ee90fcf3..146d3cf48 100644 --- a/armory/art_experimental/defences/jpeg_compression_normalized.py +++ b/armory/art_experimental/defences/jpeg_compression_normalized.py @@ -12,7 +12,6 @@ def __init__( self, clip_values, quality=50, - channel_index=3, apply_fit=True, apply_predict=False, means=None, @@ -22,7 +21,6 @@ def __init__( super().__init__( clip_values, quality=quality, - channel_index=channel_index, apply_fit=apply_fit, apply_predict=apply_predict, ) diff --git a/armory/art_experimental/defences/random_affine_pytorch.py b/armory/art_experimental/defences/random_affine_pytorch.py new file mode 100644 index 000000000..ef6c7544e --- /dev/null +++ b/armory/art_experimental/defences/random_affine_pytorch.py @@ -0,0 +1,116 @@ +from typing import Tuple, List, TYPE_CHECKING, Optional + +from art.preprocessing.expectation_over_transformation.pytorch import EoTPyTorch + +if TYPE_CHECKING: + import torch + +import logging + +logger = logging.getLogger(__name__) + + +class EoTRandomAffinePyTorch(EoTPyTorch): + """ + This module implements EoT of image affine transforms, including rotation, translation and scaling. + Code is based on https://pytorch.org/vision/stable/_modules/torchvision/transforms/transforms.html#RandomAffine + """ + + def __init__( + self, + nb_samples: int, + clip_values: Tuple[float, float], + degree: float, + translate: List[float] = [0.0, 0.0], + scale: List[float] = [1.0, 1.0], + apply_fit: bool = False, + apply_predict: bool = True, + ) -> None: + """ + Create an instance of EoTRandomAffinePyTorch. + :param nb_samples: Number of random samples per input sample. + :param clip_values: Tuple of float representing minimum and maximum values of input `(min, max)`. + :param degree: Range of rotation from [-degree, degree] + :param translate: Normalized range of horizontal and vertical translation + :param scale: Scaling range (min, max) + :param apply_fit: True if applied during fitting/training. + :param apply_predict: True if applied during predicting. + """ + super().__init__( + apply_fit=apply_fit, + apply_predict=apply_predict, + nb_samples=nb_samples, + clip_values=clip_values, + ) + + self.degree = degree + self.degree_range = ( + (-degree, degree) if isinstance(degree, (int, float)) else degree + ) + self.translate = translate + self.scale = scale + self._check_params() + + def _transform( + self, x: "torch.Tensor", y: Optional["torch.Tensor"], **kwargs + ) -> Tuple["torch.Tensor", Optional["torch.Tensor"]]: + """ + Apply random affine transforms to an image. + :param x: Input samples. + :param y: Label of the samples `x`. + :return: Transformed samples and labels. + """ + import torch + import torchvision.transforms.functional as F + + img_size = x.shape[:2] + + angle = float( + torch.empty(1) + .uniform_(float(self.degree_range[0]), float(self.degree_range[1])) + .item() + ) + + max_dx = float(self.translate[0] * img_size[1]) + max_dy = float(self.translate[1] * img_size[0]) + tx = int(round(torch.empty(1).uniform_(-max_dx, max_dx).item())) + ty = int(round(torch.empty(1).uniform_(-max_dy, max_dy).item())) + translations = (tx, ty) + + scale = float(torch.empty(1).uniform_(self.scale[0], self.scale[1]).item()) + + # x needs to have channel first + x = x.permute(2, 0, 1) + x = F.affine( + img=x, angle=angle, translate=translations, scale=scale, shear=(0.0, 0.0) + ) + x = x.permute(1, 2, 0) + + return torch.clamp(x, min=self.clip_values[0], max=self.clip_values[1]), y + + def _check_params(self) -> None: + + # pylint: disable=R0916 + if not isinstance(self.degree, (int, float)): + raise ValueError("The argument `degree` has to be a float.") + + if self.degree < 0: + raise ValueError("The argument `degree` must be positive.") + + if not isinstance(self.translate, list): + raise ValueError( + "The argument `translate` has to be a tuple of normalized translation in width and height" + ) + + for t in self.translate: + if not (0.0 <= t <= 1.0): + raise ValueError("translation values should be between 0 and 1") + + if not isinstance(self.scale, list): + raise ValueError( + "The argument `scale` has to be a tuple of minimum and maximum scaling" + ) + + for s in self.scale: + if s <= 0: + raise ValueError("scale values should be positive") diff --git a/armory/art_experimental/utils/dapricot_patch_utils.py b/armory/art_experimental/utils/dapricot_patch_utils.py deleted file mode 100644 index f8d4bd3ce..000000000 --- a/armory/art_experimental/utils/dapricot_patch_utils.py +++ /dev/null @@ -1,207 +0,0 @@ -""" -This file is used to compute the color transform for more realistic -digital patch insertion. - -Contributed by The MITRE Corporation, 2021 -""" - -import cv2 -import math -import numpy as np - - -def shape_coords(h, w, obj_shape): - if obj_shape == "octagon": - smallest_side = min(h, w) - pi = math.pi - h_init = 0 - w_init = smallest_side / 2 * math.sin(pi / 2) / math.sin(3 * pi / 8) - rads = np.linspace(0, -2 * pi, 8, endpoint=False) + 5 * pi / 8 - h_coords = [h_init * math.cos(r) - w_init * math.sin(r) + h / 2 for r in rads] - w_coords = [h_init * math.sin(r) + w_init * math.cos(r) + w / 2 for r in rads] - coords = [ - (int(round(wc)), int(round(hc))) for hc, wc in zip(h_coords, w_coords) - ] - elif obj_shape == "diamond": - if h <= w: - coords = np.array( - [ - [w // 2, 0], - [min(w - 1, int(round(w / 2 + h / 2))), h // 2], - [w // 2, h - 1], - [max(0, int(round(w / 2 - h / 2))), h // 2], - ] - ) - else: - coords = np.array( - [ - [w // 2, max(0, int(round(h / 2 - w / 2)))], - [w - 1, h // 2], - [w // 2, min(h - 1, int(round(h / 2 + w / 2)))], - [0, h // 2], - ] - ) - elif obj_shape == "rect": - # 0.8 w/h aspect ratio - if h > w: - coords = np.array( - [ - [max(0, int(round(w / 2 - 0.4 * h))), 0], - [min(w - 1, int(round(w / 2 + 0.4 * h))), 0], - [min(w - 1, int(round(w / 2 + 0.4 * h))), h - 1], - [max(0, int(round(w / 2 - 0.4 * h))), h - 1], - ] - ) - else: - coords = np.array( - [ - [0, max(0, int(round(h / 2 - w / 1.6)))], - [w - 1, max(0, int(round(h / 2 - w / 1.6)))], - [w - 1, min(h - 1, int(round(h / 2 + w / 1.6)))], - [0, min(h - 1, int(round(h / 2 + w / 1.6)))], - ] - ) - else: - raise ValueError('obj_shape can only be {"rect", "diamond", "octagon"}') - - return np.array(coords) - - -def in_polygon(x, y, vertices): - """ - Determine if a point (x,y) is inside a polygon with given vertices - - Ref: https://www.eecs.umich.edu/courses/eecs380/HANDOUTS/PROJ2/InsidePoly.html - """ - n_pts = len(vertices) - i = 0 - j = n_pts - 1 - c = False - - while i < n_pts: - if ( - # y coordinate of the point has to be between the y coordinates of the i-th and j-th vertices - ((vertices[i][1] <= y) and (y < vertices[j][1])) - or ((vertices[j][1] <= y) and (y < vertices[i][1])) - ) and ( - # x coordinate of the point is to the left of the line connecting i-th and j-th vertices - x - < (vertices[j][0] - vertices[i][0]) - * (y - vertices[i][1]) - / (vertices[j][1] - vertices[i][1]) - + vertices[i][0] - ): - c = not c - j = i - i = i + 1 - return c - - -def create_mask(mask_type, h, w): - """ - create mask according to shape - """ - - mask = np.zeros((h, w, 3)) - coords = shape_coords(h, w, mask_type) - - for i in range(h): - for j in range(w): - if in_polygon(i, j, coords): - mask[i, j, :] = 1 - - return mask - - -def insert_transformed_patch(patch, image, gs_shape, patch_coords=[], image_coords=[]): - """ - Insert patch to image based on given or selected coordinates - - attributes:: - patch - patch as numpy array - image - image as numpy array - gs_shape - green screen shape - patch_coords - patch coordinates to map to image [numpy array] - image_coords - image coordinates patch coordinates will be mapped to [numpy array] - going in clockwise direction, starting with upper left corner - - returns:: - image with patch inserted - """ - - # if no patch coords are given, just use whole image - if patch_coords == []: - patch_coords = shape_coords(patch.shape[0], patch.shape[1], gs_shape) - - # calculate homography - h, status = cv2.findHomography(patch_coords, image_coords) - - # mask to aid with insertion - mask = create_mask(gs_shape, patch.shape[0], patch.shape[1]) - mask_out = cv2.warpPerspective( - mask, h, (image.shape[1], image.shape[0]), cv2.INTER_CUBIC - ) - - # mask patch and warp it to destination coordinates - patch[mask == 0] = 0 - im_out = cv2.warpPerspective( - patch, h, (image.shape[1], image.shape[0]), cv2.INTER_CUBIC - ) - - # save image before adding shadows - im_cp_one = np.copy(image) - im_cp_one[mask_out != 0] = 0 - im_out_cp = np.copy(im_out) - before = im_cp_one.astype("float32") + im_out_cp.astype("float32") - - v_avg = 0.5647 # V value (in HSV) for the green screen, which is #00903a - - # mask image for patch insert - image_cp = np.copy(image) - image_cp[mask_out == 0] = 0 - - # convert to HSV space for shadow estimation - target_hsv = cv2.cvtColor(image_cp, cv2.COLOR_BGR2HSV) - target_hsv = target_hsv.astype("float32") - target_hsv /= 255.0 - - # apply shadows to patch - ratios = target_hsv[:, :, 2] / v_avg - im_out = im_out.astype("float32") - im_out[:, :, 0] = im_out[:, :, 0] * ratios - im_out[:, :, 1] = im_out[:, :, 1] * ratios - im_out[:, :, 2] = im_out[:, :, 2] * ratios - im_out[im_out > 255.0] = 255.0 - - im_cp_two = np.copy(image) - im_cp_two[mask_out != 0] = 0 - final = im_cp_two.astype("float32") + im_out.astype("float32") - - return before, final - - -def insert_patch(gs_coords, gs_im, patch, gs_shape): - """ - :param gs_coords: green screen coordinates in [(x0,y0),(x1,y1),...] format. Type ndarray. - :param gs_im: clean image with green screen. Type ndarray. - :param patch: adversarial patch. Type ndarray - :param gs_shape: green screen shape. Type str. - :param cc_gt: colorchecker ground truth values. Type ndarray. - :param cc_scene: colorchecker values in the scene. Type ndarray. - :param apply_realistic_effects: apply effects such as color correction, blurring, and shadowing. Type bool. - """ - patch = patch.astype("uint8") - - # convert for use with cv2 - patch = cv2.cvtColor(patch, cv2.COLOR_RGB2BGR) - - # insert transformed patch - image_digital, image_physical = insert_transformed_patch( - patch, gs_im, gs_shape, image_coords=gs_coords - ) - return image_digital diff --git a/armory/baseline_models/pytorch/cifar.py b/armory/baseline_models/pytorch/cifar.py index 351680749..84e85144e 100644 --- a/armory/baseline_models/pytorch/cifar.py +++ b/armory/baseline_models/pytorch/cifar.py @@ -59,6 +59,7 @@ def get_art_model( loss=nn.CrossEntropyLoss(), optimizer=torch.optim.Adam(model.parameters(), lr=0.003), input_shape=(32, 32, 3), + channels_first=False, nb_classes=10, clip_values=(0.0, 1.0), **wrapper_kwargs, diff --git a/armory/baseline_models/pytorch/resnet18.py b/armory/baseline_models/pytorch/resnet18.py new file mode 100644 index 000000000..e2e86f203 --- /dev/null +++ b/armory/baseline_models/pytorch/resnet18.py @@ -0,0 +1,61 @@ +""" +ResNet18 CNN model for NxNx3 image classification +""" +import logging +from typing import Optional + +from art.classifiers import PyTorchClassifier +import torch +from torchvision import models + + +logger = logging.getLogger(__name__) + +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +class OuterModel(torch.nn.Module): + def __init__( + self, weights_path: Optional[str], **model_kwargs, + ): + # default to imagenet mean and std + data_means = model_kwargs.pop("data_means", [0.485, 0.456, 0.406]) + data_stds = model_kwargs.pop("data_stds", [0.229, 0.224, 0.225]) + + super().__init__() + self.inner_model = models.resnet18(**model_kwargs) + self.inner_model.to(DEVICE) + + if weights_path: + checkpoint = torch.load(weights_path, map_location=DEVICE) + self.inner_model.load_state_dict(checkpoint) + + self.data_means = torch.tensor(data_means, dtype=torch.float32, device=DEVICE) + self.data_stdev = torch.tensor(data_stds, dtype=torch.float32, device=DEVICE) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x_norm = ((x - self.data_means) / self.data_stdev).permute(0, 3, 1, 2) + output = self.inner_model(x_norm) + + return output + + +# NOTE: PyTorchClassifier expects numpy input, not torch.Tensor input +def get_art_model( + model_kwargs: dict, wrapper_kwargs: dict, weights_path: Optional[str] = None +) -> PyTorchClassifier: + + model = OuterModel(weights_path=weights_path, **model_kwargs) + + wrapped_model = PyTorchClassifier( + model, + loss=torch.nn.CrossEntropyLoss(), + optimizer=torch.optim.SGD(model.parameters(), lr=0.2, momentum=0.9), + input_shape=wrapper_kwargs.pop( + "input_shape", (224, 224, 3) + ), # default to imagenet shape + channels_first=False, + **wrapper_kwargs, + clip_values=(0.0, 1.0), + ) + return wrapped_model diff --git a/armory/baseline_models/pytorch/resnet50.py b/armory/baseline_models/pytorch/resnet50.py index 56326b8d9..b0b475b9f 100644 --- a/armory/baseline_models/pytorch/resnet50.py +++ b/armory/baseline_models/pytorch/resnet50.py @@ -52,6 +52,7 @@ def get_art_model( loss=torch.nn.CrossEntropyLoss(), optimizer=torch.optim.Adam(model.parameters(), lr=0.003), input_shape=(224, 224, 3), + channels_first=False, **wrapper_kwargs, clip_values=(0.0, 1.0), ) diff --git a/armory/baseline_models/pytorch/ucf101_mars.py b/armory/baseline_models/pytorch/ucf101_mars.py index c6ed22a41..63291ae11 100644 --- a/armory/baseline_models/pytorch/ucf101_mars.py +++ b/armory/baseline_models/pytorch/ucf101_mars.py @@ -308,6 +308,7 @@ def get_art_model( loss=torch.nn.CrossEntropyLoss(), optimizer=model.optimizer, input_shape=(None, 240, 320, 3), + channels_first=False, nb_classes=101, clip_values=(0.0, 1.0), **wrapper_kwargs, diff --git a/armory/baseline_models/tf_graph/mscoco_frcnn.py b/armory/baseline_models/tf_graph/mscoco_frcnn.py index af77d3040..dd059de5f 100644 --- a/armory/baseline_models/tf_graph/mscoco_frcnn.py +++ b/armory/baseline_models/tf_graph/mscoco_frcnn.py @@ -43,6 +43,14 @@ def __init__(self, images): def compute_loss(self, x, y): raise NotImplementedError + def loss_gradient(self, x, y, **kwargs): + y_zero_indexed = [] + for y_dict in y: + y_dict_zero_indexed = y_dict.copy() + y_dict_zero_indexed["labels"] = y_dict_zero_indexed["labels"] - 1 + y_zero_indexed.append(y_dict_zero_indexed) + return super().loss_gradient(x, y_zero_indexed, **kwargs) + def predict(self, x, **kwargs): list_of_zero_indexed_pred_dicts = super().predict(x, **kwargs) list_of_one_indexed_pred_dicts = [] @@ -54,6 +62,10 @@ def predict(self, x, **kwargs): def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): - images = tf.placeholder(tf.float32, shape=(1, None, None, 3)) + # APRICOT inputs should have shape (1, None, None, 3) while DAPRICOT inputs have shape + # (3, None, None, 3) + images = tf.placeholder( + tf.float32, shape=(model_kwargs.get("batch_size", 1), None, None, 3) + ) model = TensorFlowFasterRCNNOneIndexed(images) return model diff --git a/armory/data/cached_s3_checksums/cifar100.txt b/armory/data/cached_s3_checksums/cifar100.txt new file mode 100644 index 000000000..4cfd50d78 --- /dev/null +++ b/armory/data/cached_s3_checksums/cifar100.txt @@ -0,0 +1 @@ +armory-public-data cifar100/cifar100_3.0.2.tar.gz 133384060 35d669af5ccb78fb00e9f0d623d61cacabccb3c864872a06d67f2cbf4c902e7b diff --git a/armory/data/cached_s3_checksums/resisc10_poison.txt b/armory/data/cached_s3_checksums/resisc10_poison.txt new file mode 100644 index 000000000..60c671626 --- /dev/null +++ b/armory/data/cached_s3_checksums/resisc10_poison.txt @@ -0,0 +1 @@ +armory-public-data resisc45/resisc10_poison_cached_1.0.0.tar.gz 17243568 728b881be2afa6a03d7580c2acd8015f4f66b81c709f5bcd0c32e2a48c3abae2 diff --git a/armory/data/datasets.py b/armory/data/datasets.py index 8089a6ec9..5eefc0b3f 100644 --- a/armory/data/datasets.py +++ b/armory/data/datasets.py @@ -36,6 +36,7 @@ from armory.data.librispeech import librispeech_dev_clean_split # noqa: F401 from armory.data.librispeech import librispeech_full as lf # noqa: F401 from armory.data.resisc45 import resisc45_split # noqa: F401 +from armory.data.resisc10 import resisc10_poison # noqa: F401 from armory.data.ucf101 import ucf101_clean as uc # noqa: F401 from armory.data.xview import xview as xv # noqa: F401 from armory.data.german_traffic_sign import german_traffic_sign as gtsrb # noqa: F401 @@ -669,8 +670,10 @@ def canonical_variable_image_preprocess(context, batch): mnist_context = ImageContext(x_shape=(28, 28, 1)) cifar10_context = ImageContext(x_shape=(32, 32, 3)) +cifar100_context = ImageContext(x_shape=(32, 32, 3)) gtsrb_context = ImageContext(x_shape=(None, None, 3)) resisc45_context = ImageContext(x_shape=(256, 256, 3)) +resisc10_context = ImageContext(x_shape=(64, 64, 3)) imagenette_context = ImageContext(x_shape=(None, None, 3)) xview_context = ImageContext(x_shape=(None, None, 3)) ucf101_context = VideoContext(x_shape=(None, None, None, 3), frame_rate=25) @@ -684,6 +687,10 @@ def cifar10_canonical_preprocessing(batch): return canonical_image_preprocess(cifar10_context, batch) +def cifar100_canonical_preprocessing(batch): + return canonical_image_preprocess(cifar100_context, batch) + + def gtsrb_canonical_preprocessing(batch): return canonical_variable_image_preprocess(gtsrb_context, batch) @@ -692,6 +699,10 @@ def resisc45_canonical_preprocessing(batch): return canonical_image_preprocess(resisc45_context, batch) +def resisc10_canonical_preprocessing(batch): + return canonical_image_preprocess(resisc10_context, batch) + + def imagenette_canonical_preprocessing(batch): return canonical_variable_image_preprocess(imagenette_context, batch) @@ -834,6 +845,39 @@ def cifar10( ) +def cifar100( + split: str = "train", + epochs: int = 1, + batch_size: int = 1, + dataset_dir: str = None, + preprocessing_fn: Callable = cifar100_canonical_preprocessing, + fit_preprocessing_fn: Callable = None, + cache_dataset: bool = True, + framework: str = "numpy", + shuffle_files: bool = True, + **kwargs, +) -> ArmoryDataGenerator: + """ + One hundred class image dataset: + https://www.cs.toronto.edu/~kriz/cifar.html + """ + preprocessing_fn = preprocessing_chain(preprocessing_fn, fit_preprocessing_fn) + + return _generator_from_tfds( + "cifar100:3.0.2", + split=split, + batch_size=batch_size, + epochs=epochs, + dataset_dir=dataset_dir, + preprocessing_fn=preprocessing_fn, + cache_dataset=cache_dataset, + framework=framework, + shuffle_files=shuffle_files, + context=cifar100_context, + **kwargs, + ) + + def digit( split: str = "train", epochs: int = 1, @@ -1157,6 +1201,47 @@ def resisc45( ) +def resisc10( + split: str = "train", + epochs: int = 1, + batch_size: int = 1, + dataset_dir: str = None, + preprocessing_fn: Callable = resisc10_canonical_preprocessing, + fit_preprocessing_fn: Callable = None, + cache_dataset: bool = True, + framework: str = "numpy", + shuffle_files: bool = True, + **kwargs, +) -> ArmoryDataGenerator: + """ + REmote Sensing Image Scene Classification (RESISC) dataset + http://http://www.escience.cn/people/JunweiHan/NWPU-RESISC45.html + + Contains 7000 images covering 10 scene classes with 700 images per class + + Dimensions of X: (7000, 64, 64, 3) of uint8, ~ 0.8 GB in memory + Each sample is a 64 x 64 3-color (RGB) image + Dimensions of y: (7000,) of int, with values in range(10) + + split - one of ("train", "validation", "test") + """ + preprocessing_fn = preprocessing_chain(preprocessing_fn, fit_preprocessing_fn) + + return _generator_from_tfds( + "resisc10_poison:1.0.0", + split=split, + batch_size=batch_size, + epochs=epochs, + dataset_dir=dataset_dir, + preprocessing_fn=preprocessing_fn, + cache_dataset=cache_dataset, + framework=framework, + shuffle_files=shuffle_files, + context=resisc10_context, + **kwargs, + ) + + def ucf101( split: str = "train", epochs: int = 1, diff --git a/armory/data/resisc10/__init__.py b/armory/data/resisc10/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/armory/data/resisc10/resisc10_poison.py b/armory/data/resisc10/resisc10_poison.py new file mode 100644 index 000000000..87657ab13 --- /dev/null +++ b/armory/data/resisc10/resisc10_poison.py @@ -0,0 +1,89 @@ +"""resisc10_poison dataset.""" + +import os +import tensorflow_datasets as tfds +import tensorflow as tf + + +_DESCRIPTION = """ +Subset of NWPU-RESISC45 image dataset with 10 classes, each class containing 700 images, +each image is 64 pixels by 64 pixels. +Train has 500 images, validation has 100 images, and test has 100 images. +""" + +_CITATION = """ +@article{cheng2017remote, + title={Remote sensing image scene classification: Benchmark and state of the art}, + author={Cheng, Gong and Han, Junwei and Lu, Xiaoqiang}, + journal={Proceedings of the IEEE}, + volume={105}, + number={10}, + pages={1865--1883}, + year={2017}, + publisher={IEEE} +} +""" + +_DL_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/resisc45/resisc_poison_64x64.tar.gz" +_LABELS = [ + "airplane", + "airport", + "harbor", + "industrial_area", + "railway", + "railway_station", + "runway", + "ship", + "storage_tank", + "thermal_power_station", +] + + +class Resisc10Poison(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for resisc10_poison dataset.""" + + VERSION = tfds.core.Version("1.0.0") + RELEASE_NOTES = { + "1.0.0": "Initial release.", + } + + def _info(self) -> tfds.core.DatasetInfo: + """Returns the dataset metadata.""" + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict( + { + "image": tfds.features.Image(shape=(64, 64, 3)), + "label": tfds.features.ClassLabel(names=_LABELS), + } + ), + supervised_keys=("image", "label"), + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + path = dl_manager.download_and_extract(_DL_URL) + splits = [] + for subdir, split in zip( + ["train", "val", "test"], + [tfds.Split.TRAIN, tfds.Split.VALIDATION, tfds.Split.TEST], + ): + splits.append( + tfds.core.SplitGenerator( + name=split, + gen_kwargs={"path": os.path.join(path, "data_64x64", subdir)}, + ) + ) + return splits + + def _generate_examples(self, path): + """Yields examples.""" + for label in _LABELS: + for filename in tf.io.gfile.glob(os.path.join(path, label, "*.jpg")): + example = { + "image": filename, + "label": label, + } + yield filename, example diff --git a/armory/data/url_checksums/resisc10_poison.txt b/armory/data/url_checksums/resisc10_poison.txt new file mode 100644 index 000000000..d572e4ded --- /dev/null +++ b/armory/data/url_checksums/resisc10_poison.txt @@ -0,0 +1 @@ +https://armory-public-data.s3.us-east-2.amazonaws.com/resisc45/resisc_poison_64x64.tar.gz 17180748 33ba9bbf691b129508914ef63e2549c78e6d8c398298218453b2049b96527b37 diff --git a/armory/eval/plot.py b/armory/eval/plot.py index 2af0b357a..0f85662d5 100644 --- a/armory/eval/plot.py +++ b/armory/eval/plot.py @@ -105,7 +105,7 @@ def get_xy( else: y.append(total - failed_benign - (epsilon >= e).sum()) - x.append(max_value) + x.add_results(max_value) if ascending_attack: y.append(failed_attack) else: diff --git a/armory/scenarios/dapricot_scenario.py b/armory/scenarios/dapricot_scenario.py index fe9b1acd7..ea1fc5099 100644 --- a/armory/scenarios/dapricot_scenario.py +++ b/armory/scenarios/dapricot_scenario.py @@ -1,5 +1,5 @@ """ -General image recognition scenario for image classification and object detection. +D-APRICOT scenario for object detection in the presence of targeted adversarial patches. """ import logging @@ -48,25 +48,51 @@ def _evaluate( ) attack_config = config["attack"] attack_type = attack_config.get("type") - if attack_type == "preloaded": + if not attack_config.get("kwargs").get("targeted", False): raise ValueError( - "D-APRICOT scenario should not have preloaded set to True in attack config" + "attack['kwargs']['targeted'] must be set to True for D-APRICOT scenario" + ) + elif attack_type == "preloaded": + raise ValueError( + "attack['type'] should not be set to 'preloaded' for D-APRICOT scenario " + "and does not need to be specified." ) elif "targeted_labels" not in attack_config: raise ValueError( - "Attack config must have 'targeted_labels' key, as the " - "D-APRICOT threat model is targeted." + "attack['targeted_labels'] must be specified, as the D-APRICOT" + " threat model is targeted." ) elif attack_config.get("use_label"): raise ValueError( "The D-APRICOT scenario threat model is targeted, and" - " thus 'use_label' should be set to false." + " thus attack['use_label'] should be set to false or unspecified." + ) + generate_kwargs = attack_config.get("generate_kwargs", {}) + if "threat_model" not in generate_kwargs: + raise ValueError( + "D-APRICOT scenario requires attack['generate_kwargs']['threat_model'] to be set to" + " one of ('physical', 'digital')" + ) + elif generate_kwargs["threat_model"].lower() not in ("physical", "digital"): + raise ValueError( + "D-APRICOT scenario requires attack['generate_kwargs']['threat_model'] to be set to" + f"' one of ('physical', 'digital'), not {generate_kwargs['threat_model']}." ) if config["dataset"].get("batch_size") != 1: - raise ValueError("batch_size of 1 is required for D-APRICOT scenario") + raise ValueError( + "dataset['batch_size'] must be set to 1 for D-APRICOT scenario." + ) model_config = config["model"] + if ( + model_config["model_kwargs"].get("batch_size") != 3 + and generate_kwargs["threat_model"].lower() == "physical" + ): + logger.warning( + "If using Armory's baseline mscoco frcnn model," + " model['model_kwargs']['batch_size'] should be set to 3 for physical attack." + ) estimator, _ = load_model(model_config) defense_config = config.get("defense") or {} @@ -114,6 +140,9 @@ def _evaluate( config["metric"], skip_benign=True, skip_attack=False, targeted=True, ) + # The D-APRICOT scenario has no non-targeted tasks + metrics_logger.adversarial_tasks = [] + eval_split = config["dataset"].get("eval_split", "test") # Evaluate the ART estimator on adversarial test examples diff --git a/armory/scenarios/poisoning_gtsrb_clbd.py b/armory/scenarios/poisoning_gtsrb_clbd.py index 7040d8157..81883c8bc 100644 --- a/armory/scenarios/poisoning_gtsrb_clbd.py +++ b/armory/scenarios/poisoning_gtsrb_clbd.py @@ -11,9 +11,22 @@ from copy import deepcopy import numpy as np -from tensorflow import set_random_seed, ConfigProto, Session -from tensorflow.keras.backend import set_session -from tensorflow.keras.utils import to_categorical + +try: + from tensorflow import set_random_seed, ConfigProto, Session + from tensorflow.keras.backend import set_session + from tensorflow.keras.utils import to_categorical +except ImportError: + from tensorflow.compat.v1 import ( + set_random_seed, + ConfigProto, + Session, + disable_v2_behavior, + ) + from tensorflow.compat.v1.keras.backend import set_session + from tensorflow.compat.v1.keras.utils import to_categorical + + disable_v2_behavior() from tqdm import tqdm from PIL import ImageOps, Image @@ -251,10 +264,10 @@ def _evaluate( # Ensure that input sample isn't overwritten by classifier x.flags.writeable = False y_pred = classifier.predict(x) - benign_validation_metric.append(y, y_pred) + benign_validation_metric.add_results(y, y_pred) y_pred_tgt_class = y_pred[y == src_class] if len(y_pred_tgt_class): - target_class_benign_metric.append( + target_class_benign_metric.add_results( [src_class] * len(y_pred_tgt_class), y_pred_tgt_class ) logger.info( @@ -293,11 +306,11 @@ def _evaluate( poisoned_indices, ) y_pred = classifier.predict(x_test) - poisoned_test_metric.append(y_test, y_pred) + poisoned_test_metric.add_results(y_test, y_pred) y_pred_targeted = y_pred[y_test == src_class] if len(y_pred_targeted): - poisoned_targeted_test_metric.append( + poisoned_targeted_test_metric.add_results( [tgt_class] * len(y_pred_targeted), y_pred_targeted ) results["poisoned_test_accuracy"] = poisoned_test_metric.mean() diff --git a/armory/scenarios/poisoning_gtsrb_scenario.py b/armory/scenarios/poisoning_gtsrb_scenario.py index fc68a4c14..e07ceec11 100644 --- a/armory/scenarios/poisoning_gtsrb_scenario.py +++ b/armory/scenarios/poisoning_gtsrb_scenario.py @@ -10,9 +10,22 @@ import random import numpy as np -from tensorflow import set_random_seed, ConfigProto, Session -from tensorflow.keras.backend import set_session -from tensorflow.keras.utils import to_categorical + +try: + from tensorflow import set_random_seed, ConfigProto, Session + from tensorflow.keras.backend import set_session + from tensorflow.keras.utils import to_categorical +except ImportError: + from tensorflow.compat.v1 import ( + set_random_seed, + ConfigProto, + Session, + disable_v2_behavior, + ) + from tensorflow.compat.v1.keras.backend import set_session + from tensorflow.compat.v1.keras.utils import to_categorical + + disable_v2_behavior() from tqdm import tqdm from PIL import ImageOps, Image @@ -54,19 +67,13 @@ def poison_scenario_preprocessing(batch): def poison_dataset(src_imgs, src_lbls, src, tgt, ds_size, attack, poisoned_indices): # In this example, all images of "src" class have a trigger # added and re-labeled as "tgt" class - # NOTE: currently art.attacks.PoisonAttackBackdoor only supports - # black-white images. One way to generate poisoned examples - # is to convert each batch of multi-channel images of shape - # (N,W,H,C) to N separate (C,W,H)-tuple, where C would be - # interpreted by PoisonAttackBackdoor as the batch size, - # and each channel would have a backdoor trigger added poison_x = [] poison_y = [] for idx in range(ds_size): if src_lbls[idx] == src and idx in poisoned_indices: - src_img = np.transpose(src_imgs[idx], (2, 0, 1)) + src_img = src_imgs[idx] p_img, p_label = attack.poison(src_img, [tgt]) - poison_x.append(np.transpose(p_img, (1, 2, 0))) + poison_x.append(p_img.astype(np.float32)) poison_y.append(p_label) else: poison_x.append(src_imgs[idx]) @@ -303,10 +310,10 @@ def _evaluate( # Ensure that input sample isn't overwritten by classifier x.flags.writeable = False y_pred = classifier.predict(x) - benign_validation_metric.append(y, y_pred) + benign_validation_metric.add_results(y, y_pred) y_pred_tgt_class = y_pred[y == src_class] if len(y_pred_tgt_class): - target_class_benign_metric.append( + target_class_benign_metric.add_results( [src_class] * len(y_pred_tgt_class), y_pred_tgt_class ) logger.info( @@ -337,8 +344,8 @@ def _evaluate( x_poison_test = np.array([xp for xp in x_poison_test], dtype=np.float32) y_pred = classifier.predict(x_poison_test) y_true = [src_class] * len(y_pred) - poisoned_targeted_test_metric.append(y_poison_test, y_pred) - poisoned_test_metric.append(y_true, y_pred) + poisoned_targeted_test_metric.add_results(y_poison_test, y_pred) + poisoned_test_metric.add_results(y_true, y_pred) test_data_clean = load_dataset( config["dataset"], epochs=1, @@ -351,7 +358,7 @@ def _evaluate( ): x_clean_test = np.array([xp for xp in x_clean_test], dtype=np.float32) y_pred = classifier.predict(x_clean_test) - poisoned_test_metric.append(y_clean_test, y_pred) + poisoned_test_metric.add_results(y_clean_test, y_pred) elif poison_dataset_flag: logger.info("Testing on poisoned test data") @@ -375,12 +382,12 @@ def _evaluate( poisoned_indices, ) y_pred = classifier.predict(x_test) - poisoned_test_metric.append(y_test, y_pred) + poisoned_test_metric.add_results(y_test, y_pred) y_pred_targeted = y_pred[y_test == src_class] if not len(y_pred_targeted): continue - poisoned_targeted_test_metric.append( + poisoned_targeted_test_metric.add_results( [tgt_class] * len(y_pred_targeted), y_pred_targeted ) diff --git a/armory/scenarios/poisoning_resisc10_scenario.py b/armory/scenarios/poisoning_resisc10_scenario.py new file mode 100644 index 000000000..376d7645e --- /dev/null +++ b/armory/scenarios/poisoning_resisc10_scenario.py @@ -0,0 +1,373 @@ +""" +Classifier evaluation within ARMORY + +Scenario Contributor: MITRE Corporation +""" + +import logging +from typing import Optional +import os +import random + +import numpy as np +from tqdm import tqdm + +from armory.utils.config_loading import ( + load_dataset, + load_model, + load, + load_fn, + load_defense_internal, +) +from armory.utils import metrics +from armory.scenarios.base import Scenario + +logger = logging.getLogger(__name__) + + +def poison_dataset(src_imgs, src_lbls, src, tgt, ds_size, attack, poisoned_indices): + # In this example, all images of "src" class have a trigger + # added and re-labeled as "tgt" class + poison_x = [] + poison_y = [] + for idx in range(ds_size): + if src_lbls[idx] == src and idx in poisoned_indices: + src_img = src_imgs[idx] + p_img, p_label = attack.poison(src_img, [tgt]) + p_img = p_img.astype(np.float32) + poison_x.append(p_img) + poison_y.append(p_label) + else: + poison_x.append(src_imgs[idx]) + poison_y.append(src_lbls[idx]) + + poison_x, poison_y = np.array(poison_x), np.array(poison_y) + + return poison_x, poison_y + + +class RESISC10(Scenario): + def _evaluate( + self, + config: dict, + num_eval_batches: Optional[int], + skip_benign: Optional[bool], + skip_attack: Optional[bool], + skip_misclassified: Optional[bool], + ) -> dict: + """ + Evaluate a config file for classification robustness against attack. + + Note: num_eval_batches shouldn't be set for poisoning scenario and will raise an + error if it is + """ + if config["sysconfig"].get("use_gpu"): + os.environ["TF_CUDNN_DETERMINISM"] = "1" + if num_eval_batches: + raise ValueError("num_eval_batches shouldn't be set for poisoning scenario") + if skip_benign: + raise ValueError("skip_benign shouldn't be set for poisoning scenario") + if skip_attack: + raise ValueError("skip_attack shouldn't be set for poisoning scenario") + if skip_misclassified: + raise ValueError( + "skip_misclassified shouldn't be set for poisoning scenario" + ) + + model_config = config["model"] + # Scenario assumes canonical preprocessing_fn is used makes images all same size + classifier, _ = load_model(model_config) + + defense_config = config.get("defense") or {} + if "data_augmentation" in defense_config: + for data_aug_config in defense_config["data_augmentation"].values(): + classifier = load_defense_internal(data_aug_config, classifier) + logger.info( + "classifier.preprocessing_defences: {}".format( + classifier.preprocessing_defences + ) + ) + + config_adhoc = config.get("adhoc") or {} + train_epochs = config_adhoc["train_epochs"] + src_class = config_adhoc["source_class"] + tgt_class = config_adhoc["target_class"] + fit_batch_size = config_adhoc.get( + "fit_batch_size", config["dataset"]["batch_size"] + ) + + if not config["sysconfig"].get("use_gpu"): + pass # is this needed for Pytorch? + # conf = ConfigProto(intra_op_parallelism_threads=1) + # set_session(Session(config=conf)) + + # Set random seed due to large variance in attack and defense success + np.random.seed(config_adhoc["split_id"]) + random.seed(config_adhoc["split_id"]) + use_poison_filtering_defense = config_adhoc.get( + "use_poison_filtering_defense", True + ) + if self.check_run: + # filtering defense requires more than a single batch to run properly + use_poison_filtering_defense = False + + logger.info(f"Loading dataset {config['dataset']['name']}...") + + clean_data = load_dataset( + config["dataset"], + epochs=1, + split=config["dataset"].get("train_split", "train"), + shuffle_files=False, + ) + + attack_config = config["attack"] + attack_type = attack_config.get("type") + + fraction_poisoned = config["adhoc"]["fraction_poisoned"] + # Flag for whether to poison dataset -- used to evaluate + # performance of defense on clean data + poison_dataset_flag = config["adhoc"]["poison_dataset"] + # detect_poison does not currently support data generators + # therefore, make in memory dataset + x_train_all, y_train_all = [], [] + + if attack_type == "preloaded": + # Number of datapoints in train split of target clasc + num_images_tgt_class = config_adhoc["num_images_target_class"] + logger.info( + f"Loading poison dataset {config_adhoc['poison_samples']['name']}..." + ) + num_poisoned = int(config_adhoc["fraction_poisoned"] * num_images_tgt_class) + if num_poisoned == 0: + raise ValueError( + "For the preloaded attack, fraction_poisoned must be set so that at least on data point is poisoned." + ) + # Set batch size to number of poisons -- read only one batch of preloaded poisons + config_adhoc["poison_samples"]["batch_size"] = num_poisoned + poison_data = load_dataset( + config["adhoc"]["poison_samples"], + epochs=1, + split="poison", + preprocessing_fn=None, + ) + + logger.info( + "Building in-memory dataset for poisoning detection and training" + ) + for x_clean, y_clean in clean_data: + x_train_all.append(x_clean) + y_train_all.append(y_clean) + x_poison, y_poison = poison_data.get_batch() + x_poison = np.array([xp for xp in x_poison], dtype=np.float32) + x_train_all.append(x_poison) + y_train_all.append(y_poison) + x_train_all = np.concatenate(x_train_all, axis=0) + y_train_all = np.concatenate(y_train_all, axis=0) + else: + attack = load(attack_config) + logger.info( + "Building in-memory dataset for poisoning detection and training" + ) + for x_train, y_train in clean_data: + x_train_all.append(x_train) + y_train_all.append(y_train) + x_train_all = np.concatenate(x_train_all, axis=0) + y_train_all = np.concatenate(y_train_all, axis=0) + if poison_dataset_flag: + total_count = np.bincount(y_train_all)[src_class] + poison_count = int(fraction_poisoned * total_count) + if poison_count == 0: + logger.warning( + f"No poisons generated with fraction_poisoned {fraction_poisoned} for class {src_class}." + ) + src_indices = np.where(y_train_all == src_class)[0] + poisoned_indices = np.sort( + np.random.choice(src_indices, size=poison_count, replace=False) + ) + x_train_all, y_train_all = poison_dataset( + x_train_all, + y_train_all, + src_class, + tgt_class, + y_train_all.shape[0], + attack, + poisoned_indices, + ) + + y_train_all_categorical = y_train_all + + # Flag to determine whether defense_classifier is trained directly + # (default API) or is trained as part of detect_poisons method + fit_defense_classifier_outside_defense = config_adhoc.get( + "fit_defense_classifier_outside_defense", True + ) + # Flag to determine whether defense_classifier uses sparse + # or categorical labels + defense_categorical_labels = config_adhoc.get( + "defense_categorical_labels", True + ) + if use_poison_filtering_defense: + if defense_categorical_labels: + y_train_defense = y_train_all_categorical + else: + y_train_defense = y_train_all + + defense_config = config["defense"] + defense_config.pop("data_augmentation") + detection_kwargs = config_adhoc.get("detection_kwargs", dict()) + + defense_model_config = config_adhoc.get("defense_model", model_config) + defense_train_epochs = config_adhoc.get( + "defense_train_epochs", train_epochs + ) + + # Assumes classifier_for_defense and classifier use same preprocessing function + classifier_for_defense, _ = load_model(defense_model_config) + logger.info( + f"Fitting model {defense_model_config['module']}.{defense_model_config['name']} " + f"for defense {defense_config['name']}..." + ) + if fit_defense_classifier_outside_defense: + classifier_for_defense.fit( + x_train_all, + y_train_defense, + batch_size=fit_batch_size, + nb_epochs=defense_train_epochs, + verbose=False, + shuffle=True, + ) + defense_fn = load_fn(defense_config) + defense = defense_fn(classifier_for_defense, x_train_all, y_train_defense) + + _, is_clean = defense.detect_poison(**detection_kwargs) + is_clean = np.array(is_clean) + logger.info(f"Total clean data points: {np.sum(is_clean)}") + + logger.info("Filtering out detected poisoned samples") + indices_to_keep = is_clean == 1 + x_train_final = x_train_all[indices_to_keep] + y_train_final = y_train_all_categorical[indices_to_keep] + else: + logger.info( + "Defense does not require filtering. Model fitting will use all data." + ) + x_train_final = x_train_all + y_train_final = y_train_all_categorical + if len(x_train_final): + logger.info( + f"Fitting model of {model_config['module']}.{model_config['name']}..." + ) + classifier.fit( + x_train_final, + y_train_final, + batch_size=fit_batch_size, + nb_epochs=train_epochs, + verbose=False, + shuffle=True, + ) + else: + logger.warning("All data points filtered by defense. Skipping training") + + logger.info("Validating on clean validation data") + val_data = load_dataset( + config["dataset"], + epochs=1, + split=config["dataset"].get("eval_split", "validation"), + shuffle_files=False, + ) + benign_validation_metric = metrics.MetricList("categorical_accuracy") + target_class_benign_metric = metrics.MetricList("categorical_accuracy") + for x, y in tqdm(val_data, desc="Testing"): + # Ensure that input sample isn't overwritten by classifier + x.flags.writeable = False + y_pred = classifier.predict(x) + benign_validation_metric.add_results(y, y_pred) + y_pred_tgt_class = y_pred[y == src_class] + if len(y_pred_tgt_class): + target_class_benign_metric.add_results( + [src_class] * len(y_pred_tgt_class), y_pred_tgt_class + ) + logger.info( + f"Unpoisoned validation accuracy: {benign_validation_metric.mean():.2%}" + ) + logger.info( + f"Unpoisoned validation accuracy on targeted class: {target_class_benign_metric.mean():.2%}" + ) + results = { + "benign_validation_accuracy": benign_validation_metric.mean(), + "benign_validation_accuracy_targeted_class": target_class_benign_metric.mean(), + } + + poisoned_test_metric = metrics.MetricList("categorical_accuracy") + poisoned_targeted_test_metric = metrics.MetricList("categorical_accuracy") + + logger.info("Testing on poisoned test data") + if attack_type == "preloaded": + test_data_poison = load_dataset( + config_adhoc["poison_samples"], + epochs=1, + split="poison_test", + preprocessing_fn=None, + ) + for x_poison_test, y_poison_test in tqdm( + test_data_poison, desc="Testing poison" + ): + x_poison_test = np.array([xp for xp in x_poison_test], dtype=np.float32) + y_pred = classifier.predict(x_poison_test) + y_true = [src_class] * len(y_pred) + poisoned_targeted_test_metric.add_results(y_poison_test, y_pred) + poisoned_test_metric.add_results(y_true, y_pred) + test_data_clean = load_dataset( + config["dataset"], + epochs=1, + split=config["dataset"].get("eval_split", "test"), + shuffle_files=False, + ) + for x_clean_test, y_clean_test in tqdm( + test_data_clean, desc="Testing clean" + ): + x_clean_test = np.array([xp for xp in x_clean_test], dtype=np.float32) + y_pred = classifier.predict(x_clean_test) + poisoned_test_metric.add_results(y_clean_test, y_pred) + + elif poison_dataset_flag: + logger.info("Testing on poisoned test data") + test_data = load_dataset( + config["dataset"], + epochs=1, + split=config["dataset"].get("eval_split", "test"), + shuffle_files=False, + ) + for x_test, y_test in tqdm(test_data, desc="Testing"): + src_indices = np.where(y_test == src_class)[0] + poisoned_indices = src_indices # Poison entire class + x_test, _ = poison_dataset( + x_test, + y_test, + src_class, + tgt_class, + len(y_test), + attack, + poisoned_indices, + ) + y_pred = classifier.predict(x_test) + poisoned_test_metric.add_results(y_test, y_pred) + + y_pred_targeted = y_pred[y_test == src_class] + if not len(y_pred_targeted): + continue + poisoned_targeted_test_metric.add_results( + [tgt_class] * len(y_pred_targeted), y_pred_targeted + ) + + if poison_dataset_flag or attack_type == "preloaded": + results["poisoned_test_accuracy"] = poisoned_test_metric.mean() + results[ + "poisoned_targeted_misclassification_accuracy" + ] = poisoned_targeted_test_metric.mean() + logger.info(f"Test accuracy: {poisoned_test_metric.mean():.2%}") + logger.info( + f"Test targeted misclassification accuracy: {poisoned_targeted_test_metric.mean():.2%}" + ) + + return results diff --git a/armory/utils/config_loading.py b/armory/utils/config_loading.py index 78f57f618..a394c5a7c 100644 --- a/armory/utils/config_loading.py +++ b/armory/utils/config_loading.py @@ -13,7 +13,6 @@ import torch # noqa: F401 except ImportError: pass -import art from art.attacks import Attack try: @@ -38,6 +37,8 @@ def load(sub_config): fn = getattr(module, sub_config["name"]) args = sub_config.get("args", []) kwargs = sub_config.get("kwargs", {}) + if "clip_values" in kwargs: + kwargs["clip_values"] = tuple(kwargs["clip_values"]) return fn(*args, **kwargs) @@ -199,18 +200,21 @@ def load_defense_internal(defense_config, classifier): defense_type = defense_config["type"] if defense_type == "Preprocessor": _check_defense_api(defense, Preprocessor) - if classifier.preprocessing_defences: - classifier.preprocessing_defences.append(defense) + preprocessing_defences = classifier.get_params().get("preprocessing_defences") + if preprocessing_defences: + preprocessing_defences.append(defense) else: - classifier.preprocessing_defences = [defense] - if art.__version__ >= "1.5": - classifier._update_preprocessing_operations() + preprocessing_defences = [defense] + classifier.set_params(preprocessing_defences=preprocessing_defences) + elif defense_type == "Postprocessor": _check_defense_api(defense, Postprocessor) - if classifier.postprocessing_defences: - classifier.postprocessing_defences.append(defense) + postprocessing_defences = classifier.get_params().get("postprocessing_defences") + if postprocessing_defences: + postprocessing_defences.append(defense) else: - classifier.postprocessing_defences = [defense] + postprocessing_defences = [defense] + classifier.set_params(postprocessing_defences=postprocessing_defences) else: raise ValueError( f"Internal defenses must be of either type [Preprocessor, Postprocessor], found {defense_type}" @@ -220,34 +224,51 @@ def load_defense_internal(defense_config, classifier): def load_label_targeter(config): - scheme = config["scheme"].lower() - if scheme == "fixed": - value = config.get("value") - return labels.FixedLabelTargeter(value) - elif scheme == "string": - value = config.get("value") - return labels.FixedStringTargeter(value) - elif scheme == "random": - num_classes = config.get("num_classes") - return labels.RandomLabelTargeter(num_classes) - elif scheme == "round-robin": - num_classes = config.get("num_classes") - offset = config.get("offset", 1) - return labels.RoundRobinTargeter(num_classes, offset) - elif scheme == "manual": - values = config.get("values") - repeat = config.get("repeat", False) - return labels.ManualTargeter(values, repeat) - elif scheme == "identity": - return labels.IdentityTargeter() - elif scheme == "matched length": - transcripts = config.get("transcripts") - return labels.MatchedTranscriptLengthTargeter(transcripts) - elif scheme == "object_detection_fixed": - value = config.get("value") - score = config.get("score", 1.0) - return labels.ObjectDetectionFixedLabelTargeteer(value, score) - else: - raise ValueError( - f'scheme {scheme} not in ("fixed", "random", "round-robin", "manual", "identity", "matched length")' + if config.get("scheme"): + logger.warning( + "The use of a 'scheme' key in attack['targeted_labels'] has been deprecated. " + "The supported means of configuring label targeters is to include 'module' " + "and 'name' keys in attack['targeted_labels'] pointing to the targeter object. " + ) + scheme = config["scheme"].lower() + if scheme == "fixed": + value = config.get("value") + return labels.FixedLabelTargeter(value=value) + elif scheme == "string": + value = config.get("value") + return labels.FixedStringTargeter(value=value) + elif scheme == "random": + num_classes = config.get("num_classes") + return labels.RandomLabelTargeter(num_classes=num_classes) + elif scheme == "round-robin": + num_classes = config.get("num_classes") + offset = config.get("offset", 1) + return labels.RoundRobinTargeter(num_classes=num_classes, offset=offset) + elif scheme == "manual": + values = config.get("values") + repeat = config.get("repeat", False) + return labels.ManualTargeter(values=values, repeat=repeat) + elif scheme == "identity": + return labels.IdentityTargeter() + elif scheme == "matched length": + transcripts = config.get("transcripts") + return labels.MatchedTranscriptLengthTargeter(transcripts=transcripts) + elif scheme == "object_detection_fixed": + value = config.get("value") + score = config.get("score", 1.0) + return labels.ObjectDetectionFixedLabelTargeter(value=value, score=score) + else: + raise ValueError( + f'scheme {scheme} not in ("fixed", "random", "round-robin", "manual", "identity", ' + f'"matched length", "object_detection_fixed")' + ) + label_targeter_module = import_module(config["module"]) + label_targeter_class = getattr(label_targeter_module, config["name"]) + label_targeter_kwargs = config["kwargs"] + label_targeter = label_targeter_class(**label_targeter_kwargs) + if not callable(getattr(label_targeter, "generate", None)): + raise AttributeError( + f"label_targeter {label_targeter} must have a 'generate()' method" + f" which returns target labels." ) + return label_targeter diff --git a/armory/utils/config_schema.json b/armory/utils/config_schema.json index e0eafe30f..53dec217e 100644 --- a/armory/utils/config_schema.json +++ b/armory/utils/config_schema.json @@ -210,6 +210,7 @@ }, "supported_metric_enum": { "enum": [ + "dapricot_patch_target_success", "dapricot_patch_targeted_AP_per_class", "apricot_patch_targeted_AP_per_class", "categorical_accuracy", @@ -239,9 +240,7 @@ "mean_image_circle_patch_diameter", "max_image_circle_patch_diameter", "word_error_rate", - "object_detection_AP_per_class", - "object_detection_class_recall", - "object_detection_class_precision" + "object_detection_AP_per_class" ] }, "sysconfig": { diff --git a/armory/utils/external_repo.py b/armory/utils/external_repo.py index 6b371b3d9..823c94fd8 100644 --- a/armory/utils/external_repo.py +++ b/armory/utils/external_repo.py @@ -73,7 +73,7 @@ def download_and_extract_repo( org_repo_name, branch = external_repo_name.split("@") else: org_repo_name = external_repo_name - branch = "master" + branch = "" repo_name = org_repo_name.split("/")[-1] if "ARMORY_GITHUB_TOKEN" in os.environ and os.getenv("ARMORY_GITHUB_TOKEN") != "": diff --git a/armory/utils/labels.py b/armory/utils/labels.py index e829147bc..e082ee572 100644 --- a/armory/utils/labels.py +++ b/armory/utils/labels.py @@ -9,7 +9,7 @@ class FixedLabelTargeter: - def __init__(self, value): + def __init__(self, *, value): if not isinstance(value, int) or value < 0: raise ValueError(f"value {value} must be a nonnegative int") self.value = value @@ -19,7 +19,7 @@ def generate(self, y): class FixedStringTargeter: - def __init__(self, value): + def __init__(self, *, value): if not isinstance(value, str): raise ValueError(f"target value {value} is not a string") self.value = value @@ -29,7 +29,7 @@ def generate(self, y): class RandomLabelTargeter: - def __init__(self, num_classes): + def __init__(self, *, num_classes): if not isinstance(num_classes, int) or num_classes < 2: raise ValueError(f"num_classes {num_classes} must be an int >= 2") self.num_classes = num_classes @@ -40,7 +40,7 @@ def generate(self, y): class RoundRobinTargeter: - def __init__(self, num_classes, offset=1): + def __init__(self, *, num_classes, offset=1): if not isinstance(num_classes, int) or num_classes < 1: raise ValueError(f"num_classes {num_classes} must be a positive int") if not isinstance(offset, int) or offset % num_classes == 0: @@ -54,7 +54,7 @@ def generate(self, y): class ManualTargeter: - def __init__(self, values, repeat=False): + def __init__(self, *, values, repeat=False): if not values: raise ValueError('"values" cannot be an empty list') self.values = values @@ -84,13 +84,13 @@ def generate(self, y): return y.copy().astype(int) -class ObjectDetectionFixedLabelTargeteer: +class ObjectDetectionFixedLabelTargeter: """ Replaces the ground truth labels with the specified value. Does not modify the number of boxes or location of boxes. """ - def __init__(self, value, score=1.0): + def __init__(self, *, value, score=1.0): if not isinstance(value, int) or value < 0: raise ValueError(f"value {value} must be a nonnegative int") self.value = value @@ -118,7 +118,7 @@ class MatchedTranscriptLengthTargeter: If two labels are tied in length, then it pseudorandomly picks one. """ - def __init__(self, transcripts): + def __init__(self, *, transcripts): if not transcripts: raise ValueError('"transcripts" cannot be None or an empty list') for t in transcripts: diff --git a/armory/utils/metrics.py b/armory/utils/metrics.py index 2c1c3e6e6..1b8d913a2 100644 --- a/armory/utils/metrics.py +++ b/armory/utils/metrics.py @@ -367,86 +367,43 @@ def image_circle_patch_diameter(x, x_adv): ] -def object_detection_class_precision(y, y_pred, score_threshold=0.5): - _check_object_detection_input(y, y_pred) - num_tps, num_fps, num_fns = _object_detection_get_tp_fp_fn( - y, y_pred[0], score_threshold=score_threshold - ) - if num_tps + num_fps > 0: - return [num_tps / (num_tps + num_fps)] - else: - return [0] - - -def object_detection_class_recall(y, y_pred, score_threshold=0.5): - _check_object_detection_input(y, y_pred) - num_tps, num_fps, num_fns = _object_detection_get_tp_fp_fn( - y, y_pred[0], score_threshold=score_threshold - ) - if num_tps + num_fns > 0: - return [num_tps / (num_tps + num_fns)] - else: - return [0] - - -def _object_detection_get_tp_fp_fn(y, y_pred, score_threshold=0.5): - """ - Helper function to compute the number of true positives, false positives, and false - negatives given a set of of object detection labels and predictions - """ - ground_truth_set_of_classes = set( - y["labels"][np.where(y["labels"] != ADV_PATCH_MAGIC_NUMBER_LABEL_ID)] - .flatten() - .tolist() - ) - predicted_set_of_classes = set( - y_pred["labels"][np.where(y_pred["scores"] > score_threshold)].tolist() - ) - - num_true_positives = len( - predicted_set_of_classes.intersection(ground_truth_set_of_classes) - ) - num_false_positives = len( - [c for c in predicted_set_of_classes if c not in ground_truth_set_of_classes] - ) - num_false_negatives = len( - [c for c in ground_truth_set_of_classes if c not in predicted_set_of_classes] - ) - - return num_true_positives, num_false_positives, num_false_negatives - - -def _check_object_detection_input(y, y_pred): +def _check_object_detection_input(y_list, y_pred_list): """ Helper function to check that the object detection labels and predictions are in - the expected format and contain the expected fields + the expected format and contain the expected fields. + + y_list (list): of length equal to the number of input examples. Each element in the list + should be a dict with "labels" and "boxes" keys mapping to a numpy array of + shape (N,) and (N, 4) respectively where N = number of boxes. + y_pred_list (list): of length equal to the number of input examples. Each element in the + list should be a dict with "labels", "boxes", and "scores" keys mapping to a numpy + array of shape (N,), (N, 4), and (N,) respectively where N = number of boxes. """ - if not isinstance(y, dict): - raise TypeError("Expected y to be a dictionary") + if not isinstance(y_pred_list, list): + raise TypeError("Expected y_pred_list to be a list") - if not isinstance(y_pred, list): - raise TypeError("Expected y_pred to be a list") + if not isinstance(y_list, list): + raise TypeError("Expected y_list to be a list") - # Current object detection pipeline only supports batch_size of 1 - if len(y_pred) != 1: + if len(y_list) != len(y_pred_list): raise ValueError( - f"Expected y_pred to be a list of length 1, found length of {len(y_pred)}" + f"Received {len(y_list)} labels but {len(y_pred_list)} predictions" ) - - y_pred = y_pred[0] + elif len(y_list) == 0: + raise ValueError("Received no labels or predictions") REQUIRED_LABEL_KEYS = ["labels", "boxes"] REQUIRED_PRED_KEYS = REQUIRED_LABEL_KEYS + ["scores"] - if not all(key in y for key in REQUIRED_LABEL_KEYS): - raise ValueError( - f"y must contain the following keys: {REQUIRED_LABEL_KEYS}. The following keys were found: {y.keys()}" - ) - - if not all(key in y_pred for key in REQUIRED_PRED_KEYS): - raise ValueError( - f"y_pred must contain the following keys: {REQUIRED_PRED_KEYS}. The following keys were found: {y_pred.keys()}" - ) + for (y, y_pred) in zip(y_list, y_pred_list): + if not all(key in y for key in REQUIRED_LABEL_KEYS): + raise ValueError( + f"y must contain the following keys: {REQUIRED_LABEL_KEYS}. The following keys were found: {y.keys()}" + ) + elif not all(key in y_pred for key in REQUIRED_PRED_KEYS): + raise ValueError( + f"y_pred must contain the following keys: {REQUIRED_PRED_KEYS}. The following keys were found: {y_pred.keys()}" + ) def _intersection_over_union(box_1, box_2): @@ -484,16 +441,22 @@ def _intersection_over_union(box_1, box_2): return iou -def object_detection_AP_per_class(list_of_ys, list_of_y_preds): +def object_detection_AP_per_class(y_list, y_pred_list, iou_threshold=0.5): """ Mean average precision for object detection. This function returns a dictionary mapping each class to the average precision (AP) for the class. The mAP can be computed - by taking the mean of the AP's across all classes. - - This metric is computed over all evaluation samples, rather than on a per-sample basis. + by taking the mean of the AP's across all classes. This metric is computed over all + evaluation samples, rather than on a per-sample basis. + + y_list (list): of length equal to the number of input examples. Each element in the list + should be a dict with "labels" and "boxes" keys mapping to a numpy array of + shape (N,) and (N, 4) respectively where N = number of boxes. + y_pred_list (list): of length equal to the number of input examples. Each element in the + list should be a dict with "labels", "boxes", and "scores" keys mapping to a numpy + array of shape (N,), (N, 4), and (N,) respectively where N = number of boxes. """ + _check_object_detection_input(y_list, y_pred_list) - IOU_THRESHOLD = 0.5 # Precision will be computed at recall points of 0, 0.1, 0.2, ..., 1 RECALL_POINTS = np.linspace(0, 1, 11) @@ -502,30 +465,26 @@ def object_detection_AP_per_class(list_of_ys, list_of_y_preds): # has the following keys "img_idx", "label", "box", as well as "score" for predicted boxes pred_boxes_list = [] gt_boxes_list = [] - # Each element in list_of_y_preds is a list with length equal to batch size - batch_size = len(list_of_y_preds[0]) - for batch_idx, (y, y_pred) in enumerate(zip(list_of_ys, list_of_y_preds)): - for img_idx in range(len(y_pred)): - global_img_idx = (batch_size * batch_idx) + img_idx - img_labels = y[img_idx]["labels"].flatten() - img_boxes = y[img_idx]["boxes"].reshape((-1, 4)) - for gt_box_idx in range(img_labels.flatten().shape[0]): - label = img_labels[gt_box_idx] - box = img_boxes[gt_box_idx] - gt_box_dict = {"img_idx": global_img_idx, "label": label, "box": box} - gt_boxes_list.append(gt_box_dict) - - for pred_box_idx in range(y_pred[img_idx]["labels"].flatten().shape[0]): - pred_label = y_pred[img_idx]["labels"][pred_box_idx] - pred_box = y_pred[img_idx]["boxes"][pred_box_idx] - pred_score = y_pred[img_idx]["scores"][pred_box_idx] - pred_box_dict = { - "img_idx": global_img_idx, - "label": pred_label, - "box": pred_box, - "score": pred_score, - } - pred_boxes_list.append(pred_box_dict) + for img_idx, (y, y_pred) in enumerate(zip(y_list, y_pred_list)): + img_labels = y["labels"].flatten() + img_boxes = y["boxes"].reshape((-1, 4)) + for gt_box_idx in range(img_labels.flatten().shape[0]): + label = img_labels[gt_box_idx] + box = img_boxes[gt_box_idx] + gt_box_dict = {"img_idx": img_idx, "label": label, "box": box} + gt_boxes_list.append(gt_box_dict) + + for pred_box_idx in range(y_pred["labels"].flatten().shape[0]): + pred_label = y_pred["labels"][pred_box_idx] + pred_box = y_pred["boxes"][pred_box_idx] + pred_score = y_pred["scores"][pred_box_idx] + pred_box_dict = { + "img_idx": img_idx, + "label": pred_label, + "box": pred_box, + "score": pred_score, + } + pred_boxes_list.append(pred_box_dict) # Union of (1) the set of all true classes and (2) the set of all predicted classes set_of_class_ids = set([i["label"] for i in gt_boxes_list]) | set( @@ -596,7 +555,7 @@ def object_detection_AP_per_class(list_of_ys, list_of_y_preds): highest_iou = iou highest_iou_gt_idx = gt_idx - if highest_iou > IOU_THRESHOLD: + if highest_iou > iou_threshold: # If the gt box has not yet been covered if ( img_idx_to_gtboxismatched_array[pred_box["img_idx"]][ @@ -654,7 +613,7 @@ def object_detection_AP_per_class(list_of_ys, list_of_y_preds): return average_precisions_by_class -def apricot_patch_targeted_AP_per_class(list_of_ys, list_of_y_preds): +def apricot_patch_targeted_AP_per_class(y_list, y_pred_list, iou_threshold=0.1): """ Average precision indicating how successfully the APRICOT patch causes the detector to predict the targeted class of the patch at the location of the patch. A higher @@ -671,11 +630,18 @@ class (at a location overlapping the patch). A false positive is the case where It returns a dictionary mapping each class to the average precision (AP) for the class. The only classes with potentially nonzero AP's are the classes targeted by the patches (see above paragraph). - """ - # From https://arxiv.org/abs/1912.08166: use a low IOU since "the patches will sometimes - # generate many small, overlapping predictions in the region of the attack" - IOU_THRESHOLD = 0.1 + From https://arxiv.org/abs/1912.08166: use a low IOU since "the patches will sometimes + generate many small, overlapping predictions in the region of the attack" + + y_list (list): of length equal to the number of input examples. Each element in the list + should be a dict with "labels" and "boxes" keys mapping to a numpy array of + shape (N,) and (N, 4) respectively where N = number of boxes. + y_pred_list (list): of length equal to the number of input examples. Each element in the + list should be a dict with "labels", "boxes", and "scores" keys mapping to a numpy + array of shape (N,), (N, 4), and (N,) respectively where N = number of boxes. + """ + _check_object_detection_input(y_list, y_pred_list) # Precision will be computed at recall points of 0, 0.1, 0.2, ..., 1 RECALL_POINTS = np.linspace(0, 1, 11) @@ -685,36 +651,32 @@ class (at a location overlapping the patch). A false positive is the case where # has the following keys "img_idx", "label", "box", as well as "score" for predicted boxes patch_boxes_list = [] overlappping_pred_boxes_list = [] - # Each element in list_of_y_preds is a list with length equal to batch size - batch_size = len(list_of_y_preds[0]) - for batch_idx, (y, y_pred) in enumerate(zip(list_of_ys, list_of_y_preds)): - for img_idx in range(len(y_pred)): - global_img_idx = (batch_size * batch_idx) + img_idx - idx_of_patch = np.where( - y[img_idx]["labels"].flatten() == ADV_PATCH_MAGIC_NUMBER_LABEL_ID - )[0] - patch_box = y[img_idx]["boxes"].reshape((-1, 4))[idx_of_patch].flatten() - patch_id = int(y[img_idx]["patch_id"].flatten()[idx_of_patch]) - patch_target_label = APRICOT_PATCHES[patch_id]["adv_target"] - patch_box_dict = { - "img_idx": global_img_idx, - "label": patch_target_label, - "box": patch_box, - } - patch_boxes_list.append(patch_box_dict) - - for pred_box_idx in range(y_pred[img_idx]["labels"].size): - box = y_pred[img_idx]["boxes"][pred_box_idx] - if _intersection_over_union(box, patch_box) > IOU_THRESHOLD: - label = y_pred[img_idx]["labels"][pred_box_idx] - score = y_pred[img_idx]["scores"][pred_box_idx] - pred_box_dict = { - "img_idx": global_img_idx, - "label": label, - "box": box, - "score": score, - } - overlappping_pred_boxes_list.append(pred_box_dict) + for img_idx, (y, y_pred) in enumerate(zip(y_list, y_pred_list)): + idx_of_patch = np.where( + y["labels"].flatten() == ADV_PATCH_MAGIC_NUMBER_LABEL_ID + )[0] + patch_box = y["boxes"].reshape((-1, 4))[idx_of_patch].flatten() + patch_id = int(y["patch_id"].flatten()[idx_of_patch]) + patch_target_label = APRICOT_PATCHES[patch_id]["adv_target"] + patch_box_dict = { + "img_idx": img_idx, + "label": patch_target_label, + "box": patch_box, + } + patch_boxes_list.append(patch_box_dict) + + for pred_box_idx in range(y_pred["labels"].size): + box = y_pred["boxes"][pred_box_idx] + if _intersection_over_union(box, patch_box) > iou_threshold: + label = y_pred["labels"][pred_box_idx] + score = y_pred["scores"][pred_box_idx] + pred_box_dict = { + "img_idx": img_idx, + "label": label, + "box": box, + "score": score, + } + overlappping_pred_boxes_list.append(pred_box_dict) # Union of (1) the set of classes targeted by patches and (2) the set of all classes # predicted at a location that overlaps the patch in the image @@ -835,7 +797,7 @@ class (at a location overlapping the patch). A false positive is the case where return average_precisions_by_class -def dapricot_patch_targeted_AP_per_class(list_of_ys, list_of_y_preds): +def dapricot_patch_targeted_AP_per_class(y_list, y_pred_list, iou_threshold=0.1): """ Average precision indicating how successfully the patch causes the detector to predict the targeted class of the patch at the location of the patch. A higher @@ -854,13 +816,21 @@ class (at a location overlapping the patch). A false positive is the case where (see above paragraph). Assumptions made for D-APRICOT dataset: each image has one ground truth box. This box corresponds - to the patch and is assigned a label of whatever the attack's target label is. There are no ground-truth - boxes of COCO objects. - """ + to the patch and is assigned a label of whatever the attack's target label is. There are no + ground-truth boxes of COCO objects. + + From https://arxiv.org/abs/1912.08166: use a low IOU since "the patches will sometimes + generate many small, overlapping predictions in the region of the attack" + + y_list (list): of length equal to the number of input examples. Each element in the list + should be a dict with "labels" and "boxes" keys mapping to a numpy array of + shape (N,) and (N, 4) respectively where N = number of boxes. + y_pred_list (list): of length equal to the number of input examples. Each element in the + list should be a dict with "labels", "boxes", and "scores" keys mapping to a numpy + array of shape (N,), (N, 4), and (N,) respectively where N = number of boxes. - # From https://arxiv.org/abs/1912.08166: use a low IOU since "the patches will sometimes - # generate many small, overlapping predictions in the region of the attack" - IOU_THRESHOLD = 0.1 + """ + _check_object_detection_input(y_list, y_pred_list) # Precision will be computed at recall points of 0, 0.1, 0.2, ..., 1 RECALL_POINTS = np.linspace(0, 1, 11) @@ -870,32 +840,28 @@ class (at a location overlapping the patch). A false positive is the case where # has the following keys "img_idx", "label", "box", as well as "score" for predicted boxes patch_boxes_list = [] overlappping_pred_boxes_list = [] - # Each element in list_of_y_preds is a list with length equal to batch size - batch_size = len(list_of_y_preds[0]) - for batch_idx, (y, y_pred) in enumerate(zip(list_of_ys, list_of_y_preds)): - for img_idx in range(len(y_pred)): - global_img_idx = (batch_size * batch_idx) + img_idx - patch_box = y[img_idx]["boxes"].flatten() - patch_target_label = int(y[img_idx]["labels"]) - patch_box_dict = { - "img_idx": global_img_idx, - "label": patch_target_label, - "box": patch_box, - } - patch_boxes_list.append(patch_box_dict) - - for pred_box_idx in range(y_pred[img_idx]["labels"].size): - box = y_pred[img_idx]["boxes"][pred_box_idx] - if _intersection_over_union(box, patch_box) > IOU_THRESHOLD: - label = y_pred[img_idx]["labels"][pred_box_idx] - score = y_pred[img_idx]["scores"][pred_box_idx] - pred_box_dict = { - "img_idx": global_img_idx, - "label": label, - "box": box, - "score": score, - } - overlappping_pred_boxes_list.append(pred_box_dict) + for img_idx, (y, y_pred) in enumerate(zip(y_list, y_pred_list)): + patch_box = y["boxes"].flatten() + patch_target_label = int(y["labels"]) + patch_box_dict = { + "img_idx": img_idx, + "label": patch_target_label, + "box": patch_box, + } + patch_boxes_list.append(patch_box_dict) + + for pred_box_idx in range(y_pred["labels"].size): + box = y_pred["boxes"][pred_box_idx] + if _intersection_over_union(box, patch_box) > iou_threshold: + label = y_pred["labels"][pred_box_idx] + score = y_pred["scores"][pred_box_idx] + pred_box_dict = { + "img_idx": img_idx, + "label": label, + "box": box, + "score": score, + } + overlappping_pred_boxes_list.append(pred_box_dict) # Only compute AP of classes targeted by patches. The D-APRICOT dataset in some # cases contains unlabeled COCO objects in the background @@ -1014,7 +980,53 @@ class (at a location overlapping the patch). A false positive is the case where return average_precisions_by_class +def dapricot_patch_target_success( + y_list, y_pred_list, iou_threshold=0.1, conf_threshold=0.5 +): + """ + Binary metric that simply indicates whether or not the model predicted the targeted + class at the location of the patch (given an IOU threshold which defaults to 0.1) with + confidence >= a confidence threshold which defaults to 0.5. + + Assumptions made for D-APRICOT dataset: each image has one ground truth box. This box + corresponds to the patch and is assigned a label of whatever the attack's target label is. + There are no ground-truth boxes of COCO objects. + + Note: from https://arxiv.org/abs/1912.08166: by default a low IOU threshold is used since + "the patches will sometimes generate many small, overlapping predictions in the region + of the attack" + + y_list (list): of length equal to the number of input examples. Each element in the list + should be a dict with "labels" and "boxes" keys mapping to a numpy array of + shape (N,) and (N, 4) respectively where N = number of boxes. + y_pred_list (list): of length equal to the number of input examples. Each element in the + list should be a dict with "labels", "boxes", and "scores" keys mapping to a numpy + array of shape (N,), (N, 4), and (N,) respectively where N = number of boxes. + """ + return [ + _dapricot_patch_target_success( + y, y_pred, iou_threshold=iou_threshold, conf_threshold=conf_threshold + ) + for y, y_pred in zip(y_list, y_pred_list) + ] + + +def _dapricot_patch_target_success(y, y_pred, iou_threshold=0.1, conf_threshold=0.5): + target_label = int(y["labels"]) + target_box = y["boxes"].reshape((4,)) + pred_indices = np.where(y_pred["scores"] > conf_threshold)[0] + for pred_idx in pred_indices: + if y_pred["labels"][pred_idx] == target_label: + if ( + _intersection_over_union(y_pred["boxes"][pred_idx], target_box) + > iou_threshold + ): + return 1 + return 0 + + SUPPORTED_METRICS = { + "dapricot_patch_target_success": dapricot_patch_target_success, "dapricot_patch_targeted_AP_per_class": dapricot_patch_targeted_AP_per_class, "apricot_patch_targeted_AP_per_class": apricot_patch_targeted_AP_per_class, "categorical_accuracy": categorical_accuracy, @@ -1035,8 +1047,6 @@ class (at a location overlapping the patch). A false positive is the case where "mars_mean_patch": mars_mean_patch, "word_error_rate": word_error_rate, "object_detection_AP_per_class": object_detection_AP_per_class, - "object_detection_class_precision": object_detection_class_precision, - "object_detection_class_recall": object_detection_class_recall, } # Image-based metrics applied to video @@ -1089,12 +1099,13 @@ def __init__(self, name, function=None): raise ValueError(f"function must be callable or None, not {function}") self.name = name self._values = [] - self._inputs = [] + self._input_labels = [] + self._input_preds = [] def clear(self): self._values.clear() - def append(self, *args, **kwargs): + def add_results(self, *args, **kwargs): value = self.function(*args, **kwargs) self._values.extend(value) @@ -1110,8 +1121,11 @@ def values(self): def mean(self): return sum(float(x) for x in self._values) / len(self._values) - def append_inputs(self, *args): - self._inputs.append(args) + def append_input_label(self, label): + self._input_labels.extend(label) + + def append_input_pred(self, pred): + self._input_preds.extend(pred) def total_wer(self): # checks if all values are tuples from the WER metric @@ -1125,23 +1139,8 @@ def total_wer(self): else: raise ValueError("total_wer() only for WER metric") - def AP_per_class(self): - # Computed at once across all samples - y_s = [i[0] for i in self._inputs] - y_preds = [i[1] for i in self._inputs] - return object_detection_AP_per_class(y_s, y_preds) - - def apricot_patch_targeted_AP_per_class(self): - # Computed at once across all samples - y_s = [i[0] for i in self._inputs] - y_preds = [i[1] for i in self._inputs] - return apricot_patch_targeted_AP_per_class(y_s, y_preds) - - def dapricot_patch_targeted_AP_per_class(self): - # Computed at once across all samples - y_s = [i[0] for i in self._inputs] - y_preds = [i[1] for i in self._inputs] - return dapricot_patch_targeted_AP_per_class(y_s, y_preds) + def compute_non_elementwise_metric(self): + return self.function(self._input_labels, self._input_preds) class MetricsLogger: @@ -1194,6 +1193,17 @@ def __init__( "No metric results will be produced. " "To change this, set one or more 'task' or 'perturbation' metrics" ) + # the following metrics must be computed at once after all predictions have been obtained + self.non_elementwise_metrics = [ + "object_detection_AP_per_class", + "apricot_patch_targeted_AP_per_class", + "dapricot_patch_targeted_AP_per_class", + ] + self.mean_ap_metrics = [ + "object_detection_AP_per_class", + "apricot_patch_targeted_AP_per_class", + "dapricot_patch_targeted_AP_per_class", + ] def _generate_counters(self, names): if names is None: @@ -1229,18 +1239,15 @@ def update_task(self, y, y_pred, adversarial=False, targeted=False): else self.tasks ) for metric in tasks: - if metric.name in [ - "object_detection_AP_per_class", - "apricot_patch_targeted_AP_per_class", - "dapricot_patch_targeted_AP_per_class", - ]: - metric.append_inputs(y, y_pred) + if metric.name in self.non_elementwise_metrics: + metric.append_input_label(y) + metric.append_input_pred(y_pred) else: - metric.append(y, y_pred) + metric.add_results(y, y_pred) def update_perturbation(self, x, x_adv): for metric in self.perturbations: - metric.append(x, x_adv) + metric.add_results(x, x_adv) def log_task(self, adversarial=False, targeted=False): if targeted: @@ -1266,31 +1273,17 @@ def log_task(self, adversarial=False, targeted=False): f"Word error rate on {task_type} examples relative to {wrt} labels: " f"{metric.total_wer():.2%}" ) - elif metric.name == "object_detection_AP_per_class": - average_precision_by_class = metric.AP_per_class() - logger.info( - f"object_detection_mAP on {task_type} examples relative to {wrt} labels: " - f"{np.fromiter(average_precision_by_class.values(), dtype=float).mean():.2%}." - f" object_detection_AP by class ID: {average_precision_by_class}" - ) - elif metric.name == "apricot_patch_targeted_AP_per_class": - apricot_patch_targeted_AP_by_class = ( - metric.apricot_patch_targeted_AP_per_class() - ) - logger.info( - f"apricot_patch_targeted_mAP on {task_type} examples: " - f"{np.fromiter(apricot_patch_targeted_AP_by_class.values(), dtype=float).mean():.2%}." - f" apricot_patch_targeted_AP by class ID: {apricot_patch_targeted_AP_by_class}" - ) - elif metric.name == "dapricot_patch_targeted_AP_per_class": - dapricot_patch_targeted_AP_by_class = ( - metric.dapricot_patch_targeted_AP_per_class() - ) + elif metric.name in self.non_elementwise_metrics: + metric_result = metric.compute_non_elementwise_metric() logger.info( - f"dapricot_patch_targeted_mAP on {task_type} examples: " - f"{np.fromiter(dapricot_patch_targeted_AP_by_class.values(), dtype=float).mean():.2%}." - f" dapricot_patch_targeted_AP by class ID: {dapricot_patch_targeted_AP_by_class}" + f"{metric.name} on {task_type} test examples relative to {wrt} labels: " + f"{metric_result}" ) + if metric.name in self.mean_ap_metrics: + logger.info( + f"mean {metric.name} on {task_type} examples relative to {wrt} labels " + f"{np.fromiter(metric_result.values(), dtype=float).mean():.2%}." + ) else: logger.info( f"Average {metric.name} on {task_type} test examples relative to {wrt} labels: " @@ -1309,39 +1302,13 @@ def results(self): (self.perturbations, "perturbation"), ]: for metric in metrics: - if metric.name == "object_detection_AP_per_class": - average_precision_by_class = metric.AP_per_class() - results[f"{prefix}_object_detection_mAP"] = np.fromiter( - average_precision_by_class.values(), dtype=float - ).mean() - results[f"{prefix}_{metric.name}"] = average_precision_by_class - continue - - if metric.name == "apricot_patch_targeted_AP_per_class": - apricot_patch_targeted_AP_by_class = ( - metric.apricot_patch_targeted_AP_per_class() - ) - results[f"{prefix}_apricot_patch_targeted_mAP"] = np.fromiter( - apricot_patch_targeted_AP_by_class.values(), dtype=float - ).mean() - results[ - f"{prefix}_{metric.name}" - ] = apricot_patch_targeted_AP_by_class - continue - - if metric.name == "dapricot_patch_targeted_AP_per_class": - # there are no non-targeted adversarial metrics for D-APRICOT scenario - if prefix == "adversarial": - continue - dapricot_patch_targeted_AP_by_class = ( - metric.dapricot_patch_targeted_AP_per_class() - ) - results[f"{prefix}_dapricot_patch_targeted_mAP"] = np.fromiter( - dapricot_patch_targeted_AP_by_class.values(), dtype=float - ).mean() - results[ - f"{prefix}_{metric.name}" - ] = dapricot_patch_targeted_AP_by_class + if metric.name in self.non_elementwise_metrics: + metric_result = metric.compute_non_elementwise_metric() + results[f"{prefix}_{metric.name}"] = metric_result + if metric.name in self.mean_ap_metrics: + results[f"{prefix}_mean_{metric.name}"] = np.fromiter( + metric_result.values(), dtype=float + ).mean() continue if self.full: diff --git a/armory/utils/triggers/bullet_holes.png b/armory/utils/triggers/bullet_holes.png new file mode 100644 index 000000000..69a321fab Binary files /dev/null and b/armory/utils/triggers/bullet_holes.png differ diff --git a/armory/utils/triggers/letter_A.png b/armory/utils/triggers/letter_A.png new file mode 100755 index 000000000..365c0ff5e Binary files /dev/null and b/armory/utils/triggers/letter_A.png differ diff --git a/docker/.build.sh.swp b/docker/.build.sh.swp new file mode 100644 index 000000000..5667b3272 Binary files /dev/null and b/docker/.build.sh.swp differ diff --git a/docker/build.sh b/docker/build.sh index 9c0d9b267..17c98e94c 100644 --- a/docker/build.sh +++ b/docker/build.sh @@ -43,6 +43,12 @@ for framework in "tf1" "tf2" "pytorch" "pytorch-deepspeech"; do echo "" echo "------------------------------------------------" echo "Building docker image for framework: $framework" + echo docker build --cache-from twosixarmory/${framework}:latest --force-rm --file docker/${framework}/Dockerfile --build-arg armory_version=${version} --target armory-${framework}${dev} -t twosixarmory/${framework}:${version} . docker build --cache-from twosixarmory/${framework}:latest --force-rm --file docker/${framework}/Dockerfile --build-arg armory_version=${version} --target armory-${framework}${dev} -t twosixarmory/${framework}:${version} . + if [[ $dev == "-dev" ]]; then + echo "Tagging docker image as latest" + echo docker tag twosixarmory/${framework}:${version} twosixarmory/${framework}:latest + docker tag twosixarmory/${framework}:${version} twosixarmory/${framework}:latest + fi fi done diff --git a/docker/pytorch-deepspeech/Dockerfile b/docker/pytorch-deepspeech/Dockerfile index 6bd7b12b9..ece7e7a6d 100644 --- a/docker/pytorch-deepspeech/Dockerfile +++ b/docker/pytorch-deepspeech/Dockerfile @@ -60,7 +60,7 @@ RUN /opt/conda/bin/pip install hydra-core==1.0.6 \ soundfile \ sox \ warpctc-pytorch==0.2.1+torch16.cuda102 \ - adversarial-robustness-toolbox==1.6.0 --no-cache-dir + adversarial-robustness-toolbox==1.6.1 --no-cache-dir ########## PyTorch 1 Deep Speech Dev ################# diff --git a/docker/pytorch/Dockerfile b/docker/pytorch/Dockerfile index d881f3aca..bc2221ac6 100644 --- a/docker/pytorch/Dockerfile +++ b/docker/pytorch/Dockerfile @@ -47,7 +47,7 @@ RUN /opt/conda/bin/conda install pytorch==1.7 \ /opt/conda/bin/conda clean --all RUN /opt/conda/bin/pip install --no-cache-dir \ tensorflow-gpu==2.4.1 \ - adversarial-robustness-toolbox==1.6.0 + adversarial-robustness-toolbox==1.6.1 ########## PyTorch 1 Dev ################# diff --git a/docker/tf1/Dockerfile b/docker/tf1/Dockerfile index 6c34fca6b..abbfee3a9 100644 --- a/docker/tf1/Dockerfile +++ b/docker/tf1/Dockerfile @@ -49,7 +49,10 @@ WORKDIR /tmp/models/research RUN protoc object_detection/protos/*.proto --python_out=. RUN cp object_detection/packages/tf1/setup.py . RUN /opt/conda/bin/pip install . -RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.0 +RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.1 + +# Note: this is necessary until the following TF issue is resolved: https://github.com/tensorflow/models/issues/9706 +RUN /opt/conda/bin/pip install --no-cache-dir numpy==1.19.2 WORKDIR /workspace diff --git a/docker/tf2/Dockerfile b/docker/tf2/Dockerfile index 0be71d5bf..fbb706023 100644 --- a/docker/tf2/Dockerfile +++ b/docker/tf2/Dockerfile @@ -50,7 +50,7 @@ WORKDIR /tmp/models/research RUN protoc object_detection/protos/*.proto --python_out=. RUN cp object_detection/packages/tf2/setup.py . RUN /opt/conda/bin/pip install . -RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.0 +RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.1 WORKDIR /workspace diff --git a/scenario_configs/apricot_frcnn.json b/scenario_configs/apricot_frcnn.json index 69fe5a6f0..97a40c613 100644 --- a/scenario_configs/apricot_frcnn.json +++ b/scenario_configs/apricot_frcnn.json @@ -44,7 +44,7 @@ "name": "ObjectDetectionTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/apricot_frcnn_defended.json b/scenario_configs/apricot_frcnn_defended.json index b4f89f943..7b2269a49 100644 --- a/scenario_configs/apricot_frcnn_defended.json +++ b/scenario_configs/apricot_frcnn_defended.json @@ -23,7 +23,6 @@ "kwargs": { "apply_fit": false, "apply_predict": true, - "channel_index": 3, "clip_values": [ 0.0, 1.0 @@ -58,7 +57,7 @@ "name": "ObjectDetectionTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/asr_deepspeech_baseline.json b/scenario_configs/asr_deepspeech_baseline.json index c3a8ca1a0..0b1aeaa16 100644 --- a/scenario_configs/asr_deepspeech_baseline.json +++ b/scenario_configs/asr_deepspeech_baseline.json @@ -7,27 +7,29 @@ "knowledge": "white", "kwargs": { "decrease_factor_alpha": 0.5, + "eps": 0.0075, "global_max_length": 562480, "increase_factor_alpha": 2.0, - "initial_eps": 0.0075, - "initial_rescale": 1.0, - "learning_rate_1st_stage": 0.0001, - "learning_rate_2nd_stage": 1e-07, - "max_iter_1st_stage": 400, - "max_iter_2nd_stage": 100, - "num_iter_decrease_alpha": 50, - "rescale_factor": 0.5 + "initial_rescale": 0.5, + "learning_rate_1": 0.0001, + "learning_rate_2": 1e-07, + "max_iter_1": 400, + "max_iter_2": 100, + "num_iter_decrease_alpha": 50 }, "module": "art.attacks.evasion.imperceptible_asr.imperceptible_asr_pytorch", - "name": "ImperceptibleASRPytorch", + "name": "ImperceptibleASRPyTorch", "targeted": true, "targeted_labels": { - "scheme": "matched length", - "transcripts": [ - "REALLY SHORT TEST STRING", - "THE TEST STRING HAS A LENGTH EQUAL TO THE MEDIAN OF THE CLEAN TEST TRANSCRIPT LENGTHS", - "THIS IS AN EXCEEDINGLY LONG TEST STRING BUT NOT REALLY AS THE LONGEST STRING HAS OVER FIVE HUNDRED CHARACTERS IN ITS TRANSCRIPT AND INCLUDES A LIST OF PEOPLE AND SPEASKS OF A SENATOR FROM NEW JERSEY" - ] + "kwargs": { + "transcripts": [ + "REALLY SHORT TEST STRING", + "THE TEST STRING HAS A LENGTH EQUAL TO THE MEDIAN OF THE CLEAN TEST TRANSCRIPT LENGTHS", + "THIS IS AN EXCEEDINGLY LONG TEST STRING BUT NOT REALLY AS THE LONGEST STRING HAS OVER FIVE HUNDRED CHARACTERS IN ITS TRANSCRIPT AND INCLUDES A LIST OF PEOPLE AND SPEASKS OF A SENATOR FROM NEW JERSEY" + ] + }, + "module": "armory.utils.labels", + "name": "MatchedTranscriptLengthTargeter" }, "use_label": false }, @@ -74,7 +76,7 @@ "name": "AutomaticSpeechRecognition" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch-deepspeech:0.13.0", + "docker_image": "twosixarmory/pytorch-deepspeech:0.13.1", "external_github_repo": "hkakitani/deepspeech.pytorch", "gpus": "all", "local_repo_path": null, diff --git a/scenario_configs/asr_deepspeech_baseline_fgsm.json b/scenario_configs/asr_deepspeech_baseline_fgsm.json index a99d105f5..5ca632d52 100644 --- a/scenario_configs/asr_deepspeech_baseline_fgsm.json +++ b/scenario_configs/asr_deepspeech_baseline_fgsm.json @@ -17,8 +17,11 @@ "name": "FastGradientMethod", "targeted": true, "targeted_labels": { - "scheme": "string", - "value": "TEST STRING" + "kwargs": { + "value": "TEST STRING" + }, + "module": "armory.utils.labels", + "name": "FixedStringTargeter" }, "use_label": false }, @@ -65,7 +68,7 @@ "name": "AutomaticSpeechRecognition" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch-deepspeech:0.13.0", + "docker_image": "twosixarmory/pytorch-deepspeech:0.13.1", "external_github_repo": "hkakitani/deepspeech.pytorch", "gpus": "all", "local_repo_path": null, diff --git a/scenario_configs/asr_deepspeech_baseline_fgsm_channel.json b/scenario_configs/asr_deepspeech_baseline_fgsm_channel.json index 762bb1cb9..481a31bdc 100644 --- a/scenario_configs/asr_deepspeech_baseline_fgsm_channel.json +++ b/scenario_configs/asr_deepspeech_baseline_fgsm_channel.json @@ -22,8 +22,11 @@ "name": "FastGradientMethod", "targeted": true, "targeted_labels": { - "scheme": "string", - "value": "TEST STRING" + "kwargs": { + "value": "TEST STRING" + }, + "module": "armory.utils.labels", + "name": "FixedStringTargeter" }, "use_label": false }, @@ -70,7 +73,7 @@ "name": "AutomaticSpeechRecognition" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch-deepspeech:0.13.0", + "docker_image": "twosixarmory/pytorch-deepspeech:0.13.1", "external_github_repo": "hkakitani/deepspeech.pytorch", "gpus": "all", "local_repo_path": null, diff --git a/scenario_configs/asr_deepspeech_baseline_kenansville.json b/scenario_configs/asr_deepspeech_baseline_kenansville.json index 7d3031db5..1b6eec608 100644 --- a/scenario_configs/asr_deepspeech_baseline_kenansville.json +++ b/scenario_configs/asr_deepspeech_baseline_kenansville.json @@ -53,7 +53,7 @@ "name": "AutomaticSpeechRecognition" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch-deepspeech:0.13.0", + "docker_image": "twosixarmory/pytorch-deepspeech:0.13.1", "external_github_repo": "hkakitani/deepspeech.pytorch", "gpus": "all", "local_repo_path": null, diff --git a/scenario_configs/asr_deepspeech_defended_baseline_kenansville.json b/scenario_configs/asr_deepspeech_defended_baseline_kenansville.json index 870732b43..cd12f0553 100644 --- a/scenario_configs/asr_deepspeech_defended_baseline_kenansville.json +++ b/scenario_configs/asr_deepspeech_defended_baseline_kenansville.json @@ -64,7 +64,7 @@ "name": "AutomaticSpeechRecognition" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch-deepspeech:0.13.0", + "docker_image": "twosixarmory/pytorch-deepspeech:0.13.1", "external_github_repo": "hkakitani/deepspeech.pytorch", "gpus": "all", "local_repo_path": null, diff --git a/scenario_configs/asr_deepspeech_snr.json b/scenario_configs/asr_deepspeech_snr.json index dc22c634e..984f4f45d 100644 --- a/scenario_configs/asr_deepspeech_snr.json +++ b/scenario_configs/asr_deepspeech_snr.json @@ -18,8 +18,11 @@ "name": "SNR_PGD_Numpy", "targeted": true, "targeted_labels": { - "scheme": "string", - "value": "TEST STRING" + "kwargs": { + "value": "TEST STRING" + }, + "module": "armory.utils.labels", + "name": "FixedStringTargeter" }, "use_label": false }, @@ -66,7 +69,7 @@ "name": "AutomaticSpeechRecognition" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch-deepspeech:0.13.0", + "docker_image": "twosixarmory/pytorch-deepspeech:0.13.1", "external_github_repo": "hkakitani/deepspeech.pytorch", "gpus": "all", "local_repo_path": null, diff --git a/scenario_configs/cifar10_baseline.json b/scenario_configs/cifar10_baseline.json index 0c6e56c1a..b6fd7ec27 100644 --- a/scenario_configs/cifar10_baseline.json +++ b/scenario_configs/cifar10_baseline.json @@ -49,7 +49,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/cifar10_defended_example.json b/scenario_configs/cifar10_defended_example.json index e183d7d4d..f89e7db6e 100644 --- a/scenario_configs/cifar10_defended_example.json +++ b/scenario_configs/cifar10_defended_example.json @@ -57,7 +57,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/dapricot_frcnn.json b/scenario_configs/dapricot_frcnn.json deleted file mode 100644 index dcd396ddb..000000000 --- a/scenario_configs/dapricot_frcnn.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "_description": "DAPRICOT object detection, contributed by MITRE Corporation", - "adhoc": null, - "attack": { - "generate_kwargs": { - "threat_model": "physical" - }, - "knowledge": "white", - "kwargs": {}, - "module": "armory.art_experimental.attacks.dapricot_patch", - "name": "DApricotPatch", - "targeted_labels": { - "scheme": "object_detection_fixed", - "value": 2 - } - }, - "dataset": { - "batch_size": 1, - "eval_split": "large+medium+small", - "framework": "numpy", - "module": "armory.data.adversarial_datasets", - "name": "dapricot_dev_adversarial" - }, - "defense": null, - "metric": { - "means": true, - "perturbation": "l0", - "record_metric_per_sample": false, - "task": [ - "dapricot_patch_targeted_AP_per_class" - ] - }, - "model": { - "fit": false, - "fit_kwargs": {}, - "model_kwargs": {}, - "module": "armory.baseline_models.tf_graph.mscoco_frcnn", - "name": "get_art_model", - "weights_file": null, - "wrapper_kwargs": {} - }, - "scenario": { - "kwargs": {}, - "module": "armory.scenarios.dapricot_scenario", - "name": "ObjectDetectionTask" - }, - "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", - "external_github_repo": null, - "gpus": "all", - "output_dir": null, - "output_filename": null, - "use_gpu": false - } -} diff --git a/scenario_configs/dapricot_frcnn_masked_pgd.json b/scenario_configs/dapricot_frcnn_masked_pgd.json index 3f21bc90e..65324e05d 100644 --- a/scenario_configs/dapricot_frcnn_masked_pgd.json +++ b/scenario_configs/dapricot_frcnn_masked_pgd.json @@ -10,7 +10,8 @@ "batch_size": 1, "eps": 1.0, "eps_step": 0.02, - "max_iter": 100 + "max_iter": 100, + "targeted": true }, "module": "armory.art_experimental.attacks.dapricot_patch", "name": "DApricotMaskedPGD", @@ -32,7 +33,8 @@ "perturbation": "l0", "record_metric_per_sample": false, "task": [ - "dapricot_patch_targeted_AP_per_class" + "dapricot_patch_targeted_AP_per_class", + "dapricot_patch_target_success" ] }, "model": { @@ -45,13 +47,14 @@ "wrapper_kwargs": {} }, "scenario": { + "export_samples": 30, "kwargs": {}, "module": "armory.scenarios.dapricot_scenario", "name": "ObjectDetectionTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", - "external_github_repo": null, + "docker_image": "twosixarmory/tf1:0.13.1", + "external_github_repo": "colour-science/colour", "gpus": "all", "output_dir": null, "output_filename": null, diff --git a/scenario_configs/gtsrb_scenario_baseline.json b/scenario_configs/gtsrb_scenario_baseline.json index 22c2d5ae7..2bb55f2b0 100644 --- a/scenario_configs/gtsrb_scenario_baseline.json +++ b/scenario_configs/gtsrb_scenario_baseline.json @@ -9,10 +9,10 @@ "experiment_id": 0, "fraction_poisoned": 0.1, "poison_dataset": true, - "source_class": 5, + "source_class": 1, "split_id": 0, - "target_class": 42, - "train_epochs": 10, + "target_class": 2, + "train_epochs": 30, "use_poison_filtering_defense": true }, "attack": { @@ -58,7 +58,7 @@ "name": "GTSRB" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/gtsrb_scenario_baseline_pytorch.json b/scenario_configs/gtsrb_scenario_baseline_pytorch.json new file mode 100644 index 000000000..8ab3d2ba5 --- /dev/null +++ b/scenario_configs/gtsrb_scenario_baseline_pytorch.json @@ -0,0 +1,69 @@ +{ + "_description": "GTSRB poison image classification, contributed by MITRE Corporation", + "adhoc": { + "detection_kwargs": { + "nb_clusters": 2, + "nb_dims": 43, + "reduce": "PCA" + }, + "experiment_id": 0, + "fraction_poisoned": 0.1, + "poison_dataset": true, + "source_class": 1, + "split_id": 0, + "target_class": 2, + "train_epochs": 30, + "use_poison_filtering_defense": true + }, + "attack": { + "knowledge": "black", + "kwargs": { + "poison_module": "art.attacks.poisoning.perturbations", + "poison_type": "pattern" + }, + "module": "armory.art_experimental.attacks.poison_loader", + "name": "poison_loader_GTSRB" + }, + "dataset": { + "batch_size": 512, + "framework": "numpy", + "module": "armory.data.datasets", + "name": "german_traffic_sign" + }, + "defense": { + "kwargs": { + "cluster_analysis": "smaller", + "clustering_method": "KMeans", + "nb_clusters": 2, + "nb_dims": 43, + "reduce": "PCA" + }, + "module": "art.defences.detector.poison.activation_defence", + "name": "ActivationDefence", + "type": "PoisonFilteringDefence" + }, + "metric": null, + "model": { + "fit": true, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.pytorch.micronnet_gtsrb", + "name": "get_art_model", + "weights_file": null, + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.poisoning_gtsrb_scenario", + "name": "GTSRB" + }, + "sysconfig": { + "docker_image": "twosixarmory/pytorch:0.13.1", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "set_pythonhashseed": true, + "use_gpu": false + } +} diff --git a/scenario_configs/gtsrb_scenario_clbd.json b/scenario_configs/gtsrb_scenario_clbd.json index 2586904e4..e7852a590 100644 --- a/scenario_configs/gtsrb_scenario_clbd.json +++ b/scenario_configs/gtsrb_scenario_clbd.json @@ -58,7 +58,7 @@ "name": "GTSRB_CLBD" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/gtsrb_scenario_clbd_bullethole.json b/scenario_configs/gtsrb_scenario_clbd_bullethole.json new file mode 100644 index 000000000..eb967b972 --- /dev/null +++ b/scenario_configs/gtsrb_scenario_clbd_bullethole.json @@ -0,0 +1,75 @@ +{ + "_description": "GTSRB poison image classification, contributed by MITRE Corporation", + "adhoc": { + "detection_kwargs": { + "nb_clusters": 2, + "nb_dims": 43, + "reduce": "PCA" + }, + "experiment_id": 0, + "poison_dataset": true, + "source_class": 1, + "split_id": 0, + "target_class": 2, + "train_epochs": 50, + "use_poison_filtering_defense": false + }, + "attack": { + "knowledge": "black", + "kwargs": { + "backdoor_kwargs": { + "backdoor_packaged_with_armory": true, + "backdoor_path": "./armory/utils/triggers/bullet_holes.png", + "poison_module": "art.attacks.poisoning.perturbations", + "poison_type": "image", + "size": [ + 17, + 15 + ] + }, + "eps": 0.03, + "eps_step": 0.001, + "max_iter": 100, + "n_classes": 43, + "norm": "inf", + "num_random_init": 2, + "pp_poison": 0.5, + "target": 2 + }, + "module": "armory.art_experimental.attacks.poison_loader_clbd", + "name": "poison_loader_clbd", + "type": "clbd", + "use_adversarial_trainer": false + }, + "dataset": { + "batch_size": 512, + "framework": "numpy", + "module": "armory.data.datasets", + "name": "german_traffic_sign" + }, + "defense": null, + "metric": null, + "model": { + "fit": true, + "fit_kwargs": {}, + "model_kwargs": {}, + "module": "armory.baseline_models.keras.micronnet_gtsrb", + "name": "get_art_model", + "weights_file": null, + "wrapper_kwargs": {} + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.poisoning_gtsrb_clbd", + "name": "GTSRB_CLBD" + }, + "sysconfig": { + "docker_image": "twosixarmory/tf1:0.13.1", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "set_pythonhashseed": true, + "use_gpu": false + } +} diff --git a/scenario_configs/gtsrb_scenario_clbd_defended.json b/scenario_configs/gtsrb_scenario_clbd_defended.json index 84865adf3..6953ea367 100644 --- a/scenario_configs/gtsrb_scenario_clbd_defended.json +++ b/scenario_configs/gtsrb_scenario_clbd_defended.json @@ -69,7 +69,7 @@ "name": "GTSRB_CLBD" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/gtsrb_scenario_poison.json b/scenario_configs/gtsrb_scenario_poison.json index 581c234e5..7d587298e 100644 --- a/scenario_configs/gtsrb_scenario_poison.json +++ b/scenario_configs/gtsrb_scenario_poison.json @@ -48,7 +48,7 @@ "name": "GTSRB" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "use_gpu": false diff --git a/scenario_configs/librispeech_baseline_sincnet.json b/scenario_configs/librispeech_baseline_sincnet.json index 0ea1d0d61..5144871b6 100644 --- a/scenario_configs/librispeech_baseline_sincnet.json +++ b/scenario_configs/librispeech_baseline_sincnet.json @@ -55,7 +55,7 @@ "name": "AudioClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "hkakitani/SincNet", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/librispeech_baseline_sincnet_snr_pgd.json b/scenario_configs/librispeech_baseline_sincnet_snr_pgd.json index 71aabe6bb..44b70427e 100644 --- a/scenario_configs/librispeech_baseline_sincnet_snr_pgd.json +++ b/scenario_configs/librispeech_baseline_sincnet_snr_pgd.json @@ -59,7 +59,7 @@ "name": "AudioClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "hkakitani/SincNet", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/librispeech_baseline_sincnet_targeted.json b/scenario_configs/librispeech_baseline_sincnet_targeted.json index 878f98500..c0881597a 100644 --- a/scenario_configs/librispeech_baseline_sincnet_targeted.json +++ b/scenario_configs/librispeech_baseline_sincnet_targeted.json @@ -14,8 +14,11 @@ "module": "art.attacks.evasion", "name": "FastGradientMethod", "targeted_labels": { - "num_classes": 40, - "scheme": "round-robin" + "kwargs": { + "num_classes": 40 + }, + "module": "armory.utils.labels", + "name": "RoundRobinTargeter" }, "use_label": false }, @@ -59,7 +62,7 @@ "name": "AudioClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "hkakitani/SincNet", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/mnist_baseline.json b/scenario_configs/mnist_baseline.json index fb2a90c7c..5b51d2103 100644 --- a/scenario_configs/mnist_baseline.json +++ b/scenario_configs/mnist_baseline.json @@ -47,7 +47,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/resisc10_poison_dlbd.json b/scenario_configs/resisc10_poison_dlbd.json new file mode 100644 index 000000000..784c07b6a --- /dev/null +++ b/scenario_configs/resisc10_poison_dlbd.json @@ -0,0 +1,125 @@ +{ + "_description": "RESISC10 poison image classification, contributed by MITRE Corporation", + "adhoc": { + "detection_kwargs": { + "nb_clusters": 2, + "nb_dims": 43, + "reduce": "PCA" + }, + "experiment_id": 0, + "fraction_poisoned": 0.1, + "poison_dataset": true, + "source_class": 1, + "split_id": 0, + "target_class": 5, + "train_epochs": 200, + "use_poison_filtering_defense": false + }, + "attack": { + "knowledge": "black", + "kwargs": { + "backdoor_packaged_with_armory": true, + "backdoor_path": "./armory/utils/triggers/letter_A.png", + "base_img_size_x": 64, + "base_img_size_y": 64, + "blend": 0.8, + "channels_first": false, + "mode": "RGB", + "poison_module": "art.attacks.poisoning.perturbations", + "poison_type": "image", + "size": [ + 10, + 10 + ] + }, + "module": "armory.art_experimental.attacks.poison_loader", + "name": "poison_loader_GTSRB" + }, + "dataset": { + "batch_size": 512, + "framework": "numpy", + "module": "armory.data.datasets", + "name": "resisc10" + }, + "defense": { + "data_augmentation": { + "rotation": { + "kwargs": { + "apply_fit": false, + "apply_predict": false, + "clip_values": [ + 0.0, + 1.0 + ], + "degree": 0.0, + "nb_samples": 1, + "scale": [ + 0.8, + 1.2 + ], + "translate": [ + 0.1, + 0.1 + ] + }, + "module": "armory.art_experimental.defences.random_affine_pytorch", + "name": "EoTRandomAffinePyTorch", + "type": "Preprocessor" + } + }, + "kwargs": { + "cluster_analysis": "smaller", + "clustering_method": "KMeans", + "nb_clusters": 2, + "nb_dims": 43, + "reduce": "PCA" + }, + "module": "art.defences.detector.poison.activation_defence", + "name": "ActivationDefence", + "type": "PoisonFilteringDefence" + }, + "metric": null, + "model": { + "fit": true, + "fit_kwargs": {}, + "model_kwargs": { + "data_means": [ + 0.37853524, + 0.38404912, + 0.36049628 + ], + "data_stds": [ + 0.18050115, + 0.17266262, + 0.173474 + ], + "num_classes": 10, + "pretrained": false + }, + "module": "armory.baseline_models.pytorch.resnet18", + "name": "get_art_model", + "weights_file": null, + "wrapper_kwargs": { + "input_shape": [ + 64, + 64, + 3 + ], + "nb_classes": 10 + } + }, + "scenario": { + "kwargs": {}, + "module": "armory.scenarios.poisoning_resisc10_scenario", + "name": "RESISC10" + }, + "sysconfig": { + "docker_image": "twosixarmory/pytorch:0.13.1", + "external_github_repo": null, + "gpus": "all", + "output_dir": null, + "output_filename": null, + "set_pythonhashseed": true, + "use_gpu": false + } +} diff --git a/scenario_configs/resisc45_baseline_densenet121.json b/scenario_configs/resisc45_baseline_densenet121.json index 6d952b401..47c78b946 100644 --- a/scenario_configs/resisc45_baseline_densenet121.json +++ b/scenario_configs/resisc45_baseline_densenet121.json @@ -45,7 +45,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/resisc45_baseline_densenet121_cascade.json b/scenario_configs/resisc45_baseline_densenet121_cascade.json index c7dc507c6..a24b5778c 100644 --- a/scenario_configs/resisc45_baseline_densenet121_cascade.json +++ b/scenario_configs/resisc45_baseline_densenet121_cascade.json @@ -70,7 +70,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/resisc45_baseline_densenet121_finetune.json b/scenario_configs/resisc45_baseline_densenet121_finetune.json index fd90247ff..7d4ae54dd 100644 --- a/scenario_configs/resisc45_baseline_densenet121_finetune.json +++ b/scenario_configs/resisc45_baseline_densenet121_finetune.json @@ -45,7 +45,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/resisc45_baseline_densenet121_patch.json b/scenario_configs/resisc45_baseline_densenet121_patch.json index fabd41848..a3a7278d2 100644 --- a/scenario_configs/resisc45_baseline_densenet121_patch.json +++ b/scenario_configs/resisc45_baseline_densenet121_patch.json @@ -19,8 +19,11 @@ "module": "art.attacks.evasion", "name": "AdversarialPatch", "targeted_labels": { - "num_classes": 45, - "scheme": "round-robin" + "kwargs": { + "num_classes": 45 + }, + "module": "armory.utils.labels", + "name": "RoundRobinTargeter" }, "type": "patch", "use_label": false @@ -55,7 +58,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/resisc45_baseline_densenet121_targeted.json b/scenario_configs/resisc45_baseline_densenet121_targeted.json index 59336918e..6261ddbe9 100644 --- a/scenario_configs/resisc45_baseline_densenet121_targeted.json +++ b/scenario_configs/resisc45_baseline_densenet121_targeted.json @@ -14,8 +14,11 @@ "module": "art.attacks.evasion", "name": "FastGradientMethod", "targeted_labels": { - "num_classes": 45, - "scheme": "round-robin" + "kwargs": { + "num_classes": 45 + }, + "module": "armory.utils.labels", + "name": "RoundRobinTargeter" }, "use_label": false }, @@ -49,7 +52,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/so2sat_baseline.json b/scenario_configs/so2sat_baseline.json index 3f3abecf7..41a5aee5d 100644 --- a/scenario_configs/so2sat_baseline.json +++ b/scenario_configs/so2sat_baseline.json @@ -92,7 +92,7 @@ "name": "So2SatClassification" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/so2sat_defended_baseline.json b/scenario_configs/so2sat_defended_baseline.json index 4aef57bf3..edc323c0a 100644 --- a/scenario_configs/so2sat_defended_baseline.json +++ b/scenario_configs/so2sat_defended_baseline.json @@ -138,7 +138,7 @@ "name": "So2SatClassification" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/ucf101_baseline_finetune.json b/scenario_configs/ucf101_baseline_finetune.json index 75f9a1b4a..2e38d2ea2 100644 --- a/scenario_configs/ucf101_baseline_finetune.json +++ b/scenario_configs/ucf101_baseline_finetune.json @@ -52,7 +52,7 @@ "name": "Ucf101" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "yusong-tan/MARS", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/ucf101_baseline_pretrained.json b/scenario_configs/ucf101_baseline_pretrained.json index 7c684d40d..8dd5e78b3 100644 --- a/scenario_configs/ucf101_baseline_pretrained.json +++ b/scenario_configs/ucf101_baseline_pretrained.json @@ -64,7 +64,7 @@ "name": "Ucf101" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "yusong-tan/MARS", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/ucf101_baseline_pretrained_frame_saliency.json b/scenario_configs/ucf101_baseline_pretrained_frame_saliency.json index 11ba7bb6b..87144df03 100644 --- a/scenario_configs/ucf101_baseline_pretrained_frame_saliency.json +++ b/scenario_configs/ucf101_baseline_pretrained_frame_saliency.json @@ -55,7 +55,7 @@ "name": "Ucf101" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "yusong-tan/MARS", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/ucf101_baseline_pretrained_pgd_patch.json b/scenario_configs/ucf101_baseline_pretrained_pgd_patch.json index 778ea125e..a64898f64 100644 --- a/scenario_configs/ucf101_baseline_pretrained_pgd_patch.json +++ b/scenario_configs/ucf101_baseline_pretrained_pgd_patch.json @@ -63,7 +63,7 @@ "name": "Ucf101" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "yusong-tan/MARS", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/ucf101_baseline_pretrained_targeted.json b/scenario_configs/ucf101_baseline_pretrained_targeted.json index dacc6856b..d06e6d530 100644 --- a/scenario_configs/ucf101_baseline_pretrained_targeted.json +++ b/scenario_configs/ucf101_baseline_pretrained_targeted.json @@ -14,8 +14,11 @@ "module": "art.attacks.evasion", "name": "FastGradientMethod", "targeted_labels": { - "num_classes": 101, - "scheme": "round-robin" + "kwargs": { + "num_classes": 101 + }, + "module": "armory.utils.labels", + "name": "RoundRobinTargeter" }, "use_label": false }, @@ -55,7 +58,7 @@ "name": "Ucf101" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "yusong-tan/MARS", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/ucf101_defended_baseline_pretrained.json b/scenario_configs/ucf101_defended_baseline_pretrained.json index 8513af89a..fddb88726 100644 --- a/scenario_configs/ucf101_defended_baseline_pretrained.json +++ b/scenario_configs/ucf101_defended_baseline_pretrained.json @@ -76,7 +76,7 @@ "name": "Ucf101" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": "yusong-tan/MARS", "gpus": "all", "output_dir": null, diff --git a/scenario_configs/xview_frcnn.json b/scenario_configs/xview_frcnn.json index 585bc5e8b..b91b76ec9 100644 --- a/scenario_configs/xview_frcnn.json +++ b/scenario_configs/xview_frcnn.json @@ -53,7 +53,7 @@ "name": "ObjectDetectionTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/xview_frcnn_defended.json b/scenario_configs/xview_frcnn_defended.json index 88b8f33f2..332a3b406 100644 --- a/scenario_configs/xview_frcnn_defended.json +++ b/scenario_configs/xview_frcnn_defended.json @@ -33,7 +33,6 @@ "kwargs": { "apply_fit": false, "apply_predict": true, - "channel_index": 3, "clip_values": [ 0.0, 1.0 @@ -67,7 +66,7 @@ "name": "ObjectDetectionTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/scenario_configs/xview_frcnn_targeted.json b/scenario_configs/xview_frcnn_targeted.json index 1c6679a75..860bfae44 100644 --- a/scenario_configs/xview_frcnn_targeted.json +++ b/scenario_configs/xview_frcnn_targeted.json @@ -22,8 +22,11 @@ "module": "armory.art_experimental.attacks.pgd_patch", "name": "PGDPatch", "targeted_labels": { - "scheme": "object_detection_fixed", - "value": 2 + "kwargs": { + "value": 2 + }, + "module": "armory.utils.labels", + "name": "ObjectDetectionFixedLabelTargeter" }, "use_label": false }, @@ -57,7 +60,7 @@ "name": "ObjectDetectionTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/broken/invalid_dataset_framework.json b/tests/scenarios/broken/invalid_dataset_framework.json index 6ab48e99e..8d330db43 100644 --- a/tests/scenarios/broken/invalid_dataset_framework.json +++ b/tests/scenarios/broken/invalid_dataset_framework.json @@ -46,7 +46,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/broken/invalid_module.json b/tests/scenarios/broken/invalid_module.json index 0b6ce0353..f44113f75 100644 --- a/tests/scenarios/broken/invalid_module.json +++ b/tests/scenarios/broken/invalid_module.json @@ -39,7 +39,7 @@ "name": "fgm_attack" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "", "output_dir": null, diff --git a/tests/scenarios/broken/missing_scenario.json b/tests/scenarios/broken/missing_scenario.json index 20d4abde9..ffbffe8b5 100644 --- a/tests/scenarios/broken/missing_scenario.json +++ b/tests/scenarios/broken/missing_scenario.json @@ -40,7 +40,7 @@ "wrapper_kwargs": {} }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/pytorch/image_classification.json b/tests/scenarios/pytorch/image_classification.json index 46223799c..cc1d71f6d 100644 --- a/tests/scenarios/pytorch/image_classification.json +++ b/tests/scenarios/pytorch/image_classification.json @@ -46,7 +46,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/pytorch/image_classification_pretrained.json b/tests/scenarios/pytorch/image_classification_pretrained.json index 134be5a01..b7ac24090 100644 --- a/tests/scenarios/pytorch/image_classification_pretrained.json +++ b/tests/scenarios/pytorch/image_classification_pretrained.json @@ -44,7 +44,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/pytorch:0.13.0", + "docker_image": "twosixarmory/pytorch:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/tf1/image_classification.json b/tests/scenarios/tf1/image_classification.json index a1cdb5d45..d7d8e2744 100644 --- a/tests/scenarios/tf1/image_classification.json +++ b/tests/scenarios/tf1/image_classification.json @@ -46,7 +46,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/tf1/image_classification_pretrained.json b/tests/scenarios/tf1/image_classification_pretrained.json index 868929439..556f41c00 100644 --- a/tests/scenarios/tf1/image_classification_pretrained.json +++ b/tests/scenarios/tf1/image_classification_pretrained.json @@ -44,7 +44,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/scenarios/tf1/image_classification_tfgraph.json b/tests/scenarios/tf1/image_classification_tfgraph.json index 0d174bd89..64cf53ed7 100644 --- a/tests/scenarios/tf1/image_classification_tfgraph.json +++ b/tests/scenarios/tf1/image_classification_tfgraph.json @@ -46,7 +46,7 @@ "name": "ImageClassificationTask" }, "sysconfig": { - "docker_image": "twosixarmory/tf1:0.13.0", + "docker_image": "twosixarmory/tf1:0.13.1", "external_github_repo": null, "gpus": "all", "output_dir": null, diff --git a/tests/test_docker/test_dataset.py b/tests/test_docker/test_dataset.py index 838cfed83..976a3437f 100644 --- a/tests/test_docker/test_dataset.py +++ b/tests/test_docker/test_dataset.py @@ -225,6 +225,23 @@ def test_cifar(): assert y.shape == (batch_size,) +def test_cifar100(): + batch_size = 500 + for split, size in [("train", 50000), ("test", 10000)]: + dataset = datasets.cifar100( + split=split, epochs=1, batch_size=batch_size, dataset_dir=DATASET_DIR, + ) + assert dataset.size == size + assert dataset.batch_size == batch_size + assert dataset.batches_per_epoch == ( + size // batch_size + bool(size % batch_size) + ) + + x, y = dataset.get_batch() + assert x.shape == (batch_size, 32, 32, 3) + assert y.shape == (batch_size,) + + def test_digit(): epochs = 1 batch_size = 1 @@ -371,6 +388,24 @@ def test_resisc45(): assert y.shape == (batch_size,) +def test_resisc10(): + for split, size in [("train", 5000), ("validation", 1000), ("test", 1000)]: + batch_size = 16 + epochs = 1 + dataset = datasets.resisc10( + split=split, epochs=epochs, batch_size=batch_size, dataset_dir=DATASET_DIR, + ) + assert dataset.size == size + assert dataset.batch_size == batch_size + assert dataset.batches_per_epoch == ( + size // batch_size + bool(size % batch_size) + ) + + x, y = dataset.get_batch() + assert x.shape == (batch_size, 64, 64, 3) + assert y.shape == (batch_size,) + + def test_librispeech_adversarial(): if not os.path.exists( os.path.join(DATASET_DIR, "librispeech_adversarial", "1.0.0") diff --git a/tests/test_docker/test_metrics.py b/tests/test_docker/test_metrics.py index 10ef8bb95..03352f461 100644 --- a/tests/test_docker/test_metrics.py +++ b/tests/test_docker/test_metrics.py @@ -110,8 +110,8 @@ def test_snr_spectrogram(): def test_metric_list(): metric_list = metrics.MetricList("categorical_accuracy") - metric_list.append([1], [1]) - metric_list.append([1, 2, 3], [1, 0, 2]) + metric_list.add_results([1], [1]) + metric_list.add_results([1, 2, 3], [1, 0, 2]) assert metric_list.mean() == 0.5 assert metric_list.values() == [1, 1, 0, 0] @@ -152,6 +152,6 @@ def test_mAP(): "scores": np.array([0.8, 0.8]), } - ap_per_class = metrics.object_detection_AP_per_class([[labels]], [[preds]]) + ap_per_class = metrics.object_detection_AP_per_class([labels], [preds]) assert ap_per_class[9] == 0 assert ap_per_class[2] >= 0.99 diff --git a/tests/test_pytorch/test_pytorch_models.py b/tests/test_pytorch/test_pytorch_models.py index b1facba06..ba792ec22 100644 --- a/tests/test_pytorch/test_pytorch_models.py +++ b/tests/test_pytorch/test_pytorch_models.py @@ -113,8 +113,8 @@ def test_pytorch_xview_pretrained(): list_of_ypreds = [] for x, y in test_dataset: y_pred = detector.predict(x) - list_of_ys.append(y) - list_of_ypreds.append(y_pred) + list_of_ys.extend(y) + list_of_ypreds.extend(y_pred) average_precision_by_class = object_detection_AP_per_class( list_of_ys, list_of_ypreds diff --git a/tests/test_tf1/test_tf1_models.py b/tests/test_tf1/test_tf1_models.py index e5a39589c..b2df48f00 100644 --- a/tests/test_tf1/test_tf1_models.py +++ b/tests/test_tf1/test_tf1_models.py @@ -60,8 +60,8 @@ def test_tf1_apricot(): list_of_ypreds = [] for x, y in dev_dataset: y_pred = detector.predict(x) - list_of_ys.append(y) - list_of_ypreds.append(y_pred) + list_of_ys.extend(y) + list_of_ypreds.extend(y_pred) average_precision_by_class = object_detection_AP_per_class( list_of_ys, list_of_ypreds @@ -96,8 +96,8 @@ def test_tf1_apricot(): list_of_ypreds = [] for x, y in test_dataset: y_pred = detector.predict(x) - list_of_ys.append(y) - list_of_ypreds.append(y_pred) + list_of_ys.extend(y) + list_of_ypreds.extend(y_pred) average_precision_by_class = object_detection_AP_per_class( list_of_ys, list_of_ypreds