merge r0.13.4 into dev

A rather complex manual merge. There may well be extra, or unmodified scenario_configs

armory/art_experimental/attacks/pgd_patch.py

@@ -1,7 +1,12 @@
+import logging
+
 from art.attacks.evasion import ProjectedGradientDescent
 import numpy as np
 
 
+logger = logging.getLogger(__name__)
+
+
 class PGDPatch(ProjectedGradientDescent):
     """
     Apply Masked PGD to image and video inputs,
@@ -15,15 +20,29 @@ def __init__(self, estimator, **kwargs):
     def generate(self, x, y=None, **generate_kwargs):
         video_input = generate_kwargs.get("video_input", False)
 
+        if "patch_ratio" in generate_kwargs:
+            patch_ratio = generate_kwargs["patch_ratio"]
+            patch_height = int(x.shape[-3] * patch_ratio ** 0.5)
+            patch_width = int(x.shape[-2] * patch_ratio ** 0.5)
+        elif "patch_height" in generate_kwargs and "patch_width" in generate_kwargs:
+            patch_height = generate_kwargs["patch_height"]
+            patch_width = generate_kwargs["patch_width"]
+        else:
+            raise ValueError(
+                "generate_kwargs did not define 'patch_ratio', or it did not define 'patch_height' and 'patch_width"
+            )
+
         if "ymin" in generate_kwargs:
             ymin = generate_kwargs["ymin"]
         else:
-            raise ValueError("generate_kwargs did not define 'ymin'")
+            logger.info("Selecting random value for patch ymin coordinate.")
+            ymin = np.random.randint(int(x.shape[-3] - patch_height))
 
         if "xmin" in generate_kwargs:
             xmin = generate_kwargs["xmin"]
         else:
-            raise ValueError("generate_kwargs did not define 'xmin'")
+            logger.info("Selecting random value for patch xmin coordinate.")
+            xmin = np.random.randint(int(x.shape[-2] - patch_width))
 
         assert x.ndim in [
             4,
@@ -36,27 +55,12 @@ def generate(self, x, y=None, **generate_kwargs):
         channels = np.where(channels_mask)[0]
 
         mask = np.zeros(shape=x.shape[1:], dtype=np.float32)
-        if "patch_ratio" in generate_kwargs:
-            patch_ratio = generate_kwargs["patch_ratio"]
-            ymax = ymin + int(x.shape[-3] * patch_ratio ** 0.5)
-            xmax = xmin + int(x.shape[-2] * patch_ratio ** 0.5)
-            if video_input:
-                mask[:, ymin:ymax, xmin:xmax, channels] = 1.0
-            else:
-                mask[ymin:ymax, xmin:xmax, channels] = 1.0
-        elif "patch_height" in generate_kwargs and "patch_width" in generate_kwargs:
-            patch_height = generate_kwargs["patch_height"]
-            patch_width = generate_kwargs["patch_width"]
-            if video_input:
-                mask[
-                    :, ymin : ymin + patch_height, xmin : xmin + patch_width, channels
-                ] = 1.0
-            else:
-                mask[
-                    ymin : ymin + patch_height, xmin : xmin + patch_width, channels
-                ] = 1.0
+
+        if video_input:
+            mask[
+                :, ymin : ymin + patch_height, xmin : xmin + patch_width, channels
+            ] = 1.0
         else:
-            raise ValueError(
-                "generate_kwargs did not define 'patch_ratio', or it did not define 'patch_height' and 'patch_width'"
-            )
+            mask[ymin : ymin + patch_height, xmin : xmin + patch_width, channels] = 1.0
+
         return super().generate(x, y=y, mask=mask)

armory/art_experimental/attacks/robust_dpatch.py

@@ -0,0 +1,31 @@
+import logging
+
+from art.attacks.evasion import RobustDPatch
+import numpy as np
+
+
+logger = logging.getLogger(__name__)
+
+
+class RobustDPatch(RobustDPatch):
+    """
+    Generate and apply patch
+    """
+
+    def __init__(self, estimator, **kwargs):
+        # allows for random patch location
+        if "patch_location" not in kwargs:
+            self.random_location = True
+        else:
+            self.random_location = False
+        super().__init__(estimator=estimator, **kwargs)
+
+    def generate(self, x, y=None, **generate_kwargs):
+        if self.random_location:
+            logger.info("Selecting random coordinates for patch_location.")
+            self.patch_location = (
+                np.random.randint(int(x.shape[-3] - self.patch_shape[0])),
+                np.random.randint(int(x.shape[-2] - self.patch_shape[1])),
+            )
+        super().generate(x, y=y, **generate_kwargs)
+        return super().apply_patch(x)

armory/art_experimental/attacks/video_frame_border.py

@@ -0,0 +1,42 @@
+from art.attacks.evasion import ProjectedGradientDescent
+import numpy as np
+
+
+class FrameBorderPatch(ProjectedGradientDescent):
+    """
+    Apply Masked PGD to video inputs, where only the
+    video frame is allowed to be perturbed.
+    Each video is assumed to have shape (NFHWC).
+    """
+
+    def __init__(self, estimator, **kwargs):
+        super().__init__(estimator=estimator, **kwargs)
+
+    def generate(self, x, y=None, patch_ratio=None, **kwargs):
+        if patch_ratio is None:
+            raise ValueError("generate_kwargs did not define 'patch_ratio'")
+        if x.ndim != 5:
+            raise ValueError("This attack is designed for videos (5-dim)")
+        width = x.shape[3]
+        height = x.shape[2]
+
+        t1 = (
+            2 * (width + height)
+            + (4 * (width + height) ** 2 - 16 * (patch_ratio * width * height)) ** 0.5
+        ) / 8
+        t2 = (
+            2 * (width + height)
+            - (4 * (width + height) ** 2 - 16 * (patch_ratio * width * height)) ** 0.5
+        ) / 8
+        thickness = int(min(t1, t2))
+
+        if (width - 2 * thickness) * (height - 2 * thickness) < (
+            1 - patch_ratio
+        ) * width * height:
+            raise ValueError("patch_ratio does not match height and width")
+
+        mask = np.ones(shape=x.shape[1:], dtype=np.float32)
+
+        mask[:, thickness : height - thickness, thickness : width - thickness, :] = 0.0
+
+        return super().generate(x, y=y, mask=mask, **kwargs)

armory/art_experimental/defences/video_compression_normalized.py

@@ -1,4 +1,5 @@
 from art.defences.preprocessor import VideoCompression
+from art.defences.preprocessor import VideoCompressionPyTorch
 import numpy as np
 
 
@@ -38,3 +39,46 @@ def __call__(self, x, y=None):
         x /= scale
         x = x.astype(self.dtype)
         return x, y
+
+
+class VideoCompressionNormalizedPytorch(VideoCompressionPyTorch):
+    """
+    Convert x from [0,1] to [0, 255] and back, if necessary
+    """
+
+    def __init__(
+        self,
+        video_format,
+        constant_rate_factor=28,
+        channels_first=False,
+        apply_fit=False,
+        apply_predict=True,
+        verbose=False,
+        dtype=np.float32,
+    ):
+        super().__init__(
+            video_format=video_format,
+            constant_rate_factor=constant_rate_factor,
+            channels_first=channels_first,
+            apply_fit=apply_fit,
+            apply_predict=apply_predict,
+            verbose=verbose,
+        )
+        self.dtype = dtype
+
+    def forward(self, x, y=None):
+        """
+        Apply video compression to sample `x`.
+
+        :param x: Sample to compress of shape NCFHW or NFHWC. `x` values are expected to be in the data range [0, 255].
+        :param y: Labels of the sample `x`. This function does not affect them in any way.
+        :return: Compressed sample.
+        """
+        scale = 1
+        if x.min() >= 0 and x.max() <= 1.0:
+            scale = 255
+
+        x = x * scale
+        x_compressed = self._compression_pytorch_numpy.apply(x)
+        x_compressed = x_compressed / scale
+        return x_compressed, y

armory/data/adversarial_datasets.py

@@ -329,6 +329,7 @@ def apricot_label_preprocessing(x, y):
         y = [y]
     for y_dict in y:
         y_dict["labels"] -= y_dict["labels"] != ADV_PATCH_MAGIC_NUMBER_LABEL_ID
+        y_dict["labels"] = y_dict["labels"].reshape((-1,))
     return y
 
 

armory/data/datasets.py

@@ -1338,6 +1338,7 @@ def xview_label_preprocessing(x, y):
         height, width = x[i].shape[:2]
         converted_boxes *= [width, height, width, height]
         label_dict["boxes"] = converted_boxes
+        label_dict["labels"] = label_dict["labels"].reshape((-1,))
         y_preprocessed.append(label_dict)
     return y_preprocessed
 

docker/pytorch-deepspeech/Dockerfile

@@ -28,7 +28,10 @@ RUN /opt/conda/bin/pip install --no-cache-dir \
     pydub==0.24.1 \
     apache-beam==2.22.0 \
     dill==0.3.1.1 \
-    pytest==6.2.2 
+    pytest==6.2.2 \
+    pandas==1.2.4 \
+    ffmpeg-python==0.2.0
+
 
 RUN /opt/conda/bin/conda install -c conda-forge ffmpeg==4.2.3 && \
     /opt/conda/bin/conda clean --all
@@ -45,22 +48,22 @@ ENV NUMBA_CACHE_DIR /tmp
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/lib64"
 
 RUN /opt/conda/bin/conda install pytorch==1.6 \
-  torchvision==0.7.0 \
-  cudatoolkit=10.1 -c pytorch && \
+    torchvision==0.7.0 \
+    cudatoolkit=10.1 -c pytorch && \
     /opt/conda/bin/conda clean --all
 
 
-RUN /opt/conda/bin/conda install -c conda-forge librosa 
+RUN /opt/conda/bin/conda install -c conda-forge librosa
 RUN /opt/conda/bin/pip install hydra-core==1.0.6 \
-  tensorflow-gpu==2.4.1 \
-  python-levenshtein \
-  torchaudio==0.6.0 \
-  numba==0.52.0 \
-  --ignore-installed llvmlite==0.35.0 \
-  soundfile \
-  sox \
-  warpctc-pytorch==0.2.1+torch16.cuda102 \
-  adversarial-robustness-toolbox==1.6.1 --no-cache-dir
+    tensorflow-gpu==2.4.1 \
+    python-levenshtein \
+    torchaudio==0.6.0 \
+    numba==0.52.0 \
+    --ignore-installed llvmlite==0.35.0 \
+    soundfile \
+    sox \
+    warpctc-pytorch==0.2.1+torch16.cuda102 \
+    adversarial-robustness-toolbox==1.6.2 --no-cache-dir
 
 
 ########## PyTorch 1 Deep Speech Dev #################
@@ -69,7 +72,7 @@ FROM armory-pytorch-deepspeech-base AS armory-pytorch-deepspeech-dev
 ARG armory_version
 
 COPY . /armory_dev/
-RUN /opt/conda/bin/pip install /armory_dev/ --no-cache-dir 
+RUN /opt/conda/bin/pip install /armory_dev/ --no-cache-dir
 
 WORKDIR /workspace
 CMD tail -f /dev/null

docker/pytorch/Dockerfile

@@ -29,7 +29,9 @@ RUN /opt/conda/bin/pip install --no-cache-dir \
     apache-beam==2.22.0 \
     dill==0.3.1.1 \
     pytest==6.2.2 \
-    opencv-python==4.5.1.48
+    opencv-python==4.5.1.48 \
+    pandas==1.2.4 \
+    ffmpeg-python==0.2.0
 
 RUN /opt/conda/bin/conda install -c conda-forge ffmpeg==4.2.3 && \
     /opt/conda/bin/conda clean --all
@@ -42,20 +44,20 @@ ARG armory_version
 FROM armory AS armory-pytorch-base
 
 RUN /opt/conda/bin/conda install pytorch==1.7 \
-   torchvision==0.8.0 \ 
-   cudatoolkit=11.0 -c pytorch -c conda-forge && \
-   /opt/conda/bin/conda clean --all
+    torchvision==0.8.0 \
+    cudatoolkit=11.0 -c pytorch -c conda-forge && \
+    /opt/conda/bin/conda clean --all
 RUN /opt/conda/bin/pip install --no-cache-dir \
-   tensorflow-gpu==2.4.1 \
-   adversarial-robustness-toolbox==1.6.1
+    tensorflow-gpu==2.4.1 \
+    adversarial-robustness-toolbox==1.6.2
 
 ########## PyTorch 1 Dev #################
 
 FROM armory-pytorch-base AS armory-pytorch-dev
 ARG armory_version
 
 COPY . /armory_dev/
-RUN /opt/conda/bin/pip install /armory_dev/ --no-cache-dir 
+RUN /opt/conda/bin/pip install /armory_dev/ --no-cache-dir
 
 WORKDIR /workspace
 CMD tail -f /dev/null

docker/tf1/Dockerfile

@@ -29,7 +29,11 @@ RUN /opt/conda/bin/pip install --no-cache-dir \
     apache-beam==2.22.0 \
     dill==0.3.1.1 \
     pytest==6.2.2 \
-    opencv-python==4.5.1.48
+    opencv-python==4.5.1.48 \
+    pandas==1.2.4 \
+    ffmpeg-python==0.2.0
+
+
 
 RUN /opt/conda/bin/conda install -c conda-forge ffmpeg==4.2.3 && \
     /opt/conda/bin/conda clean --all
@@ -49,7 +53,7 @@ WORKDIR /tmp/models/research
 RUN protoc object_detection/protos/*.proto --python_out=.
 RUN cp object_detection/packages/tf1/setup.py .
 RUN /opt/conda/bin/pip install .
-RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.1
+RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.2
 
 # Note: this is necessary until the following TF issue is resolved: https://github.com/tensorflow/models/issues/9706
 RUN /opt/conda/bin/pip install --no-cache-dir numpy==1.19.2

docker/tf2/Dockerfile

@@ -29,7 +29,10 @@ RUN /opt/conda/bin/pip install --no-cache-dir \
     apache-beam==2.22.0 \
     dill==0.3.1.1 \
     pytest==6.2.2 \
-    opencv-python==4.5.1.48
+    opencv-python==4.5.1.48 \
+    pandas==1.2.4 \
+    ffmpeg-python==0.2.0
+
 
 RUN /opt/conda/bin/conda install -c conda-forge ffmpeg==4.2.3 && \
     /opt/conda/bin/conda clean --all
@@ -50,7 +53,7 @@ WORKDIR /tmp/models/research
 RUN protoc object_detection/protos/*.proto --python_out=.
 RUN cp object_detection/packages/tf2/setup.py .
 RUN /opt/conda/bin/pip install .
-RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.1
+RUN /opt/conda/bin/pip install --no-cache-dir adversarial-robustness-toolbox==1.6.2
 
 WORKDIR /workspace