From 8952859f5f26779169755e0fd7ddd4eff0b0e75d Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Tue, 21 Mar 2023 11:49:26 +0000 Subject: [PATCH 1/9] Add copied from statements for image processors --- .../models/beit/image_processing_beit.py | 2 +- .../models/bit/image_processing_bit.py | 2 +- .../models/blip/image_processing_blip.py | 60 +++++++++++++++++- .../image_processing_bridgetower.py | 62 ++++++++++++++++--- .../image_processing_chinese_clip.py | 2 +- .../models/clip/image_processing_clip.py | 2 +- .../image_processing_conditional_detr.py | 30 +++++++++ .../convnext/image_processing_convnext.py | 55 +++++++++++++++- .../image_processing_deformable_detr.py | 30 +++++++++ .../models/deit/image_processing_deit.py | 7 ++- .../models/deta/image_processing_deta.py | 30 +++++++++ .../models/detr/image_processing_detr.py | 30 +++++++++ .../models/donut/image_processing_donut.py | 55 +++++++++++++++- .../models/dpt/image_processing_dpt.py | 53 ++++++++++++++++ .../image_processing_efficientformer.py | 2 +- .../image_processing_efficientnet.py | 9 +-- .../models/flava/image_processing_flava.py | 5 +- .../models/glpn/image_processing_glpn.py | 23 +++++++ .../imagegpt/image_processing_imagegpt.py | 5 +- .../layoutlmv2/image_processing_layoutlmv2.py | 5 +- .../layoutlmv3/image_processing_layoutlmv3.py | 58 ++++++++++++++++- .../image_processing_mask2former.py | 48 +++++++++++--- .../maskformer/image_processing_maskformer.py | 45 ++++++++++++-- .../image_processing_mobilenet_v1.py | 2 +- .../image_processing_mobilenet_v2.py | 2 +- .../oneformer/image_processing_oneformer.py | 30 +++++++++ .../models/owlvit/image_processing_owlvit.py | 32 +++++++++- .../perceiver/image_processing_perceiver.py | 60 +++++++++++++++++- .../poolformer/image_processing_poolformer.py | 2 +- .../models/pvt/image_processing_pvt.py | 55 +++++++++++++++- .../models/sam/image_processing_sam.py | 55 +++++++++++++++- .../segformer/image_processing_segformer.py | 55 +++++++++++++++- .../swin2sr/image_processing_swin2sr.py | 23 +++++++ .../models/tvlt/image_processing_tvlt.py | 2 +- .../videomae/image_processing_videomae.py | 2 +- .../models/vilt/image_processing_vilt.py | 62 ++++++++++++++++--- .../models/vit/image_processing_vit.py | 53 +++++++++++++++- .../vit_hybrid/image_processing_vit_hybrid.py | 2 +- .../models/vivit/image_processing_vivit.py | 32 +++++++++- .../models/yolos/image_processing_yolos.py | 33 +++++++++- 40 files changed, 1054 insertions(+), 68 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index 08da81113dc7aa..2346bb77e748e8 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -15,7 +15,7 @@ """Image processor class for Beit.""" import warnings -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import numpy as np diff --git a/src/transformers/models/bit/image_processing_bit.py b/src/transformers/models/bit/image_processing_bit.py index 65c7a0c946a824..48bebc2ff525c1 100644 --- a/src/transformers/models/bit/image_processing_bit.py +++ b/src/transformers/models/bit/image_processing_bit.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BiT.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 23b008f8d72b2b..1a40e3f9a21794 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BLIP.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -135,8 +135,62 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 1ae7868f47f3b3..27cd1c8d6c4893 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -226,6 +226,29 @@ def resize( output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + # Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + def center_crop( self, image: np.ndarray, @@ -248,6 +271,36 @@ def center_crop( output_size = size["shortest_edge"] return center_crop(image, size=(output_size, output_size), data_format=data_format, **kwargs) + # Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor._pad_image def _pad_image( self, @@ -290,13 +343,8 @@ def pad( The value to use for the padding if `mode` is `"constant"`. return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether to return a pixel mask. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + input_channel_dimension (`ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be inferred from the input image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. """ diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py index 974f2ea075a7b4..b49cb811a38bae 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Chinese-CLIP.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/clip/image_processing_clip.py b/src/transformers/models/clip/image_processing_clip.py index 34a1f6bce3c4c5..42b593d5bc5c52 100644 --- a/src/transformers/models/clip/image_processing_clip.py +++ b/src/transformers/models/clip/image_processing_clip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for CLIP.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index 465f224cc2fb0c..67cd2aaa52954a 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -953,6 +953,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index cfd5b38d76b9b5..53178251727235 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for ConvNeXT.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -156,6 +156,59 @@ def resize( image, size=(shortest_edge, shortest_edge), resample=resample, data_format=data_format, **kwargs ) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index 39d31fd62ec457..b9b28302209d0d 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -951,6 +951,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index 144984523e3cf9..6a6924ccd117ed 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for DeiT.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -138,8 +138,9 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) def preprocess( self, diff --git a/src/transformers/models/deta/image_processing_deta.py b/src/transformers/models/deta/image_processing_deta.py index 4ae35b101d66f5..d68795b25c3e05 100644 --- a/src/transformers/models/deta/image_processing_deta.py +++ b/src/transformers/models/deta/image_processing_deta.py @@ -624,6 +624,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 794f0f4bd4e9a9..58107e66efdefb 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -927,6 +927,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ Normalize the boxes in the annotation from `[top_left_x, top_left_y, bottom_right_x, bottom_right_y]` to diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index db72f7f3265887..d0d62fbb4646c3 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Donut.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -261,6 +261,59 @@ def resize( resized_image = resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) return resized_image + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index 417f5b3c842a64..d630ed9d047b63 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -191,6 +191,59 @@ def resize( ) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/efficientformer/image_processing_efficientformer.py b/src/transformers/models/efficientformer/image_processing_efficientformer.py index ce552c6dd0a737..228169856f1513 100644 --- a/src/transformers/models/efficientformer/image_processing_efficientformer.py +++ b/src/transformers/models/efficientformer/image_processing_efficientformer.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for EfficientFormer.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index a9cab52491a2e2..a03e9774200b4f 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for EfficientNet.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -147,8 +147,9 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) def rescale( self, @@ -173,7 +174,7 @@ def rescale( Image to rescale. scale (`int` or `float`): Scale to apply to the image. - offset (`bool`, *optional*): + offset (`bool`, *optional*): Whether to scale the image in both negative and positive directions. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. diff --git a/src/transformers/models/flava/image_processing_flava.py b/src/transformers/models/flava/image_processing_flava.py index 2ece1bd2b32c59..5f864c07948652 100644 --- a/src/transformers/models/flava/image_processing_flava.py +++ b/src/transformers/models/flava/image_processing_flava.py @@ -362,8 +362,9 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) def map_pixels(self, image: np.ndarray) -> np.ndarray: return (1 - 2 * LOGIT_LAPLACE_EPS) * image + LOGIT_LAPLACE_EPS diff --git a/src/transformers/models/glpn/image_processing_glpn.py b/src/transformers/models/glpn/image_processing_glpn.py index 3e37d0bab221e7..de2023fbea90c9 100644 --- a/src/transformers/models/glpn/image_processing_glpn.py +++ b/src/transformers/models/glpn/image_processing_glpn.py @@ -106,6 +106,29 @@ def resize( image = resize(image, (new_h, new_w), resample=resample, data_format=data_format, **kwargs) return image + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + def preprocess( self, images: Union["PIL.Image.Image", TensorType, List["PIL.Image.Image"], List[TensorType]], diff --git a/src/transformers/models/imagegpt/image_processing_imagegpt.py b/src/transformers/models/imagegpt/image_processing_imagegpt.py index c9478c3bbd4cdd..22c0ed83b803d5 100644 --- a/src/transformers/models/imagegpt/image_processing_imagegpt.py +++ b/src/transformers/models/imagegpt/image_processing_imagegpt.py @@ -131,8 +131,9 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) def normalize( self, diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index ab70a015d8a79c..459e1bf78390a1 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -162,8 +162,9 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) def preprocess( self, diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index 9ca195ce793f07..857ff4fa856d7a 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -188,8 +188,62 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index b9359251925e11..09d6a7615a5da5 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -486,25 +486,58 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale + # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs ) -> np.ndarray: """ - Rescale the image by the given factor. image = image * rescale_factor. + Rescale an image by a scale factor. image = image * scale. Args: image (`np.ndarray`): Image to rescale. - rescale_factor (`float`): - The value to use for rescaling. + scale (`float`): + The scaling factor to rescale pixel values by. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format for the output image. If unset, the channel dimension format of the input image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. """ - return rescale(image, rescale_factor, data_format=data_format) + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( @@ -771,6 +804,7 @@ def pad( return BatchFeature(data=data, tensor_type=return_tensors) + # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.encode_inputs with MaskFormer->Mask2Former def encode_inputs( self, pixel_values_list: List[ImageInput], @@ -828,7 +862,7 @@ def encode_inputs( `mask_labels[i][j]` if `class_labels[i][j]`. """ ignore_index = self.ignore_index if ignore_index is None else ignore_index - reduce_labels = self.reduce_labels if reduce_labels is None else reduce_labels + reduce_labels = self.do_reduce_labels if reduce_labels is None else reduce_labels pixel_values_list = [to_numpy_array(pixel_values) for pixel_values in pixel_values_list] encoded_inputs = self.pad(pixel_values_list, return_tensors=return_tensors) diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index b5880566748b5b..8b56210289cc11 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -494,25 +494,58 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs ) -> np.ndarray: """ - Rescale the image by the given factor. image = image * rescale_factor. + Rescale an image by a scale factor. image = image * scale. Args: image (`np.ndarray`): Image to rescale. - rescale_factor (`float`): - The value to use for rescaling. + scale (`float`): + The scaling factor to rescale pixel values by. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format for the output image. If unset, the channel dimension format of the input image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. """ - return rescale(image, rescale_factor, data_format=data_format) + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) def convert_segmentation_map_to_binary_masks( self, diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py index a9f3cc0c38c535..4c4fc8ed85e0cc 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for MobileNetV1.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py index d3edd8236f02df..965b3b2acdd92b 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for MobileNetV2.""" -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Iterable, List, Optional, Tuple, Union import numpy as np diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index cd0bdf08507564..ea94a668a28f02 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -497,6 +497,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( self, diff --git a/src/transformers/models/owlvit/image_processing_owlvit.py b/src/transformers/models/owlvit/image_processing_owlvit.py index c230fda88f266d..c721d25bb3ce74 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit.py +++ b/src/transformers/models/owlvit/image_processing_owlvit.py @@ -15,7 +15,7 @@ """Image processor class for OwlViT""" import warnings -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Iterable, List, Optional, Tuple, Union import numpy as np @@ -216,6 +216,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index de490ab8d81ba3..a51bcacb222fb4 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Perceiver.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -177,8 +177,62 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/poolformer/image_processing_poolformer.py b/src/transformers/models/poolformer/image_processing_poolformer.py index 03774f401870d5..70b19580517f88 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer.py +++ b/src/transformers/models/poolformer/image_processing_poolformer.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for PoolFormer.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/pvt/image_processing_pvt.py b/src/transformers/models/pvt/image_processing_pvt.py index d0a7cc99a45960..cbe5a0a5ce7f03 100644 --- a/src/transformers/models/pvt/image_processing_pvt.py +++ b/src/transformers/models/pvt/image_processing_pvt.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Pvt.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -127,6 +127,59 @@ def resize( output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 2ee0b47250cc51..397eca6eac9284 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -16,7 +16,7 @@ import math from copy import deepcopy from itertools import product -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import numpy as np @@ -212,6 +212,59 @@ def resize( output_height, output_width = self._get_preprocess_shape(input_size, size["longest_edge"]) return resize(image, size=(output_height, output_width), resample=resample, data_format=data_format, **kwargs) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/segformer/image_processing_segformer.py b/src/transformers/models/segformer/image_processing_segformer.py index 4424f5dd646f45..20aa59e184b611 100644 --- a/src/transformers/models/segformer/image_processing_segformer.py +++ b/src/transformers/models/segformer/image_processing_segformer.py @@ -15,7 +15,7 @@ """Image processor class for Segformer.""" import warnings -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import numpy as np @@ -162,6 +162,59 @@ def resize( output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.beit.image_processing_beit.BeitImageProcessor.reduce_label def reduce_label(self, label: ImageInput) -> np.ndarray: label = to_numpy_array(label) diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 0558fd467bf6ee..6db884b1d5126e 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -57,6 +57,29 @@ def __init__( self.do_pad = do_pad self.pad_size = pad_size + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + def pad(self, image: np.ndarray, size: int, data_format: Optional[Union[str, ChannelDimension]] = None): """ Pad an image to make the height and width divisible by `size`. diff --git a/src/transformers/models/tvlt/image_processing_tvlt.py b/src/transformers/models/tvlt/image_processing_tvlt.py index 0b4d928a3e01da..ca2c08897cad7e 100644 --- a/src/transformers/models/tvlt/image_processing_tvlt.py +++ b/src/transformers/models/tvlt/image_processing_tvlt.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Image processor class for TVLT.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index 3992e5c5a79d65..e171c6fa95542b 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for VideoMAE.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index c45c2c6b25a0b7..464e108d2dcff2 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -229,6 +229,59 @@ def resize( output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor._pad_image def _pad_image( self, @@ -271,13 +324,8 @@ def pad( The value to use for the padding if `mode` is `"constant"`. return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether to return a pixel mask. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + input_channel_dimension (`ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be inferred from the input image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. """ diff --git a/src/transformers/models/vit/image_processing_vit.py b/src/transformers/models/vit/image_processing_vit.py index 8004ac32e07db2..1ac24cf7cc93a0 100644 --- a/src/transformers/models/vit/image_processing_vit.py +++ b/src/transformers/models/vit/image_processing_vit.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for ViT.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -126,6 +126,57 @@ def resize( output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py index 26b1e53adaebf9..b54a2a8f2cb3b0 100644 --- a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py +++ b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for ViT hybrid.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np diff --git a/src/transformers/models/vivit/image_processing_vivit.py b/src/transformers/models/vivit/image_processing_vivit.py index 0790c5d82bb5a9..aeca20810b13a6 100644 --- a/src/transformers/models/vivit/image_processing_vivit.py +++ b/src/transformers/models/vivit/image_processing_vivit.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Image processor class for Vivit.""" -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import numpy as np @@ -203,6 +203,36 @@ def rescale( return rescaled_image + # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + def _preprocess_image( self, image: ImageInput, diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index 3a42c653bd432a..44a7a8162662f3 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -862,6 +862,36 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ @@ -892,11 +922,12 @@ def _pad_image( ) return padded_image + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.pad def pad( self, images: List[np.ndarray], constant_values: Union[float, Iterable[float]] = 0, - return_pixel_mask: bool = False, + return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, ) -> BatchFeature: From 0e28ec04125a1e4b00080a8ace9b0855cc8ceacb Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Tue, 25 Jul 2023 17:04:34 +0100 Subject: [PATCH 2/9] Move out rescale and normalize to base image processor --- src/transformers/image_processing_utils.py | 43 +++++++-------- .../models/beit/image_processing_beit.py | 2 +- .../models/bit/image_processing_bit.py | 2 +- .../models/blip/image_processing_blip.py | 55 +------------------ .../image_processing_bridgetower.py | 53 ------------------ .../image_processing_chinese_clip.py | 2 +- .../models/clip/image_processing_clip.py | 2 +- .../image_processing_conditional_detr.py | 30 ---------- .../convnext/image_processing_convnext.py | 55 +------------------ .../image_processing_deformable_detr.py | 30 ---------- .../models/deit/image_processing_deit.py | 2 +- .../models/deta/image_processing_deta.py | 30 ---------- .../models/detr/image_processing_detr.py | 30 ---------- .../models/donut/image_processing_donut.py | 55 +------------------ .../models/dpt/image_processing_dpt.py | 53 ------------------ .../image_processing_efficientformer.py | 2 +- .../image_processing_efficientnet.py | 2 +- .../models/glpn/image_processing_glpn.py | 23 -------- .../layoutlmv3/image_processing_layoutlmv3.py | 53 ------------------ .../image_processing_mask2former.py | 54 ------------------ .../maskformer/image_processing_maskformer.py | 54 ------------------ .../image_processing_mobilenet_v1.py | 2 +- .../image_processing_mobilenet_v2.py | 2 +- .../oneformer/image_processing_oneformer.py | 30 ---------- .../models/owlvit/image_processing_owlvit.py | 32 +---------- .../perceiver/image_processing_perceiver.py | 55 +------------------ .../poolformer/image_processing_poolformer.py | 2 +- .../models/pvt/image_processing_pvt.py | 55 +------------------ .../models/sam/image_processing_sam.py | 55 +------------------ .../segformer/image_processing_segformer.py | 55 +------------------ .../swin2sr/image_processing_swin2sr.py | 23 -------- .../models/tvlt/image_processing_tvlt.py | 2 +- .../videomae/image_processing_videomae.py | 2 +- .../models/vilt/image_processing_vilt.py | 53 ------------------ .../vit_hybrid/image_processing_vit_hybrid.py | 2 +- .../models/vivit/image_processing_vivit.py | 32 +---------- .../models/yolos/image_processing_yolos.py | 30 ---------- 37 files changed, 43 insertions(+), 1021 deletions(-) diff --git a/src/transformers/image_processing_utils.py b/src/transformers/image_processing_utils.py index 07885890918369..9fba216b8287ab 100644 --- a/src/transformers/image_processing_utils.py +++ b/src/transformers/image_processing_utils.py @@ -520,28 +520,6 @@ def __call__(self, images, **kwargs) -> BatchFeature: def preprocess(self, images, **kwargs) -> BatchFeature: raise NotImplementedError("Each image processor must implement its own preprocess method") - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - def normalize( self, image: np.ndarray, @@ -594,6 +572,27 @@ def center_crop( if "height" not in size or "width" not in size: raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}") return center_crop(image, size=(size["height"], size["width"]), data_format=data_format, **kwargs) + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) VALID_SIZE_DICT_KEYS = ({"height", "width"}, {"shortest_edge"}, {"shortest_edge", "longest_edge"}, {"longest_edge"}) diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index 2346bb77e748e8..08da81113dc7aa 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -15,7 +15,7 @@ """Image processor class for Beit.""" import warnings -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np diff --git a/src/transformers/models/bit/image_processing_bit.py b/src/transformers/models/bit/image_processing_bit.py index 48bebc2ff525c1..65c7a0c946a824 100644 --- a/src/transformers/models/bit/image_processing_bit.py +++ b/src/transformers/models/bit/image_processing_bit.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BiT.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 1a40e3f9a21794..1a0525dfdea5ad 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BLIP.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -139,59 +139,6 @@ def resize( image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs ) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 27cd1c8d6c4893..9f4d069e8b6709 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -226,29 +226,6 @@ def resize( output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) - # Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - def center_crop( self, image: np.ndarray, @@ -271,36 +248,6 @@ def center_crop( output_size = size["shortest_edge"] return center_crop(image, size=(output_size, output_size), data_format=data_format, **kwargs) - # Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor._pad_image def _pad_image( self, diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py index b49cb811a38bae..974f2ea075a7b4 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Chinese-CLIP.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/clip/image_processing_clip.py b/src/transformers/models/clip/image_processing_clip.py index 42b593d5bc5c52..34a1f6bce3c4c5 100644 --- a/src/transformers/models/clip/image_processing_clip.py +++ b/src/transformers/models/clip/image_processing_clip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for CLIP.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index 67cd2aaa52954a..465f224cc2fb0c 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -953,36 +953,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index 53178251727235..cfd5b38d76b9b5 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for ConvNeXT.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -156,59 +156,6 @@ def resize( image, size=(shortest_edge, shortest_edge), resample=resample, data_format=data_format, **kwargs ) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index b9b28302209d0d..39d31fd62ec457 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -951,36 +951,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index 6a6924ccd117ed..8f25bcd8ca5f35 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for DeiT.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/deta/image_processing_deta.py b/src/transformers/models/deta/image_processing_deta.py index d68795b25c3e05..4ae35b101d66f5 100644 --- a/src/transformers/models/deta/image_processing_deta.py +++ b/src/transformers/models/deta/image_processing_deta.py @@ -624,36 +624,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 58107e66efdefb..794f0f4bd4e9a9 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -927,36 +927,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ Normalize the boxes in the annotation from `[top_left_x, top_left_y, bottom_right_x, bottom_right_y]` to diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index d0d62fbb4646c3..db72f7f3265887 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Donut.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -261,59 +261,6 @@ def resize( resized_image = resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) return resized_image - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index d630ed9d047b63..417f5b3c842a64 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -191,59 +191,6 @@ def resize( ) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/efficientformer/image_processing_efficientformer.py b/src/transformers/models/efficientformer/image_processing_efficientformer.py index 228169856f1513..ce552c6dd0a737 100644 --- a/src/transformers/models/efficientformer/image_processing_efficientformer.py +++ b/src/transformers/models/efficientformer/image_processing_efficientformer.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for EfficientFormer.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index a03e9774200b4f..15616420e17088 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for EfficientNet.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/glpn/image_processing_glpn.py b/src/transformers/models/glpn/image_processing_glpn.py index de2023fbea90c9..3e37d0bab221e7 100644 --- a/src/transformers/models/glpn/image_processing_glpn.py +++ b/src/transformers/models/glpn/image_processing_glpn.py @@ -106,29 +106,6 @@ def resize( image = resize(image, (new_h, new_w), resample=resample, data_format=data_format, **kwargs) return image - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - def preprocess( self, images: Union["PIL.Image.Image", TensorType, List["PIL.Image.Image"], List[TensorType]], diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index 857ff4fa856d7a..f53bf956310ec2 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -192,59 +192,6 @@ def resize( image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs ) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index 09d6a7615a5da5..ba13308c47bf5c 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -25,7 +25,6 @@ PaddingMode, get_resize_output_image_size, pad, - rescale, resize, to_channel_dimension_format, ) @@ -486,59 +485,6 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image - # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( self, diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index 8b56210289cc11..ab0561012899b8 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -25,7 +25,6 @@ PaddingMode, get_resize_output_image_size, pad, - rescale, resize, to_channel_dimension_format, ) @@ -494,59 +493,6 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def convert_segmentation_map_to_binary_masks( self, segmentation_map: "np.ndarray", diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py index 4c4fc8ed85e0cc..a9f3cc0c38c535 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for MobileNetV1.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py index 965b3b2acdd92b..d3edd8236f02df 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for MobileNetV2.""" -from typing import Dict, Iterable, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import numpy as np diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index ea94a668a28f02..cd0bdf08507564 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -497,36 +497,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( self, diff --git a/src/transformers/models/owlvit/image_processing_owlvit.py b/src/transformers/models/owlvit/image_processing_owlvit.py index c721d25bb3ce74..c230fda88f266d 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit.py +++ b/src/transformers/models/owlvit/image_processing_owlvit.py @@ -15,7 +15,7 @@ """Image processor class for OwlViT""" import warnings -from typing import Dict, Iterable, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import numpy as np @@ -216,36 +216,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index a51bcacb222fb4..0ad01bcc85f2b8 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Perceiver.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -181,59 +181,6 @@ def resize( image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs ) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/poolformer/image_processing_poolformer.py b/src/transformers/models/poolformer/image_processing_poolformer.py index 70b19580517f88..03774f401870d5 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer.py +++ b/src/transformers/models/poolformer/image_processing_poolformer.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for PoolFormer.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/pvt/image_processing_pvt.py b/src/transformers/models/pvt/image_processing_pvt.py index cbe5a0a5ce7f03..d0a7cc99a45960 100644 --- a/src/transformers/models/pvt/image_processing_pvt.py +++ b/src/transformers/models/pvt/image_processing_pvt.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for Pvt.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -127,59 +127,6 @@ def resize( output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 397eca6eac9284..2ee0b47250cc51 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -16,7 +16,7 @@ import math from copy import deepcopy from itertools import product -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np @@ -212,59 +212,6 @@ def resize( output_height, output_width = self._get_preprocess_shape(input_size, size["longest_edge"]) return resize(image, size=(output_height, output_width), resample=resample, data_format=data_format, **kwargs) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/segformer/image_processing_segformer.py b/src/transformers/models/segformer/image_processing_segformer.py index 20aa59e184b611..4424f5dd646f45 100644 --- a/src/transformers/models/segformer/image_processing_segformer.py +++ b/src/transformers/models/segformer/image_processing_segformer.py @@ -15,7 +15,7 @@ """Image processor class for Segformer.""" import warnings -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np @@ -162,59 +162,6 @@ def resize( output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.beit.image_processing_beit.BeitImageProcessor.reduce_label def reduce_label(self, label: ImageInput) -> np.ndarray: label = to_numpy_array(label) diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 6db884b1d5126e..0558fd467bf6ee 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -57,29 +57,6 @@ def __init__( self.do_pad = do_pad self.pad_size = pad_size - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - def pad(self, image: np.ndarray, size: int, data_format: Optional[Union[str, ChannelDimension]] = None): """ Pad an image to make the height and width divisible by `size`. diff --git a/src/transformers/models/tvlt/image_processing_tvlt.py b/src/transformers/models/tvlt/image_processing_tvlt.py index ca2c08897cad7e..0b4d928a3e01da 100644 --- a/src/transformers/models/tvlt/image_processing_tvlt.py +++ b/src/transformers/models/tvlt/image_processing_tvlt.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Image processor class for TVLT.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index e171c6fa95542b..3992e5c5a79d65 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for VideoMAE.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index 464e108d2dcff2..17e87174c41d1e 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -229,59 +229,6 @@ def resize( output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.rescale - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor._pad_image def _pad_image( self, diff --git a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py index b54a2a8f2cb3b0..26b1e53adaebf9 100644 --- a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py +++ b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for ViT hybrid.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np diff --git a/src/transformers/models/vivit/image_processing_vivit.py b/src/transformers/models/vivit/image_processing_vivit.py index aeca20810b13a6..0790c5d82bb5a9 100644 --- a/src/transformers/models/vivit/image_processing_vivit.py +++ b/src/transformers/models/vivit/image_processing_vivit.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Image processor class for Vivit.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -203,36 +203,6 @@ def rescale( return rescaled_image - # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def _preprocess_image( self, image: ImageInput, diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index 44a7a8162662f3..209b0d441d8459 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -862,36 +862,6 @@ def rescale( """ return rescale(image, rescale_factor, data_format=data_format) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ From 57179a5f5b05bf652c78723cb86e378944bc17a9 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:21:32 +0100 Subject: [PATCH 3/9] Remove rescale and normalize from vit (post rebase) --- src/transformers/image_processing_utils.py | 43 +++++++-------- .../maskformer/image_processing_maskformer.py | 10 ++++ .../models/vit/image_processing_vit.py | 53 +------------------ 3 files changed, 33 insertions(+), 73 deletions(-) diff --git a/src/transformers/image_processing_utils.py b/src/transformers/image_processing_utils.py index 9fba216b8287ab..07885890918369 100644 --- a/src/transformers/image_processing_utils.py +++ b/src/transformers/image_processing_utils.py @@ -520,6 +520,28 @@ def __call__(self, images, **kwargs) -> BatchFeature: def preprocess(self, images, **kwargs) -> BatchFeature: raise NotImplementedError("Each image processor must implement its own preprocess method") + def rescale( + self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + def normalize( self, image: np.ndarray, @@ -572,27 +594,6 @@ def center_crop( if "height" not in size or "width" not in size: raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}") return center_crop(image, size=(size["height"], size["width"]), data_format=data_format, **kwargs) - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) VALID_SIZE_DICT_KEYS = ({"height", "width"}, {"shortest_edge"}, {"shortest_edge", "longest_edge"}, {"longest_edge"}) diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index ab0561012899b8..32ad8f81228057 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -25,6 +25,7 @@ PaddingMode, get_resize_output_image_size, pad, + rescale, resize, to_channel_dimension_format, ) @@ -493,6 +494,15 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image + # Copied from transformers.models.detr.image_processing_detr.rescale + def rescale( + self, image: np.ndarray, rescale_factor: float, data_format: Optional[ChannelDimension] = None + ) -> np.ndarray: + """ + Rescale the image by the given factor. + """ + return rescale(image, rescale_factor, data_format=data_format) + def convert_segmentation_map_to_binary_masks( self, segmentation_map: "np.ndarray", diff --git a/src/transformers/models/vit/image_processing_vit.py b/src/transformers/models/vit/image_processing_vit.py index 1ac24cf7cc93a0..8004ac32e07db2 100644 --- a/src/transformers/models/vit/image_processing_vit.py +++ b/src/transformers/models/vit/image_processing_vit.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for ViT.""" -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np @@ -126,57 +126,6 @@ def resize( output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) - def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs - ) -> np.ndarray: - """ - Rescale an image by a scale factor. image = image * scale. - - Args: - image (`np.ndarray`): - Image to rescale. - scale (`float`): - The scaling factor to rescale pixel values by. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The rescaled image. - """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) - - def normalize( - self, - image: np.ndarray, - mean: Union[float, Iterable[float]], - std: Union[float, Iterable[float]], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs, - ) -> np.ndarray: - """ - Normalize an image. image = (image - image_mean) / image_std. - - Args: - image (`np.ndarray`): - Image to normalize. - mean (`float` or `Iterable[float]`): - Image mean to use for normalization. - std (`float` or `Iterable[float]`): - Image standard deviation to use for normalization. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Returns: - `np.ndarray`: The normalized image. - """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, From 8857882ad7b59f1a035ae028acfbae3316d72656 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:40:35 +0100 Subject: [PATCH 4/9] Update docstrings and tidy up --- .../image_processing_bridgetower.py | 9 +++++-- .../image_processing_mask2former.py | 24 +++++++++++++++++-- .../maskformer/image_processing_maskformer.py | 17 ++++++++++--- .../models/vilt/image_processing_vilt.py | 9 +++++-- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 9f4d069e8b6709..1ae7868f47f3b3 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -290,8 +290,13 @@ def pad( The value to use for the padding if `mode` is `"constant"`. return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether to return a pixel mask. - input_channel_dimension (`ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be inferred from the input image. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. """ diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index ba13308c47bf5c..b9359251925e11 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -25,6 +25,7 @@ PaddingMode, get_resize_output_image_size, pad, + rescale, resize, to_channel_dimension_format, ) @@ -485,6 +486,26 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale + def rescale( + self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + ) -> np.ndarray: + """ + Rescale the image by the given factor. image = image * rescale_factor. + + Args: + image (`np.ndarray`): + Image to rescale. + rescale_factor (`float`): + The value to use for rescaling. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + """ + return rescale(image, rescale_factor, data_format=data_format) + # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( self, @@ -750,7 +771,6 @@ def pad( return BatchFeature(data=data, tensor_type=return_tensors) - # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.encode_inputs with MaskFormer->Mask2Former def encode_inputs( self, pixel_values_list: List[ImageInput], @@ -808,7 +828,7 @@ def encode_inputs( `mask_labels[i][j]` if `class_labels[i][j]`. """ ignore_index = self.ignore_index if ignore_index is None else ignore_index - reduce_labels = self.do_reduce_labels if reduce_labels is None else reduce_labels + reduce_labels = self.reduce_labels if reduce_labels is None else reduce_labels pixel_values_list = [to_numpy_array(pixel_values) for pixel_values in pixel_values_list] encoded_inputs = self.pad(pixel_values_list, return_tensors=return_tensors) diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index 32ad8f81228057..b5880566748b5b 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -494,12 +494,23 @@ def resize( image = resize(image, size=size, resample=resample, data_format=data_format) return image - # Copied from transformers.models.detr.image_processing_detr.rescale + # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[ChannelDimension] = None + self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None ) -> np.ndarray: """ - Rescale the image by the given factor. + Rescale the image by the given factor. image = image * rescale_factor. + + Args: + image (`np.ndarray`): + Image to rescale. + rescale_factor (`float`): + The value to use for rescaling. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ return rescale(image, rescale_factor, data_format=data_format) diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index 17e87174c41d1e..c45c2c6b25a0b7 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -271,8 +271,13 @@ def pad( The value to use for the padding if `mode` is `"constant"`. return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether to return a pixel mask. - input_channel_dimension (`ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be inferred from the input image. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. """ From c4c6125b39104a122ed24e84cb4be461496a059d Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 27 Jul 2023 21:21:18 +0100 Subject: [PATCH 5/9] PR comments --- src/transformers/models/blip/image_processing_blip.py | 5 ++--- src/transformers/models/deit/image_processing_deit.py | 5 ++--- .../models/efficientnet/image_processing_efficientnet.py | 7 +++---- src/transformers/models/flava/image_processing_flava.py | 5 ++--- .../models/imagegpt/image_processing_imagegpt.py | 5 ++--- .../models/layoutlmv2/image_processing_layoutlmv2.py | 5 ++--- .../models/layoutlmv3/image_processing_layoutlmv3.py | 5 ++--- .../models/perceiver/image_processing_perceiver.py | 5 ++--- 8 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 1a0525dfdea5ad..23b008f8d72b2b 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -135,9 +135,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index 8f25bcd8ca5f35..144984523e3cf9 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -138,9 +138,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index 15616420e17088..a9cab52491a2e2 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -147,9 +147,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def rescale( self, @@ -174,7 +173,7 @@ def rescale( Image to rescale. scale (`int` or `float`): Scale to apply to the image. - offset (`bool`, *optional*): + offset (`bool`, *optional*): Whether to scale the image in both negative and positive directions. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. diff --git a/src/transformers/models/flava/image_processing_flava.py b/src/transformers/models/flava/image_processing_flava.py index 5f864c07948652..2ece1bd2b32c59 100644 --- a/src/transformers/models/flava/image_processing_flava.py +++ b/src/transformers/models/flava/image_processing_flava.py @@ -362,9 +362,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def map_pixels(self, image: np.ndarray) -> np.ndarray: return (1 - 2 * LOGIT_LAPLACE_EPS) * image + LOGIT_LAPLACE_EPS diff --git a/src/transformers/models/imagegpt/image_processing_imagegpt.py b/src/transformers/models/imagegpt/image_processing_imagegpt.py index 22c0ed83b803d5..c9478c3bbd4cdd 100644 --- a/src/transformers/models/imagegpt/image_processing_imagegpt.py +++ b/src/transformers/models/imagegpt/image_processing_imagegpt.py @@ -131,9 +131,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def normalize( self, diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index 459e1bf78390a1..ab70a015d8a79c 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -162,9 +162,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index f53bf956310ec2..9ca195ce793f07 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -188,9 +188,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index 0ad01bcc85f2b8..de490ab8d81ba3 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -177,9 +177,8 @@ def resize( size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs - ) + output_size = (size["height"], size["width"]) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, From e85cf639dd8f16a32f19918caffdd3248431affe Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Tue, 8 Aug 2023 19:02:18 +0000 Subject: [PATCH 6/9] Add input_data_format as preprocess argument --- src/transformers/image_processing_utils.py | 43 ++++- .../models/beit/image_processing_beit.py | 52 ++++- .../models/bit/image_processing_bit.py | 61 +++++- .../models/blip/image_processing_blip.py | 59 +++++- .../image_processing_bridgetower.py | 130 ++++++++++--- .../image_processing_chinese_clip.py | 60 +++++- .../models/clip/image_processing_clip.py | 61 +++++- .../image_processing_conditional_detr.py | 179 ++++++++++++++---- .../convnext/image_processing_convnext.py | 65 ++++++- .../image_processing_deformable_detr.py | 179 ++++++++++++++---- .../models/deit/image_processing_deit.py | 58 +++++- .../models/deta/image_processing_deta.py | 173 +++++++++++++---- .../models/detr/image_processing_detr.py | 179 ++++++++++++++---- .../models/donut/image_processing_donut.py | 98 ++++++++-- .../models/dpt/image_processing_dpt.py | 60 +++++- .../image_processing_efficientformer.py | 50 ++++- .../image_processing_efficientnet.py | 72 ++++++- .../models/flava/image_processing_flava.py | 49 ++++- .../models/glpn/image_processing_glpn.py | 44 ++++- .../imagegpt/image_processing_imagegpt.py | 52 ++++- .../layoutlmv2/image_processing_layoutlmv2.py | 35 +++- .../layoutlmv3/image_processing_layoutlmv3.py | 54 +++++- .../models/levit/image_processing_levit.py | 44 ++++- .../image_processing_mask2former.py | 135 ++++++++++--- .../maskformer/image_processing_maskformer.py | 147 +++++++++++--- .../image_processing_mobilenet_v1.py | 55 +++++- .../image_processing_mobilenet_v2.py | 57 +++++- .../mobilevit/image_processing_mobilevit.py | 61 +++++- .../oneformer/image_processing_oneformer.py | 126 +++++++++--- .../models/owlvit/image_processing_owlvit.py | 97 +++++++++- .../perceiver/image_processing_perceiver.py | 71 ++++++- .../pix2struct/image_processing_pix2struct.py | 65 ++++++- .../poolformer/image_processing_poolformer.py | 63 +++++- .../models/pvt/image_processing_pvt.py | 52 ++++- .../models/sam/image_processing_sam.py | 73 +++++-- .../segformer/image_processing_segformer.py | 54 +++++- .../swin2sr/image_processing_swin2sr.py | 59 +++++- .../models/tvlt/image_processing_tvlt.py | 45 ++++- .../videomae/image_processing_videomae.py | 45 ++++- .../models/vilt/image_processing_vilt.py | 116 ++++++++++-- .../models/vit/image_processing_vit.py | 52 ++++- .../vit_hybrid/image_processing_vit_hybrid.py | 55 +++++- .../models/vivit/image_processing_vivit.py | 52 ++++- .../models/yolos/image_processing_yolos.py | 164 +++++++++++++--- 44 files changed, 2940 insertions(+), 561 deletions(-) diff --git a/src/transformers/image_processing_utils.py b/src/transformers/image_processing_utils.py index 07885890918369..5f935566cd8d86 100644 --- a/src/transformers/image_processing_utils.py +++ b/src/transformers/image_processing_utils.py @@ -521,7 +521,12 @@ def preprocess(self, images, **kwargs) -> BatchFeature: raise NotImplementedError("Each image processor must implement its own preprocess method") def rescale( - self, image: np.ndarray, scale: float, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + self, + image: np.ndarray, + scale: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, ) -> np.ndarray: """ Rescale an image by a scale factor. image = image * scale. @@ -536,11 +541,16 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. Returns: `np.ndarray`: The rescaled image. """ - return rescale(image, scale=scale, data_format=data_format, **kwargs) + return rescale(image, scale=scale, data_format=data_format, input_data_format=input_data_format, **kwargs) def normalize( self, @@ -548,6 +558,7 @@ def normalize( mean: Union[float, Iterable[float]], std: Union[float, Iterable[float]], data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -565,17 +576,25 @@ def normalize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. Returns: `np.ndarray`: The normalized image. """ - return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + return normalize( + image, mean=mean, std=std, data_format=data_format, input_data_format=input_data_format, **kwargs + ) def center_crop( self, image: np.ndarray, size: Dict[str, int], data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -588,12 +607,26 @@ def center_crop( size (`Dict[str, int]`): Size of the output image. data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be the same as the input image. + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}") - return center_crop(image, size=(size["height"], size["width"]), data_format=data_format, **kwargs) + return center_crop( + image, + size=(size["height"], size["width"]), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) VALID_SIZE_DICT_KEYS = ({"height", "width"}, {"shortest_edge"}, {"shortest_edge", "longest_edge"}, {"longest_edge"}) diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index 08da81113dc7aa..59ea1d88dc8c26 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -27,6 +27,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -145,6 +146,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -159,12 +161,19 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=True, param_name="size") if "height" not in size or "width" not in size: raise ValueError(f"The `size` argument must contain `height` and `width` keys. Got {size.keys()}") return resize( - image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + image, + size=(size["height"], size["width"]), + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, ) def reduce_label(self, label: ImageInput) -> np.ndarray: @@ -189,21 +198,22 @@ def _preprocess( do_normalize: bool = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): if do_reduce_labels: image = self.reduce_label(image) if do_resize: - image = self.resize(image=image, size=size, resample=resample) + image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) if do_center_crop: - image = self.center_crop(image=image, size=crop_size) + image = self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image=image, scale=rescale_factor) + image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image=image, mean=image_mean, std=image_std) + image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) return image @@ -221,10 +231,16 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format( + image, input_data_format=input_data_format, num_channels=num_channels + ) image = self._preprocess( image, do_reduce_labels=False, @@ -238,9 +254,10 @@ def _preprocess_image( do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, + input_data_format=input_data_format, ) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def _preprocess_segmentation_map( @@ -252,6 +269,7 @@ def _preprocess_segmentation_map( do_center_crop: bool = None, crop_size: Dict[str, int] = None, do_reduce_labels: bool = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): """Preprocesses a single segmentation map.""" # All transformations expect numpy arrays. @@ -260,8 +278,11 @@ def _preprocess_segmentation_map( if segmentation_map.ndim == 2: segmentation_map = segmentation_map[None, ...] added_dimension = True + input_data_format = ChannelDimension.FIRST else: added_dimension = False + if input_data_format is None: + input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) segmentation_map = self._preprocess( image=segmentation_map, do_reduce_labels=do_reduce_labels, @@ -272,6 +293,7 @@ def _preprocess_segmentation_map( crop_size=crop_size, do_normalize=False, do_rescale=False, + input_data_format=ChannelDimension.FIRST, ) # Remove extra axis if added if added_dimension: @@ -301,6 +323,8 @@ def preprocess( do_reduce_labels: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -344,8 +368,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -403,6 +437,8 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in images ] diff --git a/src/transformers/models/bit/image_processing_bit.py b/src/transformers/models/bit/image_processing_bit.py index 65c7a0c946a824..6f21ca3f169f52 100644 --- a/src/transformers/models/bit/image_processing_bit.py +++ b/src/transformers/models/bit/image_processing_bit.py @@ -31,6 +31,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -125,6 +126,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -140,12 +142,23 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}") - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -163,6 +176,8 @@ def preprocess( do_convert_rgb: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -205,9 +220,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: defaults to the channel dimension format of the input image. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -250,19 +274,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 23b008f8d72b2b..294c3139513d3f 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -26,6 +26,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -111,6 +112,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -128,6 +130,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -136,7 +145,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -152,6 +168,8 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, do_convert_rgb: bool = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -190,8 +208,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -229,16 +257,31 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 1ae7868f47f3b3..81541e0de008eb 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -50,7 +50,9 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.vilt.image_processing_vilt.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -60,33 +62,40 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask # Copied from transformers.models.vilt.image_processing_vilt.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.vilt.image_processing_vilt.get_resize_output_image_size def get_resize_output_image_size( - input_image: np.ndarray, shorter: int = 800, longer: int = 1333, size_divisor: int = 32 + input_image: np.ndarray, + shorter: int = 800, + longer: int = 1333, + size_divisor: int = 32, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: - input_height, input_width = get_image_size(input_image) + input_height, input_width = get_image_size(input_image, input_data_format=input_data_format) min_size, max_size = shorter, longer scale = min_size / min(input_height, input_width) @@ -197,6 +206,7 @@ def resize( size_divisor: int = 32, resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -217,20 +227,32 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}") shorter = size["shortest_edge"] longer = int(1333 / 800 * shorter) - output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, shorter=shorter, longer=longer, size_divisor=size_divisor, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def center_crop( self, image: np.ndarray, size: Dict[str, int], data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -244,9 +266,18 @@ def center_crop( Size of the output image in the form `{"height": h, "width": w}`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input + image. """ output_size = size["shortest_edge"] - return center_crop(image, size=(output_size, output_size), data_format=data_format, **kwargs) + return center_crop( + image, + size=(output_size, output_size), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor._pad_image def _pad_image( @@ -255,18 +286,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -278,6 +315,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -299,17 +337,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -330,6 +379,8 @@ def preprocess( do_center_crop: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -374,8 +425,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size_divisor = size_divisor if size_divisor is not None else self.size_divisor @@ -414,22 +475,41 @@ def preprocess( if do_resize: images = [ - self.resize(image=image, size=size, size_divisor=size_divisor, resample=resample) for image in images + self.resize( + image=image, + size=size, + size_divisor=size_divisor, + resample=resample, + input_data_format=input_data_format, + ) + for image in images ] if do_center_crop: - images = [self.center_crop(image=image, size=size) for image in images] + images = [ + self.center_crop(image=image, size=size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] if do_pad: - encoded_outputs = self.pad(images, return_pixel_mask=True, return_tensors=return_tensors) + encoded_outputs = self.pad( + images, return_pixel_mask=True, return_tensors=return_tensors, input_data_format=input_data_format + ) else: encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py index 974f2ea075a7b4..14ce13e26c95e5 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py @@ -31,6 +31,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -124,6 +125,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -139,12 +141,22 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input + image. """ size = get_size_dict(size, default_to_square=False) output_size = get_resize_output_image_size( - image, size=(size["height"], size["width"]), default_to_square=False + image, size=(size["height"], size["width"]), default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, ) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, @@ -162,6 +174,8 @@ def preprocess( do_convert_rgb: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -204,9 +218,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: defaults to the channel dimension format of the input image. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -249,19 +272,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/clip/image_processing_clip.py b/src/transformers/models/clip/image_processing_clip.py index 34a1f6bce3c4c5..9e6f18f5db287a 100644 --- a/src/transformers/models/clip/image_processing_clip.py +++ b/src/transformers/models/clip/image_processing_clip.py @@ -31,6 +31,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -124,6 +125,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -139,12 +141,23 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}") - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -162,6 +175,8 @@ def preprocess( do_convert_rgb: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -204,9 +219,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: defaults to the channel dimension format of the input image. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -249,19 +273,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index 465f224cc2fb0c..3f973103be9a9d 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -124,7 +124,10 @@ def get_size_with_aspect_ratio(image_size, size, max_size=None) -> Tuple[int, in # Copied from transformers.models.detr.image_processing_detr.get_resize_output_image_size def get_resize_output_image_size( - input_image: np.ndarray, size: Union[int, Tuple[int, int], List[int]], max_size: Optional[int] = None + input_image: np.ndarray, + size: Union[int, Tuple[int, int], List[int]], + max_size: Optional[int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: """ Computes the output image size given the input image size and the desired output size. If the desired output size @@ -138,8 +141,10 @@ def get_resize_output_image_size( The desired output size. max_size (`int`, *optional*): The maximum allowed output size. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image) + image_size = get_image_size(input_image, input_data_format=input_data_format) if isinstance(size, (list, tuple)): return size @@ -209,23 +214,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -235,7 +245,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -277,11 +287,16 @@ def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndar # Copied from transformers.models.detr.image_processing_detr.prepare_coco_detection_annotation with DETR->ConditionalDetr -def prepare_coco_detection_annotation(image, target, return_segmentation_masks: bool = False): +def prepare_coco_detection_annotation( + image, + target, + return_segmentation_masks: bool = False, + input_data_format: Optional[Union[ChannelDimension, str]] = None, +): """ Convert the target in COCO format into the format expected by ConditionalDetr. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) image_id = target["image_id"] image_id = np.asarray([image_id], dtype=np.int64) @@ -366,12 +381,16 @@ def masks_to_boxes(masks: np.ndarray) -> np.ndarray: # Copied from transformers.models.detr.image_processing_detr.prepare_coco_panoptic_annotation with DETR->ConditionalDetr def prepare_coco_panoptic_annotation( - image: np.ndarray, target: Dict, masks_path: Union[str, pathlib.Path], return_masks: bool = True + image: np.ndarray, + target: Dict, + masks_path: Union[str, pathlib.Path], + return_masks: bool = True, + input_data_format: Union[ChannelDimension, str] = None, ) -> Dict: """ Prepare a coco panoptic annotation for ConditionalDetr. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) annotation_path = pathlib.Path(masks_path) / target["file_name"] new_target = {} @@ -842,6 +861,7 @@ def prepare_annotation( format: Optional[AnnotionFormat] = None, return_segmentation_masks: bool = None, masks_path: Optional[Union[str, pathlib.Path]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Dict: """ Prepare an annotation for feeding into ConditionalDetr model. @@ -850,11 +870,17 @@ def prepare_annotation( if format == AnnotionFormat.COCO_DETECTION: return_segmentation_masks = False if return_segmentation_masks is None else return_segmentation_masks - target = prepare_coco_detection_annotation(image, target, return_segmentation_masks) + target = prepare_coco_detection_annotation( + image, target, return_segmentation_masks, input_data_format=input_data_format + ) elif format == AnnotionFormat.COCO_PANOPTIC: return_segmentation_masks = True if return_segmentation_masks is None else return_segmentation_masks target = prepare_coco_panoptic_annotation( - image, target, masks_path=masks_path, return_masks=return_segmentation_masks + image, + target, + masks_path=masks_path, + return_masks=return_segmentation_masks, + input_data_format=input_data_format, ) else: raise ValueError(f"Format {format} is not supported.") @@ -892,11 +918,26 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be `min_size` (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Dictionary containing the size to resize to. Can contain the keys `shortest_edge` and `longest_edge` or + `height` and `width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use if resizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if "max_size" in kwargs: logger.warning_once( @@ -908,7 +949,9 @@ def resize( max_size = None size = get_size_dict(size, max_size=max_size, default_to_square=False) if "shortest_edge" in size and "longest_edge" in size: - size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"]) + size = get_resize_output_image_size( + image, size["shortest_edge"], size["longest_edge"], input_data_format=input_data_format + ) elif "height" in size and "width" in size: size = (size["height"], size["width"]) else: @@ -916,7 +959,9 @@ def resize( "Size must contain 'height' and 'width' keys or 'shortest_edge' and 'longest_edge' keys. Got" f" {size.keys()}." ) - image = resize(image, size=size, resample=resample, data_format=data_format) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs + ) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.resize_annotation @@ -935,7 +980,11 @@ def resize_annotation( # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -950,8 +999,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: @@ -968,18 +1022,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -991,6 +1051,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -1012,17 +1073,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -1046,6 +1118,8 @@ def preprocess( format: Optional[Union[str, AnnotionFormat]] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1091,8 +1165,20 @@ def preprocess( Format of the annotations. return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): Type of tensors to return. If `None`, will return the list of images. - data_format (`str` or `ChannelDimension`, *optional*, defaults to self.data_format): - The channel dimension format of the image. If not provided, it will be the same as the input image. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1177,13 +1263,22 @@ def preprocess( # All transformations expect numpy arrays images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: prepared_images = [] prepared_annotations = [] for image, target in zip(images, annotations): target = self.prepare_annotation( - image, target, format, return_segmentation_masks=return_segmentation_masks, masks_path=masks_path + image, + target, + format, + return_segmentation_masks=return_segmentation_masks, + masks_path=masks_path, + input_data_format=input_data_format, ) prepared_images.append(image) prepared_annotations.append(target) @@ -1197,32 +1292,46 @@ def preprocess( resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): orig_size = get_image_size(image) - resized_image = self.resize(image, size=size, max_size=max_size, resample=resample) - resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) + resized_image = self.resize( + image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format + ) + resized_annotation = self.resize_annotation( + target, orig_size, get_image_size(resized_image), input_data_format=input_data_format + ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) images = resized_images annotations = resized_annotations del resized_images, resized_annotations else: - images = [self.resize(image, size=size, resample=resample) for image in images] + images = [ + self.resize(image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image, rescale_factor) for image in images] + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] if do_normalize: - images = [self.normalize(image, image_mean, image_std) for image in images] + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image)) + self.normalize_annotation(annotation, get_image_size(image), input_data_format=input_data_format) for annotation, image in zip(annotations, images) ] if do_pad: # Pads images and returns their mask: {'pixel_values': ..., 'pixel_mask': ...} - data = self.pad(images, return_pixel_mask=True, data_format=data_format) + data = self.pad( + images, return_pixel_mask=True, data_format=data_format, input_data_format=input_data_format + ) else: - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + for image in images + ] data = {"pixel_values": images} encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index cfd5b38d76b9b5..4b687a48993ea4 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -31,6 +31,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -118,6 +119,7 @@ def resize( crop_pct: float, resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -137,6 +139,9 @@ def resize( Resampling filter to use when resizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input + image. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: @@ -146,14 +151,27 @@ def resize( if shortest_edge < 384: # maintain same ratio, resizing shortest edge to shortest_edge/crop_pct resize_shortest_edge = int(shortest_edge / crop_pct) - resize_size = get_resize_output_image_size(image, size=resize_shortest_edge, default_to_square=False) + resize_size = get_resize_output_image_size( + image, size=resize_shortest_edge, default_to_square=False, input_data_format=input_data_format + ) image = resize(image=image, size=resize_size, resample=resample, data_format=data_format, **kwargs) # then crop to (shortest_edge, shortest_edge) - return center_crop(image=image, size=(shortest_edge, shortest_edge), data_format=data_format, **kwargs) + return center_crop( + image=image, + size=(shortest_edge, shortest_edge), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) else: # warping (no cropping) when evaluated at 384 or larger return resize( - image, size=(shortest_edge, shortest_edge), resample=resample, data_format=data_format, **kwargs + image, + size=(shortest_edge, shortest_edge), + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, ) def preprocess( @@ -170,6 +188,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -209,8 +229,18 @@ def preprocess( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): The channel dimension format for the output image. Can be one of: - - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize crop_pct = crop_pct if crop_pct is not None else self.crop_pct @@ -247,16 +277,33 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, crop_pct=crop_pct, resample=resample) for image in images] + images = [ + self.resize( + image=image, size=size, crop_pct=crop_pct, resample=resample, input_data_format=input_data_format + ) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index 39d31fd62ec457..f2dc0609bc9156 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -123,7 +123,10 @@ def get_size_with_aspect_ratio(image_size, size, max_size=None) -> Tuple[int, in # Copied from transformers.models.detr.image_processing_detr.get_resize_output_image_size def get_resize_output_image_size( - input_image: np.ndarray, size: Union[int, Tuple[int, int], List[int]], max_size: Optional[int] = None + input_image: np.ndarray, + size: Union[int, Tuple[int, int], List[int]], + max_size: Optional[int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: """ Computes the output image size given the input image size and the desired output size. If the desired output size @@ -137,8 +140,10 @@ def get_resize_output_image_size( The desired output size. max_size (`int`, *optional*): The maximum allowed output size. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image) + image_size = get_image_size(input_image, input_data_format=input_data_format) if isinstance(size, (list, tuple)): return size @@ -208,23 +213,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -234,7 +244,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -276,11 +286,16 @@ def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndar # Copied from transformers.models.detr.image_processing_detr.prepare_coco_detection_annotation with DETR->DeformableDetr -def prepare_coco_detection_annotation(image, target, return_segmentation_masks: bool = False): +def prepare_coco_detection_annotation( + image, + target, + return_segmentation_masks: bool = False, + input_data_format: Optional[Union[ChannelDimension, str]] = None, +): """ Convert the target in COCO format into the format expected by DeformableDetr. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) image_id = target["image_id"] image_id = np.asarray([image_id], dtype=np.int64) @@ -365,12 +380,16 @@ def masks_to_boxes(masks: np.ndarray) -> np.ndarray: # Copied from transformers.models.detr.image_processing_detr.prepare_coco_panoptic_annotation with DETR->DeformableDetr def prepare_coco_panoptic_annotation( - image: np.ndarray, target: Dict, masks_path: Union[str, pathlib.Path], return_masks: bool = True + image: np.ndarray, + target: Dict, + masks_path: Union[str, pathlib.Path], + return_masks: bool = True, + input_data_format: Union[ChannelDimension, str] = None, ) -> Dict: """ Prepare a coco panoptic annotation for DeformableDetr. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) annotation_path = pathlib.Path(masks_path) / target["file_name"] new_target = {} @@ -840,6 +859,7 @@ def prepare_annotation( format: Optional[AnnotionFormat] = None, return_segmentation_masks: bool = None, masks_path: Optional[Union[str, pathlib.Path]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Dict: """ Prepare an annotation for feeding into DeformableDetr model. @@ -848,11 +868,17 @@ def prepare_annotation( if format == AnnotionFormat.COCO_DETECTION: return_segmentation_masks = False if return_segmentation_masks is None else return_segmentation_masks - target = prepare_coco_detection_annotation(image, target, return_segmentation_masks) + target = prepare_coco_detection_annotation( + image, target, return_segmentation_masks, input_data_format=input_data_format + ) elif format == AnnotionFormat.COCO_PANOPTIC: return_segmentation_masks = True if return_segmentation_masks is None else return_segmentation_masks target = prepare_coco_panoptic_annotation( - image, target, masks_path=masks_path, return_masks=return_segmentation_masks + image, + target, + masks_path=masks_path, + return_masks=return_segmentation_masks, + input_data_format=input_data_format, ) else: raise ValueError(f"Format {format} is not supported.") @@ -890,11 +916,26 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be `min_size` (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Dictionary containing the size to resize to. Can contain the keys `shortest_edge` and `longest_edge` or + `height` and `width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use if resizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if "max_size" in kwargs: logger.warning_once( @@ -906,7 +947,9 @@ def resize( max_size = None size = get_size_dict(size, max_size=max_size, default_to_square=False) if "shortest_edge" in size and "longest_edge" in size: - size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"]) + size = get_resize_output_image_size( + image, size["shortest_edge"], size["longest_edge"], input_data_format=input_data_format + ) elif "height" in size and "width" in size: size = (size["height"], size["width"]) else: @@ -914,7 +957,9 @@ def resize( "Size must contain 'height' and 'width' keys or 'shortest_edge' and 'longest_edge' keys. Got" f" {size.keys()}." ) - image = resize(image, size=size, resample=resample, data_format=data_format) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs + ) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.resize_annotation @@ -933,7 +978,11 @@ def resize_annotation( # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -948,8 +997,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: @@ -966,18 +1020,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -989,6 +1049,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -1010,17 +1071,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -1044,6 +1116,8 @@ def preprocess( format: Optional[Union[str, AnnotionFormat]] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1089,8 +1163,20 @@ def preprocess( Format of the annotations. return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): Type of tensors to return. If `None`, will return the list of images. - data_format (`str` or `ChannelDimension`, *optional*, defaults to self.data_format): - The channel dimension format of the image. If not provided, it will be the same as the input image. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1175,13 +1261,22 @@ def preprocess( # All transformations expect numpy arrays images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: prepared_images = [] prepared_annotations = [] for image, target in zip(images, annotations): target = self.prepare_annotation( - image, target, format, return_segmentation_masks=return_segmentation_masks, masks_path=masks_path + image, + target, + format, + return_segmentation_masks=return_segmentation_masks, + masks_path=masks_path, + input_data_format=input_data_format, ) prepared_images.append(image) prepared_annotations.append(target) @@ -1195,32 +1290,46 @@ def preprocess( resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): orig_size = get_image_size(image) - resized_image = self.resize(image, size=size, max_size=max_size, resample=resample) - resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) + resized_image = self.resize( + image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format + ) + resized_annotation = self.resize_annotation( + target, orig_size, get_image_size(resized_image), input_data_format=input_data_format + ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) images = resized_images annotations = resized_annotations del resized_images, resized_annotations else: - images = [self.resize(image, size=size, resample=resample) for image in images] + images = [ + self.resize(image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image, rescale_factor) for image in images] + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] if do_normalize: - images = [self.normalize(image, image_mean, image_std) for image in images] + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image)) + self.normalize_annotation(annotation, get_image_size(image), input_data_format=input_data_format) for annotation, image in zip(annotations, images) ] if do_pad: # Pads images and returns their mask: {'pixel_values': ..., 'pixel_mask': ...} - data = self.pad(images, return_pixel_mask=True, data_format=data_format) + data = self.pad( + images, return_pixel_mask=True, data_format=data_format, input_data_format=input_data_format + ) else: - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + for image in images + ] data = {"pixel_values": images} encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index 144984523e3cf9..d786d7c30f6ce8 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -26,6 +26,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -114,6 +115,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -131,6 +133,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -139,7 +148,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -156,6 +172,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -197,6 +215,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -235,19 +262,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/deta/image_processing_deta.py b/src/transformers/models/deta/image_processing_deta.py index 4ae35b101d66f5..ae8805f01a6edd 100644 --- a/src/transformers/models/deta/image_processing_deta.py +++ b/src/transformers/models/deta/image_processing_deta.py @@ -115,7 +115,10 @@ def get_size_with_aspect_ratio(image_size, size, max_size=None) -> Tuple[int, in # Copied from transformers.models.detr.image_processing_detr.get_resize_output_image_size def get_resize_output_image_size( - input_image: np.ndarray, size: Union[int, Tuple[int, int], List[int]], max_size: Optional[int] = None + input_image: np.ndarray, + size: Union[int, Tuple[int, int], List[int]], + max_size: Optional[int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: """ Computes the output image size given the input image size and the desired output size. If the desired output size @@ -129,8 +132,10 @@ def get_resize_output_image_size( The desired output size. max_size (`int`, *optional*): The maximum allowed output size. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image) + image_size = get_image_size(input_image, input_data_format=input_data_format) if isinstance(size, (list, tuple)): return size @@ -200,23 +205,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -226,7 +236,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -268,11 +278,16 @@ def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndar # Copied from transformers.models.detr.image_processing_detr.prepare_coco_detection_annotation with DETR->DETA -def prepare_coco_detection_annotation(image, target, return_segmentation_masks: bool = False): +def prepare_coco_detection_annotation( + image, + target, + return_segmentation_masks: bool = False, + input_data_format: Optional[Union[ChannelDimension, str]] = None, +): """ Convert the target in COCO format into the format expected by DETA. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) image_id = target["image_id"] image_id = np.asarray([image_id], dtype=np.int64) @@ -357,12 +372,16 @@ def masks_to_boxes(masks: np.ndarray) -> np.ndarray: # Copied from transformers.models.detr.image_processing_detr.prepare_coco_panoptic_annotation with DETR->DETA def prepare_coco_panoptic_annotation( - image: np.ndarray, target: Dict, masks_path: Union[str, pathlib.Path], return_masks: bool = True + image: np.ndarray, + target: Dict, + masks_path: Union[str, pathlib.Path], + return_masks: bool = True, + input_data_format: Union[ChannelDimension, str] = None, ) -> Dict: """ Prepare a coco panoptic annotation for DETA. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) annotation_path = pathlib.Path(masks_path) / target["file_name"] new_target = {} @@ -522,6 +541,7 @@ def prepare_annotation( format: Optional[AnnotionFormat] = None, return_segmentation_masks: bool = None, masks_path: Optional[Union[str, pathlib.Path]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Dict: """ Prepare an annotation for feeding into DETA model. @@ -530,11 +550,17 @@ def prepare_annotation( if format == AnnotionFormat.COCO_DETECTION: return_segmentation_masks = False if return_segmentation_masks is None else return_segmentation_masks - target = prepare_coco_detection_annotation(image, target, return_segmentation_masks) + target = prepare_coco_detection_annotation( + image, target, return_segmentation_masks, input_data_format=input_data_format + ) elif format == AnnotionFormat.COCO_PANOPTIC: return_segmentation_masks = True if return_segmentation_masks is None else return_segmentation_masks target = prepare_coco_panoptic_annotation( - image, target, masks_path=masks_path, return_masks=return_segmentation_masks + image, + target, + masks_path=masks_path, + return_masks=return_segmentation_masks, + input_data_format=input_data_format, ) else: raise ValueError(f"Format {format} is not supported.") @@ -571,15 +597,32 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be `min_size` (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + The desired output size. Can contain keys `shortest_edge` and `longest_edge` or `height` and `width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use if resizing the image. + data_format (`ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input + image. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" in size and "longest_edge" in size: - size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"]) + size = get_resize_output_image_size( + image, size["shortest_edge"], size["longest_edge"], input_data_format=input_data_format + ) elif "height" in size and "width" in size: size = (size["height"], size["width"]) else: @@ -587,7 +630,9 @@ def resize( "Size must contain 'height' and 'width' keys or 'shortest_edge' and 'longest_edge' keys. Got" f" {size.keys()}." ) - image = resize(image, size=size, resample=resample, data_format=data_format) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format + ) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.resize_annotation @@ -606,7 +651,11 @@ def resize_annotation( # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -621,8 +670,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: @@ -639,18 +693,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -662,6 +722,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -683,17 +744,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -716,6 +788,8 @@ def preprocess( format: Optional[Union[str, AnnotionFormat]] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -761,8 +835,20 @@ def preprocess( Format of the annotations. return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): Type of tensors to return. If `None`, will return the list of images. - data_format (`str` or `ChannelDimension`, *optional*, defaults to self.data_format): - The channel dimension format of the image. If not provided, it will be the same as the input image. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -839,13 +925,22 @@ def preprocess( # All transformations expect numpy arrays images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: prepared_images = [] prepared_annotations = [] for image, target in zip(images, annotations): target = self.prepare_annotation( - image, target, format, return_segmentation_masks=return_segmentation_masks, masks_path=masks_path + image, + target, + format, + return_segmentation_masks=return_segmentation_masks, + masks_path=masks_path, + input_data_format=input_data_format, ) prepared_images.append(image) prepared_annotations.append(target) @@ -859,7 +954,9 @@ def preprocess( resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): orig_size = get_image_size(image) - resized_image = self.resize(image, size=size, resample=resample) + resized_image = self.resize( + image, size=size, resample=resample, input_data_format=input_data_format + ) resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) resized_images.append(resized_image) resized_annotations.append(resized_annotation) @@ -867,13 +964,18 @@ def preprocess( annotations = resized_annotations del resized_images, resized_annotations else: - images = [self.resize(image, size=size, resample=resample) for image in images] + images = [ + self.resize(image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image, rescale_factor) for image in images] + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] if do_normalize: - images = [self.normalize(image, image_mean, image_std) for image in images] + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] if annotations is not None: annotations = [ self.normalize_annotation(annotation, get_image_size(image)) @@ -882,9 +984,14 @@ def preprocess( if do_pad: # Pads images and returns their mask: {'pixel_values': ..., 'pixel_mask': ...} - data = self.pad(images, return_pixel_mask=True, data_format=data_format) + data = self.pad( + images, return_pixel_mask=True, data_format=data_format, input_data_format=input_data_format + ) else: - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + for image in images + ] data = {"pixel_values": images} encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 794f0f4bd4e9a9..7115c3aa5a014b 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -121,7 +121,10 @@ def get_size_with_aspect_ratio(image_size, size, max_size=None) -> Tuple[int, in def get_resize_output_image_size( - input_image: np.ndarray, size: Union[int, Tuple[int, int], List[int]], max_size: Optional[int] = None + input_image: np.ndarray, + size: Union[int, Tuple[int, int], List[int]], + max_size: Optional[int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: """ Computes the output image size given the input image size and the desired output size. If the desired output size @@ -135,8 +138,10 @@ def get_resize_output_image_size( The desired output size. max_size (`int`, *optional*): The maximum allowed output size. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image) + image_size = get_image_size(input_image, input_data_format=input_data_format) if isinstance(size, (list, tuple)): return size @@ -203,23 +208,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.vilt.image_processing_vilt.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.vilt.image_processing_vilt.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -229,7 +239,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -271,11 +281,16 @@ def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndar # inspired by https://github.com/facebookresearch/detr/blob/master/datasets/coco.py#L50 -def prepare_coco_detection_annotation(image, target, return_segmentation_masks: bool = False): +def prepare_coco_detection_annotation( + image, + target, + return_segmentation_masks: bool = False, + input_data_format: Optional[Union[ChannelDimension, str]] = None, +): """ Convert the target in COCO format into the format expected by DETR. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) image_id = target["image_id"] image_id = np.asarray([image_id], dtype=np.int64) @@ -358,12 +373,16 @@ def masks_to_boxes(masks: np.ndarray) -> np.ndarray: def prepare_coco_panoptic_annotation( - image: np.ndarray, target: Dict, masks_path: Union[str, pathlib.Path], return_masks: bool = True + image: np.ndarray, + target: Dict, + masks_path: Union[str, pathlib.Path], + return_masks: bool = True, + input_data_format: Union[ChannelDimension, str] = None, ) -> Dict: """ Prepare a coco panoptic annotation for DETR. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) annotation_path = pathlib.Path(masks_path) / target["file_name"] new_target = {} @@ -822,6 +841,7 @@ def prepare_annotation( format: Optional[AnnotionFormat] = None, return_segmentation_masks: bool = None, masks_path: Optional[Union[str, pathlib.Path]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Dict: """ Prepare an annotation for feeding into DETR model. @@ -830,11 +850,17 @@ def prepare_annotation( if format == AnnotionFormat.COCO_DETECTION: return_segmentation_masks = False if return_segmentation_masks is None else return_segmentation_masks - target = prepare_coco_detection_annotation(image, target, return_segmentation_masks) + target = prepare_coco_detection_annotation( + image, target, return_segmentation_masks, input_data_format=input_data_format + ) elif format == AnnotionFormat.COCO_PANOPTIC: return_segmentation_masks = True if return_segmentation_masks is None else return_segmentation_masks target = prepare_coco_panoptic_annotation( - image, target, masks_path=masks_path, return_masks=return_segmentation_masks + image, + target, + masks_path=masks_path, + return_masks=return_segmentation_masks, + input_data_format=input_data_format, ) else: raise ValueError(f"Format {format} is not supported.") @@ -867,11 +893,26 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be `min_size` (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Dictionary containing the size to resize to. Can contain the keys `shortest_edge` and `longest_edge` or + `height` and `width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use if resizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if "max_size" in kwargs: logger.warning_once( @@ -883,7 +924,9 @@ def resize( max_size = None size = get_size_dict(size, max_size=max_size, default_to_square=False) if "shortest_edge" in size and "longest_edge" in size: - size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"]) + size = get_resize_output_image_size( + image, size["shortest_edge"], size["longest_edge"], input_data_format=input_data_format + ) elif "height" in size and "width" in size: size = (size["height"], size["width"]) else: @@ -891,7 +934,9 @@ def resize( "Size must contain 'height' and 'width' keys or 'shortest_edge' and 'longest_edge' keys. Got" f" {size.keys()}." ) - image = resize(image, size=size, resample=resample, data_format=data_format) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs + ) return image def resize_annotation( @@ -909,7 +954,11 @@ def resize_annotation( # TODO (Amy) - update to use `rescale_factor` instead of `scale` def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -924,8 +973,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: """ @@ -940,18 +994,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -962,6 +1022,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -983,17 +1044,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -1016,6 +1088,8 @@ def preprocess( format: Optional[Union[str, AnnotionFormat]] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1061,8 +1135,20 @@ def preprocess( Format of the annotations. return_tensors (`str` or `TensorType`, *optional*, defaults to self.return_tensors): Type of tensors to return. If `None`, will return the list of images. - data_format (`str` or `ChannelDimension`, *optional*, defaults to self.data_format): - The channel dimension format of the image. If not provided, it will be the same as the input image. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1147,13 +1233,22 @@ def preprocess( # All transformations expect numpy arrays images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: prepared_images = [] prepared_annotations = [] for image, target in zip(images, annotations): target = self.prepare_annotation( - image, target, format, return_segmentation_masks=return_segmentation_masks, masks_path=masks_path + image, + target, + format, + return_segmentation_masks=return_segmentation_masks, + masks_path=masks_path, + input_data_format=input_data_format, ) prepared_images.append(image) prepared_annotations.append(target) @@ -1167,32 +1262,46 @@ def preprocess( resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): orig_size = get_image_size(image) - resized_image = self.resize(image, size=size, max_size=max_size, resample=resample) - resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) + resized_image = self.resize( + image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format + ) + resized_annotation = self.resize_annotation( + target, orig_size, get_image_size(resized_image), input_data_format=input_data_format + ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) images = resized_images annotations = resized_annotations del resized_images, resized_annotations else: - images = [self.resize(image, size=size, resample=resample) for image in images] + images = [ + self.resize(image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image, rescale_factor) for image in images] + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] if do_normalize: - images = [self.normalize(image, image_mean, image_std) for image in images] + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image)) + self.normalize_annotation(annotation, get_image_size(image), input_data_format=input_data_format) for annotation, image in zip(annotations, images) ] if do_pad: # Pads images and returns their mask: {'pixel_values': ..., 'pixel_mask': ...} - data = self.pad(images, return_pixel_mask=True, data_format=data_format) + data = self.pad( + images, return_pixel_mask=True, data_format=data_format, input_data_format=input_data_format + ) else: - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + for image in images + ] data = {"pixel_values": images} encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index db72f7f3265887..c435ef0875a573 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -32,6 +32,7 @@ ImageInput, PILImageResampling, get_image_size, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -122,7 +123,11 @@ def __init__( self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD def align_long_axis( - self, image: np.ndarray, size: Dict[str, int], data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + size: Dict[str, int], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Align the long axis of the image to the longest axis of the specified size. @@ -132,11 +137,15 @@ def align_long_axis( The image to be aligned. size (`Dict[str, int]`): The size `{"height": h, "width": w}` to align the long axis to. + data_format (`str` or `ChannelDimension`, *optional*): + The data format of the output image. If unset, the same format as the input image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. Returns: `np.ndarray`: The aligned image. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = size["height"], size["width"] if (output_width < output_height and input_width > input_height) or ( @@ -145,7 +154,7 @@ def align_long_axis( image = np.rot90(image, 3) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image @@ -155,6 +164,7 @@ def pad_image( size: Dict[str, int], random_padding: bool = False, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad the image to the specified size. @@ -168,9 +178,11 @@ def pad_image( Whether to use random padding or not. data_format (`str` or `ChannelDimension`, *optional*): The data format of the output image. If unset, the same format as the input image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ output_height, output_width = size["height"], size["width"] - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) delta_width = output_width - input_width delta_height = output_height - input_height @@ -186,7 +198,7 @@ def pad_image( pad_right = delta_width - pad_left padding = ((pad_top, pad_bottom), (pad_left, pad_right)) - return pad(image, padding, data_format=data_format) + return pad(image, padding, data_format=data_format, input_data_format=input_data_format) def pad(self, *args, **kwargs): logger.info("pad is deprecated and will be removed in version 4.27. Please use pad_image instead.") @@ -198,6 +210,7 @@ def thumbnail( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -213,8 +226,10 @@ def thumbnail( The resampling filter to use. data_format (`Optional[Union[str, ChannelDimension]]`, *optional*): The data format of the output image. If unset, the same format as the input image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = size["height"], size["width"] # We always resize to the smallest of either the input or output size. @@ -230,7 +245,13 @@ def thumbnail( height = int(input_height * width / input_width) return resize( - image, size=(height, width), resample=resample, reducing_gap=2.0, data_format=data_format, **kwargs + image, + size=(height, width), + resample=resample, + reducing_gap=2.0, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, ) def resize( @@ -239,6 +260,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -254,11 +276,22 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size) shortest_edge = min(size["height"], size["width"]) - output_size = get_resize_output_image_size(image, size=shortest_edge, default_to_square=False) - resized_image = resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=shortest_edge, default_to_square=False, input_data_format=input_data_format + ) + resized_image = resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) return resized_image def preprocess( @@ -278,6 +311,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -327,6 +362,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: defaults to the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. if `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -367,25 +411,45 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_align_long_axis: - images = [self.align_long_axis(image, size=size) for image in images] + images = [self.align_long_axis(image, size=size, input_data_format=input_data_format) for image in images] if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_thumbnail: - images = [self.thumbnail(image=image, size=size) for image in images] + images = [self.thumbnail(image=image, size=size, input_data_format=input_data_format) for image in images] if do_pad: - images = [self.pad_image(image=image, size=size, random_padding=random_padding) for image in images] + images = [ + self.pad_image( + image=image, size=size, random_padding=random_padding, input_data_format=input_data_format + ) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index 417f5b3c842a64..00da6f5b4690d6 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -28,6 +28,7 @@ ImageInput, PILImageResampling, get_image_size, + infer_channel_dimension_format, is_torch_available, is_torch_tensor, make_list_of_images, @@ -48,7 +49,11 @@ def get_resize_output_image_size( - input_image: np.ndarray, output_size: Union[int, Iterable[int]], keep_aspect_ratio: bool, multiple: int + input_image: np.ndarray, + output_size: Union[int, Iterable[int]], + keep_aspect_ratio: bool, + multiple: int, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: def constraint_to_multiple_of(val, multiple, min_val=0, max_val=None): x = round(val / multiple) * multiple @@ -63,7 +68,7 @@ def constraint_to_multiple_of(val, multiple, min_val=0, max_val=None): output_size = (output_size, output_size) if isinstance(output_size, int) else output_size - input_height, input_width = get_image_size(input_image) + input_height, input_width = get_image_size(input_image, input_data_format=input_data_format) output_height, output_width = output_size # determine new height and width @@ -156,6 +161,7 @@ def resize( ensure_multiple_of: int = 1, resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -179,8 +185,10 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - size = get_size_dict(size) + size = get_size_dict(size, input_data_format=input_data_format) if "height" not in size or "width" not in size: raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}") output_size = get_resize_output_image_size( @@ -188,8 +196,16 @@ def resize( output_size=(size["height"], size["width"]), keep_aspect_ratio=keep_aspect_ratio, multiple=ensure_multiple_of, + input_data_format=input_data_format, + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, ) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def preprocess( self, @@ -206,6 +222,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -249,6 +267,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -282,16 +309,31 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/efficientformer/image_processing_efficientformer.py b/src/transformers/models/efficientformer/image_processing_efficientformer.py index ce552c6dd0a737..eda2d6f492689d 100644 --- a/src/transformers/models/efficientformer/image_processing_efficientformer.py +++ b/src/transformers/models/efficientformer/image_processing_efficientformer.py @@ -30,6 +30,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, is_batched, to_numpy_array, valid_images, @@ -116,6 +117,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -133,6 +135,8 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. Returns: `np.ndarray`: The resized image. @@ -140,13 +144,17 @@ def resize( size = get_size_dict(size) if "shortest_edge" in size: - size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) + size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) # size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"]) elif "height" in size and "width" in size: size = (size["height"], size["width"]) else: raise ValueError(f"Size must contain 'height' and 'width' keys or 'shortest_edge' key. Got {size.keys()}") - return resize(image, size=size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs + ) def preprocess( self, @@ -163,6 +171,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -205,6 +215,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -241,19 +260,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size_dict, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size_dict, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index a9cab52491a2e2..7f445be979601f 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -26,6 +26,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -123,6 +124,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.NEAREST, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -140,6 +142,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -148,7 +157,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def rescale( self, @@ -156,6 +172,7 @@ def rescale( scale: Union[int, float], offset: bool = True, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ): """ @@ -177,8 +194,12 @@ def rescale( Whether to scale the image in both negative and positive directions. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - rescaled_image = rescale(image, scale=scale, data_format=data_format, **kwargs) + rescaled_image = rescale( + image, scale=scale, data_format=data_format, input_data_format=input_data_format, **kwargs + ) if offset: rescaled_image = rescaled_image - 1 @@ -202,6 +223,8 @@ def preprocess( include_top: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -247,6 +270,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -287,22 +319,44 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor, offset=rescale_offset) for image in images] + images = [ + self.rescale( + image=image, scale=rescale_factor, offset=rescale_offset, input_data_format=input_data_format + ) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] if include_top: - images = [self.normalize(image=image, mean=[0, 0, 0], std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=[0, 0, 0], std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/flava/image_processing_flava.py b/src/transformers/models/flava/image_processing_flava.py index 2ece1bd2b32c59..1fea5b861b4e86 100644 --- a/src/transformers/models/flava/image_processing_flava.py +++ b/src/transformers/models/flava/image_processing_flava.py @@ -29,6 +29,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -338,6 +339,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -355,6 +357,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -363,7 +372,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def map_pixels(self, image: np.ndarray) -> np.ndarray: return (1 - 2 * LOGIT_LAPLACE_EPS) * image + LOGIT_LAPLACE_EPS @@ -383,6 +399,8 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, do_map_pixels: bool = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[ChannelDimension] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -397,23 +415,27 @@ def _preprocess_image( # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + if do_resize: - image = self.resize(image=image, size=size, resample=resample) + image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) if do_center_crop: - image = self.center_crop(image=image, size=crop_size) + image = self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image=image, scale=rescale_factor) + image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image=image, mean=image_mean, std=image_std) + image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) if do_map_pixels: image = self.map_pixels(image) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def preprocess( @@ -452,6 +474,8 @@ def preprocess( codebook_image_std: Optional[Iterable[float]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -533,6 +557,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -615,6 +648,8 @@ def preprocess( image_std=image_std, do_map_pixels=False, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in images ] @@ -636,6 +671,8 @@ def preprocess( image_std=codebook_image_std, do_map_pixels=codebook_do_map_pixels, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in images ] diff --git a/src/transformers/models/glpn/image_processing_glpn.py b/src/transformers/models/glpn/image_processing_glpn.py index 3e37d0bab221e7..7be723cc1effc9 100644 --- a/src/transformers/models/glpn/image_processing_glpn.py +++ b/src/transformers/models/glpn/image_processing_glpn.py @@ -25,6 +25,7 @@ ChannelDimension, PILImageResampling, get_image_size, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -75,6 +76,7 @@ def resize( size_divisor: int, resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -95,15 +97,27 @@ def resize( image is used. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not set, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. Returns: `np.ndarray`: The resized image. """ - height, width = get_image_size(image) + height, width = get_image_size(image, channel_dim=input_data_format) # Rounds the height and width down to the closest multiple of size_divisor new_h = height // size_divisor * size_divisor new_w = width // size_divisor * size_divisor - image = resize(image, (new_h, new_w), resample=resample, data_format=data_format, **kwargs) + image = resize( + image, + (new_h, new_w), + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) return image def preprocess( @@ -115,6 +129,8 @@ def preprocess( do_rescale: Optional[bool] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -144,6 +160,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -161,13 +186,22 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(img) for img in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image, size_divisor=size_divisor, resample=resample) for image in images] + images = [ + self.resize(image, size_divisor=size_divisor, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image, scale=1 / 255) for image in images] + images = [self.rescale(image, scale=1 / 255, input_data_format=input_data_format) for image in images] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/imagegpt/image_processing_imagegpt.py b/src/transformers/models/imagegpt/image_processing_imagegpt.py index c9478c3bbd4cdd..0fc0b545b981f2 100644 --- a/src/transformers/models/imagegpt/image_processing_imagegpt.py +++ b/src/transformers/models/imagegpt/image_processing_imagegpt.py @@ -24,6 +24,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -107,6 +108,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -124,6 +126,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -132,12 +141,20 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def normalize( self, image: np.ndarray, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Normalizes an images' pixel values to between [-1, 1]. @@ -147,8 +164,10 @@ def normalize( Image to normalize. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - image = rescale(image=image, scale=1 / 127.5, data_format=data_format) + image = rescale(image=image, scale=1 / 127.5, data_format=data_format, input_data_format=input_data_format) image = image - 1 return image @@ -163,6 +182,8 @@ def preprocess( clusters: Optional[Union[List[List[int]], np.ndarray]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -197,6 +218,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. Only has an effect if `do_color_quantize` is set to `False`. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -224,14 +254,21 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image) for image in images] + images = [self.normalize(image=image, input_data_format=input_data_format) for image in images] if do_color_quantize: - images = [to_channel_dimension_format(image, ChannelDimension.LAST) for image in images] + images = [to_channel_dimension_format(image, ChannelDimension.LAST, input_data_format) for image in images] # color quantize from (batch_size, height, width, 3) to (batch_size, height, width) images = np.array(images) images = color_quantize(images, clusters).reshape(images.shape[:-1]) @@ -243,7 +280,10 @@ def preprocess( # We need to convert back to a list of images to keep consistent behaviour across processors. images = list(images) else: - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + for image in images + ] data = {"input_ids": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index ab70a015d8a79c..e816ba36bd4020 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -24,6 +24,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -138,6 +139,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -155,6 +157,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -163,7 +172,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -176,6 +192,8 @@ def preprocess( tesseract_config: Optional[str] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -233,6 +251,10 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if apply_ocr: requires_backends(self, "pytesseract") words_batch = [] @@ -243,11 +265,16 @@ def preprocess( boxes_batch.append(boxes) if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] # flip color channels from RGB to BGR (as Detectron2 requires this) - images = [flip_channel_order(image) for image in images] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [flip_channel_order(image, input_data_format=input_data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index 9ca195ce793f07..f770011d1b742c 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -26,6 +26,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -164,6 +165,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -181,6 +183,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -189,7 +198,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -207,6 +223,8 @@ def preprocess( tesseract_config: Optional[str] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -252,6 +270,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -286,6 +313,10 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + # Tesseract OCR to get words + normalized bounding boxes if apply_ocr: requires_backends(self, "pytesseract") @@ -297,15 +328,26 @@ def preprocess( boxes_batch.append(boxes) if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/levit/image_processing_levit.py b/src/transformers/models/levit/image_processing_levit.py index fc96d7bf327223..37240235e3c560 100644 --- a/src/transformers/models/levit/image_processing_levit.py +++ b/src/transformers/models/levit/image_processing_levit.py @@ -30,6 +30,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -119,6 +120,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -143,19 +145,28 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size_dict = get_size_dict(size, default_to_square=False) # size_dict is a dict with either keys "height" and "width" or "shortest_edge" if "shortest_edge" in size: shortest_edge = int((256 / 224) * size["shortest_edge"]) - output_size = get_resize_output_image_size(image, size=shortest_edge, default_to_square=False) + output_size = get_resize_output_image_size( + image, size=shortest_edge, default_to_square=False, input_data_format=input_data_format + ) size_dict = {"height": output_size[0], "width": output_size[1]} if "height" not in size_dict or "width" not in size_dict: raise ValueError( f"Size dict must have keys 'height' and 'width' or 'shortest_edge'. Got {size_dict.keys()}" ) return resize( - image, size=(size_dict["height"], size_dict["width"]), resample=resample, data_format=data_format, **kwargs + image, + size=(size_dict["height"], size_dict["width"]), + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, ) def preprocess( @@ -173,6 +184,8 @@ def preprocess( image_std: Optional[Union[float, Iterable[float]]] = None, return_tensors: Optional[TensorType] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -217,6 +230,15 @@ def preprocess( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. if `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -255,19 +277,27 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image, size, resample) for image in images] + images = [self.resize(image, size, resample, input_data_format=input_data_format) for image in images] if do_center_crop: - images = [self.center_crop(image, crop_size) for image in images] + images = [self.center_crop(image, crop_size, input_data_format=input_data_format) for image in images] if do_rescale: - images = [self.rescale(image, rescale_factor) for image in images] + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] if do_normalize: - images = [self.normalize(image, image_mean, image_std) for image in images] + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index b9359251925e11..5db1387460524b 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -66,23 +66,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -92,7 +97,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -297,6 +302,7 @@ def get_mask2former_resize_output_image_size( max_size: Optional[int] = None, size_divisor: int = 0, default_to_square: bool = True, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> tuple: """ Computes the output size given the desired size. @@ -317,7 +323,11 @@ def get_mask2former_resize_output_image_size( `Tuple[int, int]`: The output size. """ output_size = get_resize_output_image_size( - input_image=image, size=size, default_to_square=default_to_square, max_size=max_size + input_image=image, + size=size, + default_to_square=default_to_square, + max_size=max_size, + input_data_format=input_data_format, ) if size_divisor > 0: @@ -450,11 +460,27 @@ def resize( size_divisor: int = 0, resample: PILImageResampling = PILImageResampling.BILINEAR, data_format=None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be min_size (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + The size of the output image. + size_divisor (`int`, *optional*, defaults to `0`): + If size_divisor is given, the output image size will be divisible by the number. + resample (`PILImageResampling` resampling filter, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use when resizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if "max_size" in kwargs: warnings.warn( @@ -482,13 +508,20 @@ def resize( max_size=max_size, size_divisor=size_divisor, default_to_square=False, + input_data_format=input_data_format, + ) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs ) - image = resize(image, size=size, resample=resample, data_format=data_format) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -503,8 +536,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( @@ -538,13 +576,16 @@ def _preprocess( do_normalize: bool = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): if do_resize: - image = self.resize(image, size=size, size_divisor=size_divisor, resample=resample) + image = self.resize( + image, size=size, size_divisor=size_divisor, resample=resample, input_data_format=input_data_format + ) if do_rescale: - image = self.rescale(image, rescale_factor=rescale_factor) + image = self.rescale(image, rescale_factor=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image, mean=image_mean, std=image_std) + image = self.normalize(image, mean=image_mean, std=image_std, input_data_format=input_data_format) return image def _preprocess_image( @@ -560,10 +601,14 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) image = self._preprocess( image=image, do_resize=do_resize, @@ -575,9 +620,10 @@ def _preprocess_image( do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, + input_data_format=input_data_format, ) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def _preprocess_mask( @@ -590,10 +636,14 @@ def _preprocess_mask( """Preprocesses a single mask.""" segmentation_map = to_numpy_array(segmentation_map) # Add channel dimension if missing - needed for certain transformations - added_channel_dim = False if segmentation_map.ndim == 2: added_channel_dim = True segmentation_map = segmentation_map[None, ...] + input_data_format = ChannelDimension.FIRST + else: + added_channel_dim = False + if input_data_format is None: + input_data_format = infer_channel_dimension_format(segmentation_map) # TODO: (Amy) # Remork segmentation map processing to include reducing labels and resizing which doesn't # drop segment IDs > 255. @@ -605,6 +655,7 @@ def _preprocess_mask( size_divisor=size_divisor, do_rescale=False, do_normalize=False, + input_data_format=input_data_format, ) # Remove extra channel dimension if added for processing if added_channel_dim: @@ -629,6 +680,8 @@ def preprocess( reduce_labels: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: if "pad_and_return_pixel_mask" in kwargs: @@ -691,17 +744,27 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for image in images ] if segmentation_maps is not None: segmentation_maps = [ - self._preprocess_mask(segmentation_map, do_resize, size, size_divisor) + self._preprocess_mask( + segmentation_map, do_resize, size, size_divisor, input_data_format=input_data_format + ) for segmentation_map in segmentation_maps ] encoded_inputs = self.encode_inputs( - images, segmentation_maps, instance_id_to_semantic_id, ignore_index, reduce_labels, return_tensors + images, + segmentation_maps, + instance_id_to_semantic_id, + ignore_index, + reduce_labels, + return_tensors, + input_data_format=input_data_format, ) return encoded_inputs @@ -712,18 +775,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -735,6 +804,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -756,17 +826,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -779,6 +860,7 @@ def encode_inputs( ignore_index: Optional[int] = None, reduce_labels: bool = False, return_tensors: Optional[Union[str, TensorType]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): """ Pad images up to the largest image in a batch and create a corresponding `pixel_mask`. @@ -815,6 +897,9 @@ def encode_inputs( If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor` objects. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. + Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: @@ -831,6 +916,10 @@ def encode_inputs( reduce_labels = self.reduce_labels if reduce_labels is None else reduce_labels pixel_values_list = [to_numpy_array(pixel_values) for pixel_values in pixel_values_list] + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(pixel_values_list[0]) + encoded_inputs = self.pad(pixel_values_list, return_tensors=return_tensors) if segmentation_maps is not None: diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index b5880566748b5b..5a00233124298d 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -70,23 +70,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -96,7 +101,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -299,6 +304,7 @@ def get_maskformer_resize_output_image_size( max_size: Optional[int] = None, size_divisor: int = 0, default_to_square: bool = True, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> tuple: """ Computes the output size given the desired size. @@ -319,7 +325,11 @@ def get_maskformer_resize_output_image_size( `Tuple[int, int]`: The output size. """ output_size = get_resize_output_image_size( - input_image=image, size=size, default_to_square=default_to_square, max_size=max_size + input_image=image, + size=size, + default_to_square=default_to_square, + max_size=max_size, + input_data_format=input_data_format, ) if size_divisor > 0: @@ -458,11 +468,27 @@ def resize( size_divisor: int = 0, resample: PILImageResampling = PILImageResampling.BILINEAR, data_format=None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be min_size (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + The size of the output image. + size_divisor (`int`, *optional*, defaults to `0`): + If size_divisor is given, the output image size will be divisible by the number. + resample (`PILImageResampling` resampling filter, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use when resizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if "max_size" in kwargs: warnings.warn( @@ -490,13 +516,20 @@ def resize( max_size=max_size, size_divisor=size_divisor, default_to_square=False, + input_data_format=input_data_format, + ) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs ) - image = resize(image, size=size, resample=resample, data_format=data_format) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -511,8 +544,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) def convert_segmentation_map_to_binary_masks( self, @@ -545,13 +583,16 @@ def _preprocess( do_normalize: bool = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): if do_resize: - image = self.resize(image, size=size, size_divisor=size_divisor, resample=resample) + image = self.resize( + image, size=size, size_divisor=size_divisor, resample=resample, input_data_format=input_data_format + ) if do_rescale: - image = self.rescale(image, rescale_factor=rescale_factor) + image = self.rescale(image, rescale_factor=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image, mean=image_mean, std=image_std) + image = self.normalize(image, mean=image_mean, std=image_std, input_data_format=input_data_format) return image def _preprocess_image( @@ -567,10 +608,14 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) image = self._preprocess( image=image, do_resize=do_resize, @@ -582,9 +627,10 @@ def _preprocess_image( do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, + input_data_format=input_data_format, ) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def _preprocess_mask( @@ -593,14 +639,19 @@ def _preprocess_mask( do_resize: bool = None, size: Dict[str, int] = None, size_divisor: int = 0, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """Preprocesses a single mask.""" segmentation_map = to_numpy_array(segmentation_map) # Add channel dimension if missing - needed for certain transformations - added_channel_dim = False if segmentation_map.ndim == 2: added_channel_dim = True segmentation_map = segmentation_map[None, ...] + input_data_format = ChannelDimension.FIRST + else: + added_channel_dim = False + if input_data_format is None: + input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) # TODO: (Amy) # Remork segmentation map processing to include reducing labels and resizing which doesn't # drop segment IDs > 255. @@ -612,6 +663,7 @@ def _preprocess_mask( size_divisor=size_divisor, do_rescale=False, do_normalize=False, + input_data_format=input_data_format, ) # Remove extra channel dimension if added for processing if added_channel_dim: @@ -636,6 +688,8 @@ def preprocess( do_reduce_labels: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: if "pad_and_return_pixel_mask" in kwargs: @@ -708,17 +762,27 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for image in images ] if segmentation_maps is not None: segmentation_maps = [ - self._preprocess_mask(segmentation_map, do_resize, size, size_divisor) + self._preprocess_mask( + segmentation_map, do_resize, size, size_divisor, input_data_format=input_data_format + ) for segmentation_map in segmentation_maps ] encoded_inputs = self.encode_inputs( - images, segmentation_maps, instance_id_to_semantic_id, ignore_index, do_reduce_labels, return_tensors + images, + segmentation_maps, + instance_id_to_semantic_id, + ignore_index, + do_reduce_labels, + return_tensors, + input_data_format=input_data_format, ) return encoded_inputs @@ -729,18 +793,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -752,6 +822,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -773,17 +844,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -796,6 +878,7 @@ def encode_inputs( ignore_index: Optional[int] = None, reduce_labels: bool = False, return_tensors: Optional[Union[str, TensorType]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): """ Pad images up to the largest image in a batch and create a corresponding `pixel_mask`. @@ -848,12 +931,18 @@ def encode_inputs( reduce_labels = self.do_reduce_labels if reduce_labels is None else reduce_labels pixel_values_list = [to_numpy_array(pixel_values) for pixel_values in pixel_values_list] - encoded_inputs = self.pad(pixel_values_list, return_tensors=return_tensors) + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(pixel_values_list[0]) + + encoded_inputs = self.pad( + pixel_values_list, return_tensors=return_tensors, input_data_format=input_data_format + ) if segmentation_maps is not None: mask_labels = [] class_labels = [] - pad_size = get_max_height_width(pixel_values_list) + pad_size = get_max_height_width(pixel_values_list, input_data_format=input_data_format) # Convert to list of binary masks and labels for idx, segmentation_map in enumerate(segmentation_maps): segmentation_map = to_numpy_array(segmentation_map) @@ -869,7 +958,13 @@ def encode_inputs( # this will be removed in the future masks = [mask[None, ...] for mask in masks] masks = [ - self._pad_image(image=mask, output_size=pad_size, constant_values=ignore_index) for mask in masks + self._pad_image( + image=mask, + output_size=pad_size, + constant_values=ignore_index, + input_data_format=ChannelDimension.FIRST, + ) + for mask in masks ] masks = np.concatenate(masks, axis=0) mask_labels.append(torch.from_numpy(masks)) diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py index a9f3cc0c38c535..1996539cb10c20 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py @@ -30,6 +30,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -118,6 +119,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -133,12 +135,23 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}") - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -155,6 +168,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ): """ @@ -197,6 +212,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -234,19 +258,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py index d3edd8236f02df..907aebafd71285 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py @@ -30,6 +30,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -122,6 +123,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -137,12 +139,23 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}") - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -159,6 +172,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ): """ @@ -201,6 +216,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -238,19 +262,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/mobilevit/image_processing_mobilevit.py b/src/transformers/models/mobilevit/image_processing_mobilevit.py index 147050099a6fa7..e8fc3e20939c56 100644 --- a/src/transformers/models/mobilevit/image_processing_mobilevit.py +++ b/src/transformers/models/mobilevit/image_processing_mobilevit.py @@ -29,6 +29,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -114,6 +115,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -129,15 +131,29 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}") - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def flip_channel_order( - self, image: np.ndarray, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Flip the color channels from RGB to BGR or vice versa. @@ -147,8 +163,10 @@ def flip_channel_order( The image, represented as a numpy array. data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - return flip_channel_order(image, data_format=data_format) + return flip_channel_order(image, data_format=data_format, input_data_format=input_data_format) def preprocess( self, @@ -163,6 +181,8 @@ def preprocess( do_flip_channel_order: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -199,6 +219,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -234,20 +263,34 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] # the pretrained checkpoints assume images are BGR, not RGB if do_flip_channel_order: - images = [self.flip_channel_order(image=image) for image in images] + images = [self.flip_channel_order(image=image, input_data_format=input_data_format) for image in images] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index cd0bdf08507564..842414fe6cf87f 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -67,23 +67,28 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -93,7 +98,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -295,6 +300,7 @@ def get_oneformer_resize_output_image_size( size: Union[int, Tuple[int, int], List[int], Tuple[int]], max_size: Optional[int] = None, default_to_square: bool = True, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> tuple: """ Computes the output size given the desired size. @@ -304,16 +310,20 @@ def get_oneformer_resize_output_image_size( The input image. size (`int`, `Tuple[int, int]`, `List[int]`, `Tuple[int]`): The size of the output image. - default_to_square (`bool`, *optional*, defaults to `True`): - Whether to default to square if no size is provided. max_size (`int`, *optional*): The maximum size of the output image. + default_to_square (`bool`, *optional*, defaults to `True`): + Whether to default to square if no size is provided. Returns: `Tuple[int, int]`: The output size. """ output_size = get_resize_output_image_size( - input_image=image, size=size, default_to_square=default_to_square, max_size=max_size + input_image=image, + size=size, + default_to_square=default_to_square, + max_size=max_size, + input_data_format=input_data_format, ) return output_size @@ -442,6 +452,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format=None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -469,17 +480,20 @@ def resize( f" {size.keys()}." ) size = get_oneformer_resize_output_image_size( - image=image, - size=size, - max_size=max_size, - default_to_square=False, + image=image, size=size, max_size=max_size, default_to_square=False, input_data_format=input_data_format + ) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format ) - image = resize(image, size=size, resample=resample, data_format=data_format) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -494,8 +508,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) # Copied from transformers.models.maskformer.image_processing_maskformer.MaskFormerImageProcessor.convert_segmentation_map_to_binary_masks def convert_segmentation_map_to_binary_masks( @@ -528,13 +547,14 @@ def _preprocess( do_normalize: bool = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): if do_resize: - image = self.resize(image, size=size, resample=resample) + image = self.resize(image, size=size, resample=resample, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image, rescale_factor=rescale_factor) + image = self.rescale(image, rescale_factor=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image, mean=image_mean, std=image_std) + image = self.normalize(image, mean=image_mean, std=image_std, input_data_format=input_data_format) return image def _preprocess_image( @@ -549,10 +569,14 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) image = self._preprocess( image=image, do_resize=do_resize, @@ -563,9 +587,10 @@ def _preprocess_image( do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, + input_data_format=input_data_format, ) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def _preprocess_mask( @@ -573,14 +598,19 @@ def _preprocess_mask( segmentation_map: ImageInput, do_resize: bool = None, size: Dict[str, int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """Preprocesses a single mask.""" segmentation_map = to_numpy_array(segmentation_map) # Add channel dimension if missing - needed for certain transformations - added_channel_dim = False if segmentation_map.ndim == 2: added_channel_dim = True segmentation_map = segmentation_map[None, ...] + input_data_format = ChannelDimension.FIRST + else: + added_channel_dim = False + if input_data_format is None: + input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) # TODO: (Amy) # Remork segmentation map processing to include reducing labels and resizing which doesn't # drop segment IDs > 255. @@ -591,6 +621,7 @@ def _preprocess_mask( size=size, do_rescale=False, do_normalize=False, + input_data_format=input_data_format, ) # Remove extra channel dimension if added for processing if added_channel_dim: @@ -615,6 +646,8 @@ def preprocess( do_reduce_labels: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: if "pad_and_return_pixel_mask" in kwargs: @@ -691,13 +724,16 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for image in images ] if segmentation_maps is not None: segmentation_maps = [ - self._preprocess_mask(segmentation_map, do_resize, size) for segmentation_map in segmentation_maps + self._preprocess_mask(segmentation_map, do_resize, size, input_data_format=input_data_format) + for segmentation_map in segmentation_maps ] encoded_inputs = self.encode_inputs( images, @@ -707,6 +743,7 @@ def preprocess( ignore_index, do_reduce_labels, return_tensors, + input_data_format=input_data_format, ) return encoded_inputs @@ -717,18 +754,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -740,6 +783,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -761,17 +805,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -882,6 +937,7 @@ def encode_inputs( ignore_index: Optional[int] = None, reduce_labels: bool = False, return_tensors: Optional[Union[str, TensorType]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): """ Pad images up to the largest image in a batch and create a corresponding `pixel_mask`. @@ -921,6 +977,10 @@ def encode_inputs( If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor` objects. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input + image. + Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: @@ -938,8 +998,14 @@ def encode_inputs( ignore_index = self.ignore_index if ignore_index is None else ignore_index reduce_labels = self.do_reduce_labels if reduce_labels is None else reduce_labels pixel_values_list = [to_numpy_array(pixel_values) for pixel_values in pixel_values_list] - pad_size = get_max_height_width(pixel_values_list) - encoded_inputs = self.pad(pixel_values_list, return_tensors=return_tensors) + + if input_data_format is None: + input_data_format = infer_channel_dimension_format(pixel_values_list[0]) + + pad_size = get_max_height_width(pixel_values_list, input_data_format=input_data_format) + encoded_inputs = self.pad( + pixel_values_list, return_tensors=return_tensors, input_data_format=input_data_format + ) annotations = None if segmentation_maps is not None: diff --git a/src/transformers/models/owlvit/image_processing_owlvit.py b/src/transformers/models/owlvit/image_processing_owlvit.py index c230fda88f266d..f5ebf61166cd62 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit.py +++ b/src/transformers/models/owlvit/image_processing_owlvit.py @@ -33,6 +33,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -169,36 +170,79 @@ def resize( size: Dict[str, int], resample: PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize an image to a certain size. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + The size to resize the image to. Must contain height and width keys. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + The resampling filter to use when resizing the input. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=True) if "height" not in size or "width" not in size: raise ValueError("size dictionary must contain height and width keys") - return resize(image, (size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs) + return resize( + image, + (size["height"], size["width"]), + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def center_crop( self, image: np.ndarray, crop_size: Dict[str, int], data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Center crop an image to a certain size. + + Args: + image (`np.ndarray`): + Image to center crop. + crop_size (`Dict[str, int]`): + The size to center crop the image to. Must contain height and width keys. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ crop_size = get_size_dict(crop_size, default_to_square=True) if "height" not in crop_size or "width" not in crop_size: raise ValueError("crop_size dictionary must contain height and width keys") - return center_crop(image, (crop_size["height"], crop_size["width"]), data_format=data_format, **kwargs) + return center_crop( + image, + (crop_size["height"], crop_size["width"]), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -213,8 +257,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) def preprocess( self, @@ -231,6 +280,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -277,6 +328,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: defaults to the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -312,19 +372,36 @@ def preprocess( # All transformations expect numpy arrays images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image, size=size, resample=resample) for image in images] + images = [ + self.resize(image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image, crop_size=crop_size) for image in images] + images = [ + self.center_crop(image, crop_size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image, rescale_factor=rescale_factor) for image in images] + images = [ + self.rescale(image, rescale_factor=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] encoded_inputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) return encoded_inputs diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index de490ab8d81ba3..0ca916fe16c0e5 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -27,6 +27,7 @@ ImageInput, PILImageResampling, get_image_size, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -117,6 +118,7 @@ def center_crop( crop_size: Dict[str, int], size: Optional[int] = None, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -135,16 +137,24 @@ def center_crop( Size of the image after resizing. If not provided, the self.size attribute will be used. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = self.size if size is None else size size = get_size_dict(size) crop_size = get_size_dict(crop_size, param_name="crop_size") - height, width = get_image_size(image) + height, width = get_image_size(image, channel_dim=input_data_format) min_dim = min(height, width) cropped_height = (size["height"] / crop_size["height"]) * min_dim cropped_width = (size["width"] / crop_size["width"]) * min_dim - return center_crop(image, size=(cropped_height, cropped_width), data_format=data_format, **kwargs) + return center_crop( + image, + size=(cropped_height, cropped_width), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # Copied from transformers.models.vit.image_processing_vit.ViTImageProcessor.resize with PILImageResampling.BILINEAR->PILImageResampling.BICUBIC def resize( @@ -153,6 +163,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -170,6 +181,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -178,7 +196,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -195,6 +220,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -235,6 +262,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop crop_size = crop_size if crop_size is not None else self.crop_size @@ -272,19 +308,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_center_crop: - images = [self.center_crop(image, crop_size, size=size) for image in images] + images = [ + self.center_crop(image, crop_size, size=size, input_data_format=input_data_format) for image in images + ] if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/pix2struct/image_processing_pix2struct.py b/src/transformers/models/pix2struct/image_processing_pix2struct.py index 3e72bc5ad54a48..789666bef6ee5e 100644 --- a/src/transformers/models/pix2struct/image_processing_pix2struct.py +++ b/src/transformers/models/pix2struct/image_processing_pix2struct.py @@ -157,7 +157,9 @@ def render_text( # Adapted from https://github.com/google-research/pix2struct/blob/0e1779af0f4db4b652c1d92b3bbd2550a7399123/pix2struct/preprocessing/preprocessing_utils.py#L87 -def render_header(image: np.ndarray, header: str, **kwargs): +def render_header( + image: np.ndarray, header: str, input_data_format: Optional[Union[str, ChildProcessError]] = None, **kwargs +): """ Renders the input text as a header on the input image. @@ -176,7 +178,7 @@ def render_header(image: np.ndarray, header: str, **kwargs): requires_backends(render_header, "vision") # Convert to PIL image if necessary - image = to_pil_image(image) + image = to_pil_image(image, input_data_format=input_data_format) header_image = render_text(header, **kwargs) new_width = max(header_image.width, image.width) @@ -236,7 +238,14 @@ def __init__( self.max_patches = max_patches self.is_vqa = is_vqa - def extract_flattened_patches(self, image: np.ndarray, max_patches: int, patch_size: dict, **kwargs) -> np.ndarray: + def extract_flattened_patches( + self, + image: np.ndarray, + max_patches: int, + patch_size: dict, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: """ Extract flattened patches from an image. @@ -256,11 +265,11 @@ def extract_flattened_patches(self, image: np.ndarray, max_patches: int, patch_s _check_torch_version() # convert to torch - image = to_channel_dimension_format(image, ChannelDimension.FIRST) + image = to_channel_dimension_format(image, ChannelDimension.FIRST, input_data_format) image = torch.from_numpy(image) patch_height, patch_width = patch_size["height"], patch_size["width"] - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, input_data_format=ChannelDimension.FIRST) # maximize scale s.t. scale = math.sqrt(max_patches * (patch_height / image_height) * (patch_width / image_width)) @@ -312,7 +321,11 @@ def extract_flattened_patches(self, image: np.ndarray, max_patches: int, patch_s return result def normalize( - self, image: np.ndarray, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + self, + image: np.ndarray, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, ) -> np.ndarray: """ Normalize an image. image = (image - image_mean) / image_std. @@ -323,6 +336,11 @@ def normalize( Args: image (`np.ndarray`): Image to normalize. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if image.dtype == np.uint8: image = image.astype(np.float32) @@ -332,7 +350,14 @@ def normalize( std = np.std(image) adjusted_stddev = max(std, 1.0 / math.sqrt(np.prod(image.shape))) - return normalize(image, mean=mean, std=adjusted_stddev, **kwargs) + return normalize( + image, + mean=mean, + std=adjusted_stddev, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -344,6 +369,8 @@ def preprocess( patch_size: Optional[Dict[str, int]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> ImageInput: """ @@ -374,6 +401,20 @@ def preprocess( - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. if `input_data_format` is unset. """ do_normalize = do_normalize if do_normalize is not None else self.do_normalize do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb @@ -399,6 +440,10 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if is_vqa: if header_text is None: raise ValueError("A header text must be provided for VQA models.") @@ -414,11 +459,13 @@ def preprocess( ] if do_normalize: - images = [self.normalize(image=image) for image in images] + images = [self.normalize(image=image, input_data_format=input_data_format) for image in images] # convert to torch tensor and permute images = [ - self.extract_flattened_patches(image=image, max_patches=max_patches, patch_size=patch_size) + self.extract_flattened_patches( + image=image, max_patches=max_patches, patch_size=patch_size, input_data_format=input_data_format + ) for image in images ] diff --git a/src/transformers/models/poolformer/image_processing_poolformer.py b/src/transformers/models/poolformer/image_processing_poolformer.py index 03774f401870d5..d51f8304cf88b4 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer.py +++ b/src/transformers/models/poolformer/image_processing_poolformer.py @@ -30,6 +30,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -137,6 +138,7 @@ def resize( crop_pct: Optional[float] = None, resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -166,6 +168,8 @@ def resize( Resampling filter to use when resizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size and ("height" not in size or "width" not in size): @@ -181,16 +185,27 @@ def resize( else: raise ValueError("Invalid size for resize: {}".format(size)) - output_size = get_resize_output_image_size(image, size=scale_size, default_to_square=False) + output_size = get_resize_output_image_size( + image, size=scale_size, default_to_square=False, input_data_format=input_data_format + ) else: if "shortest_edge" in size: - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) elif "height" in size and "width" in size: output_size = (size["height"], size["width"]) else: raise ValueError("Invalid size for resize: {}".format(size)) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -208,6 +223,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -250,6 +267,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize crop_pct = crop_pct if crop_pct is not None else self.crop_pct @@ -289,19 +315,38 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, crop_pct=crop_pct, resample=resample) for image in images] + images = [ + self.resize( + image=image, size=size, crop_pct=crop_pct, resample=resample, input_data_format=input_data_format + ) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] - - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/pvt/image_processing_pvt.py b/src/transformers/models/pvt/image_processing_pvt.py index d0a7cc99a45960..69a7cf6059984c 100644 --- a/src/transformers/models/pvt/image_processing_pvt.py +++ b/src/transformers/models/pvt/image_processing_pvt.py @@ -26,6 +26,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -100,6 +101,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -117,6 +119,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -125,7 +134,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -140,6 +156,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ): """ @@ -178,6 +196,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -207,16 +234,31 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size_dict, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size_dict, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 2ee0b47250cc51..5fe7d9ec0a7eb2 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -143,6 +143,7 @@ def pad_image( image: np.ndarray, pad_size: Dict[str, int], data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -156,14 +157,22 @@ def pad_image( data_format (`str` or `ChannelDimension`, *optional*): The data format of the image. Can be either "channels_first" or "channels_last". If `None`, the `data_format` of the `image` will be used. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ output_height, output_width = pad_size["height"], pad_size["width"] - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) pad_width = output_width - input_width pad_height = output_height - input_height - padded_image = pad(image, ((0, pad_height), (0, pad_width)), data_format=data_format, **kwargs) + padded_image = pad( + image, + ((0, pad_height), (0, pad_width)), + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) return padded_image def _get_preprocess_shape(self, old_shape: Tuple[int, int], longest_edge: int): @@ -183,6 +192,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -202,15 +212,28 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + Returns: `np.ndarray`: The resized image. """ size = get_size_dict(size) if "longest_edge" not in size: raise ValueError(f"The `size` dictionary must contain the key `longest_edge`. Got {size.keys()}") - input_size = get_image_size(image) + input_size = get_image_size(image, channel_dim=input_data_format) output_height, output_width = self._get_preprocess_shape(input_size, size["longest_edge"]) - return resize(image, size=(output_height, output_width), resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=(output_height, output_width), + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -228,6 +251,8 @@ def preprocess( do_convert_rgb: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ): """ @@ -272,6 +297,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -314,23 +348,40 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] - original_sizes = [get_image_size(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + + original_sizes = [get_image_size(image, channel_dim=input_data_format) for image in images] if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] - reshaped_input_sizes = [get_image_size(image) for image in images] + reshaped_input_sizes = [get_image_size(image, channel_dim=input_data_format) for image in images] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] if do_pad: - images = [self.pad_image(image=image, pad_size=pad_size) for image in images] + images = [ + self.pad_image(image=image, pad_size=pad_size, input_data_format=input_data_format) for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] encoded_outputs = BatchFeature( data={ "pixel_values": images, diff --git a/src/transformers/models/segformer/image_processing_segformer.py b/src/transformers/models/segformer/image_processing_segformer.py index 4424f5dd646f45..9697a70119b3e1 100644 --- a/src/transformers/models/segformer/image_processing_segformer.py +++ b/src/transformers/models/segformer/image_processing_segformer.py @@ -27,6 +27,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -135,6 +136,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -152,6 +154,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -160,7 +169,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # Copied from transformers.models.beit.image_processing_beit.BeitImageProcessor.reduce_label def reduce_label(self, label: ImageInput) -> np.ndarray: @@ -183,18 +199,19 @@ def _preprocess( rescale_factor: Optional[float] = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ): if do_reduce_labels: image = self.reduce_label(image) if do_resize: - image = self.resize(image=image, size=size, resample=resample) + image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image=image, scale=rescale_factor) + image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image=image, mean=image_mean, std=image_std) + image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) return image @@ -210,10 +227,14 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) image = self._preprocess( image=image, do_reduce_labels=False, @@ -225,9 +246,10 @@ def _preprocess_image( do_normalize=do_normalize, image_mean=image_mean, image_std=image_std, + input_data_format=input_data_format, ) if data_format is not None: - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def _preprocess_mask( @@ -236,14 +258,19 @@ def _preprocess_mask( do_reduce_labels: bool = None, do_resize: bool = None, size: Dict[str, int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """Preprocesses a single mask.""" segmentation_map = to_numpy_array(segmentation_map) # Add channel dimension if missing - needed for certain transformations - added_channel_dim = False if segmentation_map.ndim == 2: added_channel_dim = True segmentation_map = segmentation_map[None, ...] + input_data_format = ChannelDimension.FIRST + else: + added_channel_dim = False + if input_data_format is None: + input_data_format = infer_channel_dimension_format(segmentation_map, num_channels=1) # reduce zero label if needed segmentation_map = self._preprocess( image=segmentation_map, @@ -253,6 +280,7 @@ def _preprocess_mask( size=size, do_rescale=False, do_normalize=False, + input_data_format=input_data_format, ) # Remove extra channel dimension if added for processing if added_channel_dim: @@ -284,6 +312,8 @@ def preprocess( do_reduce_labels: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -326,6 +356,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -374,6 +413,8 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in images ] @@ -387,6 +428,7 @@ def preprocess( do_reduce_labels=do_reduce_labels, do_resize=do_resize, size=size, + input_data_format=input_data_format, ) for segmentation_map in segmentation_maps ] diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 0558fd467bf6ee..0380d5b40e6242 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -20,7 +20,14 @@ from ...image_processing_utils import BaseImageProcessor, BatchFeature from ...image_transforms import get_image_size, pad, to_channel_dimension_format -from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images +from ...image_utils import ( + ChannelDimension, + ImageInput, + infer_channel_dimension_format, + make_list_of_images, + to_numpy_array, + valid_images, +) from ...utils import TensorType, logging @@ -57,7 +64,13 @@ def __init__( self.do_pad = do_pad self.pad_size = pad_size - def pad(self, image: np.ndarray, size: int, data_format: Optional[Union[str, ChannelDimension]] = None): + def pad( + self, + image: np.ndarray, + size: int, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ): """ Pad an image to make the height and width divisible by `size`. @@ -71,6 +84,11 @@ def pad(self, image: np.ndarray, size: int, data_format: Optional[Union[str, Cha image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. Returns: `np.ndarray`: The padded image. @@ -79,7 +97,13 @@ def pad(self, image: np.ndarray, size: int, data_format: Optional[Union[str, Cha pad_height = (old_height // size + 1) * size - old_height pad_width = (old_width // size + 1) * size - old_width - return pad(image, ((0, pad_height), (0, pad_width)), mode="symmetric", data_format=data_format) + return pad( + image, + ((0, pad_height), (0, pad_width)), + mode="symmetric", + data_format=data_format, + input_data_format=input_data_format, + ) def preprocess( self, @@ -90,6 +114,8 @@ def preprocess( pad_size: Optional[int] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ): """ @@ -109,7 +135,8 @@ def preprocess( return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be one of: - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of typ, input_data_format=input_data_formate + `tf.Tensor`. - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. @@ -118,6 +145,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. if `input_data_format` is unset. """ do_rescale = do_rescale if do_rescale is not None else self.do_rescale rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor @@ -138,13 +174,22 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_pad: - images = [self.pad(image, size=pad_size) for image in images] + images = [self.pad(image, size=pad_size, input_data_format=input_data_format) for image in images] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/tvlt/image_processing_tvlt.py b/src/transformers/models/tvlt/image_processing_tvlt.py index 0b4d928a3e01da..9ae4075e709baa 100644 --- a/src/transformers/models/tvlt/image_processing_tvlt.py +++ b/src/transformers/models/tvlt/image_processing_tvlt.py @@ -29,6 +29,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, is_valid_image, to_numpy_array, valid_images, @@ -155,6 +156,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -171,15 +173,26 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" in size: - output_size = get_resize_output_image_size(image, size["shortest_edge"], default_to_square=False) + output_size = get_resize_output_image_size( + image, size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) elif "height" in size and "width" in size: output_size = (size["height"], size["width"]) else: raise ValueError(f"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}") - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def _preprocess_image( self, @@ -195,6 +208,8 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -212,18 +227,21 @@ def _preprocess_image( # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + if do_resize: - image = self.resize(image=image, size=size, resample=resample) + image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) if do_center_crop: - image = self.center_crop(image, size=crop_size) + image = self.center_crop(image, size=crop_size, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image=image, scale=rescale_factor) + image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image=image, mean=image_mean, std=image_std) - image = to_channel_dimension_format(image, data_format) + image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def preprocess( @@ -244,6 +262,8 @@ def preprocess( is_mixed: bool = False, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -291,6 +311,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the inferred channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. if `input_data_format` is unset. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: @@ -361,6 +390,8 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in video ] diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index 3992e5c5a79d65..6e63998e2a2545 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -30,6 +30,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, is_valid_image, to_numpy_array, valid_images, @@ -134,6 +135,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -150,15 +152,26 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" in size: - output_size = get_resize_output_image_size(image, size["shortest_edge"], default_to_square=False) + output_size = get_resize_output_image_size( + image, size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) elif "height" in size and "width" in size: output_size = (size["height"], size["width"]) else: raise ValueError(f"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}") - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def _preprocess_image( self, @@ -174,6 +187,8 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -191,19 +206,22 @@ def _preprocess_image( # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + if do_resize: - image = self.resize(image=image, size=size, resample=resample) + image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) if do_center_crop: - image = self.center_crop(image, size=crop_size) + image = self.center_crop(image, size=crop_size, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image=image, scale=rescale_factor) + image = self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image=image, mean=image_mean, std=image_std) + image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def preprocess( @@ -221,6 +239,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -262,6 +282,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the inferred channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -300,6 +329,8 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in video ] diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index c45c2c6b25a0b7..08c2c8367c0e4e 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -49,7 +49,9 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: return [max(values_i) for values_i in zip(*values)] -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -59,31 +61,38 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) def get_resize_output_image_size( - input_image: np.ndarray, shorter: int = 800, longer: int = 1333, size_divisor: int = 32 + input_image: np.ndarray, + shorter: int = 800, + longer: int = 1333, + size_divisor: int = 32, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: - input_height, input_width = get_image_size(input_image) + input_height, input_width = get_image_size(input_image, input_data_format=input_data_format) min_size, max_size = shorter, longer scale = min_size / min(input_height, input_width) @@ -200,6 +209,7 @@ def resize( size_divisor: int = 32, resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -220,14 +230,25 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}") shorter = size["shortest_edge"] longer = int(1333 / 800 * shorter) - output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, shorter=shorter, longer=longer, size_divisor=size_divisor, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor._pad_image def _pad_image( @@ -236,18 +257,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -259,6 +286,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -280,17 +308,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -310,6 +349,8 @@ def preprocess( do_pad: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -353,6 +394,15 @@ def preprocess( The channel dimension format for the output image. Can be one of: - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size_divisor = size_divisor if size_divisor is not None else self.size_divisor @@ -387,21 +437,45 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: images = [ - self.resize(image=image, size=size, size_divisor=size_divisor, resample=resample) for image in images + self.resize( + image=image, + size=size, + size_divisor=size_divisor, + resample=resample, + input_data_format=input_data_format, + ) + for image in images ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format( + image, data_format, input_channel_dim=input_data_format, input_data_format=input_data_format + ) + for image in images + ] if do_pad: - encoded_outputs = self.pad(images, return_pixel_mask=True, return_tensors=return_tensors) + encoded_outputs = self.pad( + images, return_pixel_mask=True, return_tensors=return_tensors, input_data_format=input_data_format + ) else: encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/vit/image_processing_vit.py b/src/transformers/models/vit/image_processing_vit.py index 8004ac32e07db2..926cd4be847f9d 100644 --- a/src/transformers/models/vit/image_processing_vit.py +++ b/src/transformers/models/vit/image_processing_vit.py @@ -26,6 +26,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -99,6 +100,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -116,6 +118,13 @@ def resize( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. Returns: `np.ndarray`: The resized image. @@ -124,7 +133,14 @@ def resize( if "height" not in size or "width" not in size: raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") output_size = (size["height"], size["width"]) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -139,6 +155,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ): """ @@ -177,6 +195,15 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -206,16 +233,31 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size_dict, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size_dict, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py index 26b1e53adaebf9..d1484cd9717df0 100644 --- a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py +++ b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py @@ -31,6 +31,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, make_list_of_images, to_numpy_array, valid_images, @@ -125,6 +126,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -140,12 +142,23 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" not in size: raise ValueError(f"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}") - output_size = get_resize_output_image_size(image, size=size["shortest_edge"], default_to_square=False) - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + output_size = get_resize_output_image_size( + image, size=size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) def preprocess( self, @@ -163,6 +176,8 @@ def preprocess( do_convert_rgb: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -208,6 +223,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: defaults to the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. if `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -250,19 +274,36 @@ def preprocess( # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + if do_resize: - images = [self.resize(image=image, size=size, resample=resample) for image in images] + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] + images = [ + self.center_crop(image=image, size=crop_size, input_data_format=input_data_format) for image in images + ] if do_rescale: - images = [self.rescale(image=image, scale=rescale_factor) for image in images] + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] if do_normalize: - images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] data = {"pixel_values": images} return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/vivit/image_processing_vivit.py b/src/transformers/models/vivit/image_processing_vivit.py index 0790c5d82bb5a9..83c782e5f53fea 100644 --- a/src/transformers/models/vivit/image_processing_vivit.py +++ b/src/transformers/models/vivit/image_processing_vivit.py @@ -33,6 +33,7 @@ ChannelDimension, ImageInput, PILImageResampling, + infer_channel_dimension_format, is_valid_image, to_numpy_array, valid_images, @@ -141,6 +142,7 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ @@ -157,15 +159,26 @@ def resize( Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ size = get_size_dict(size, default_to_square=False) if "shortest_edge" in size: - output_size = get_resize_output_image_size(image, size["shortest_edge"], default_to_square=False) + output_size = get_resize_output_image_size( + image, size["shortest_edge"], default_to_square=False, input_data_format=input_data_format + ) elif "height" in size and "width" in size: output_size = (size["height"], size["width"]) else: raise ValueError(f"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}") - return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # Copied from transformers.models.efficientnet.image_processing_efficientnet.EfficientNetImageProcessor.rescale def rescale( @@ -174,6 +187,7 @@ def rescale( scale: Union[int, float], offset: bool = True, data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ): """ @@ -195,8 +209,12 @@ def rescale( Whether to scale the image in both negative and positive directions. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - rescaled_image = rescale(image, scale=scale, data_format=data_format, **kwargs) + rescaled_image = rescale( + image, scale=scale, data_format=data_format, input_data_format=input_data_format, **kwargs + ) if offset: rescaled_image = rescaled_image - 1 @@ -218,6 +236,8 @@ def _preprocess_image( image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -238,19 +258,22 @@ def _preprocess_image( # All transformations expect numpy arrays. image = to_numpy_array(image) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + if do_resize: - image = self.resize(image=image, size=size, resample=resample) + image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) if do_center_crop: - image = self.center_crop(image, size=crop_size) + image = self.center_crop(image, size=crop_size, input_data_format=input_data_format) if do_rescale: - image = self.rescale(image=image, scale=rescale_factor, offset=offset) + image = self.rescale(image=image, scale=rescale_factor, offset=offset, input_data_format=input_data_format) if do_normalize: - image = self.normalize(image=image, mean=image_mean, std=image_std) + image = self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) - image = to_channel_dimension_format(image, data_format) + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) return image def preprocess( @@ -269,6 +292,8 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -312,6 +337,15 @@ def preprocess( - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `ChannelDimension.LAST`: image in (height, width, num_channels) format. - Unset: Use the inferred channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -352,6 +386,8 @@ def preprocess( image_mean=image_mean, image_std=image_std, data_format=data_format, + input_data_format=input_data_format, + num_channels=num_channels, ) for img in video ] diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index 209b0d441d8459..6776e8d8f14821 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -88,18 +88,21 @@ class AnnotionFormat(ExplicitEnum): # Copied from transformers.models.detr.image_processing_detr.get_max_height_width -def get_max_height_width(images: List[np.ndarray]) -> List[int]: +def get_max_height_width( + images: List[np.ndarray], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> List[int]: """ Get the maximum height and width across all images in a batch. """ - input_channel_dimension = infer_channel_dimension_format(images[0]) + if input_data_format is None: + input_data_format = infer_channel_dimension_format(images[0]) - if input_channel_dimension == ChannelDimension.FIRST: + if input_data_format == ChannelDimension.FIRST: _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: + elif input_data_format == ChannelDimension.LAST: max_height, max_width, _ = max_across_indices([img.shape for img in images]) else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + raise ValueError(f"Invalid channel dimension format: {input_data_format}") return (max_height, max_width) @@ -137,7 +140,10 @@ def get_size_with_aspect_ratio(image_size, size, max_size=None) -> Tuple[int, in # Copied from transformers.models.detr.image_processing_detr.get_resize_output_image_size def get_resize_output_image_size( - input_image: np.ndarray, size: Union[int, Tuple[int, int], List[int]], max_size: Optional[int] = None + input_image: np.ndarray, + size: Union[int, Tuple[int, int], List[int]], + max_size: Optional[int] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: """ Computes the output image size given the input image size and the desired output size. If the desired output size @@ -151,8 +157,10 @@ def get_resize_output_image_size( The desired output size. max_size (`int`, *optional*): The maximum allowed output size. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image) + image_size = get_image_size(input_image, input_data_format=input_data_format) if isinstance(size, (list, tuple)): return size @@ -222,7 +230,9 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: # Copied from transformers.models.detr.image_processing_detr.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: +def make_pixel_mask( + image: np.ndarray, output_size: Tuple[int, int], input_data_format: Optional[Union[str, ChannelDimension]] = None +) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -232,7 +242,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr output_size (`Tuple[int, int]`): Output size of the mask. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) mask = np.zeros(output_size, dtype=np.int64) mask[:input_height, :input_width] = 1 return mask @@ -274,11 +284,16 @@ def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndar # Copied from transformers.models.detr.image_processing_detr.prepare_coco_detection_annotation -def prepare_coco_detection_annotation(image, target, return_segmentation_masks: bool = False): +def prepare_coco_detection_annotation( + image, + target, + return_segmentation_masks: bool = False, + input_data_format: Optional[Union[ChannelDimension, str]] = None, +): """ Convert the target in COCO format into the format expected by DETR. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) image_id = target["image_id"] image_id = np.asarray([image_id], dtype=np.int64) @@ -363,12 +378,16 @@ def masks_to_boxes(masks: np.ndarray) -> np.ndarray: # Copied from transformers.models.detr.image_processing_detr.prepare_coco_panoptic_annotation with DETR->YOLOS def prepare_coco_panoptic_annotation( - image: np.ndarray, target: Dict, masks_path: Union[str, pathlib.Path], return_masks: bool = True + image: np.ndarray, + target: Dict, + masks_path: Union[str, pathlib.Path], + return_masks: bool = True, + input_data_format: Union[ChannelDimension, str] = None, ) -> Dict: """ Prepare a coco panoptic annotation for YOLOS. """ - image_height, image_width = get_image_size(image) + image_height, image_width = get_image_size(image, channel_dim=input_data_format) annotation_path = pathlib.Path(masks_path) / target["file_name"] new_target = {} @@ -751,6 +770,7 @@ def prepare_annotation( format: Optional[AnnotionFormat] = None, return_segmentation_masks: bool = None, masks_path: Optional[Union[str, pathlib.Path]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Dict: """ Prepare an annotation for feeding into DETR model. @@ -759,11 +779,17 @@ def prepare_annotation( if format == AnnotionFormat.COCO_DETECTION: return_segmentation_masks = False if return_segmentation_masks is None else return_segmentation_masks - target = prepare_coco_detection_annotation(image, target, return_segmentation_masks) + target = prepare_coco_detection_annotation( + image, target, return_segmentation_masks, input_data_format=input_data_format + ) elif format == AnnotionFormat.COCO_PANOPTIC: return_segmentation_masks = True if return_segmentation_masks is None else return_segmentation_masks target = prepare_coco_panoptic_annotation( - image, target, masks_path=masks_path, return_masks=return_segmentation_masks + image, + target, + masks_path=masks_path, + return_masks=return_segmentation_masks, + input_data_format=input_data_format, ) else: raise ValueError(f"Format {format} is not supported.") @@ -801,11 +827,26 @@ def resize( size: Dict[str, int], resample: PILImageResampling = PILImageResampling.BILINEAR, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs, ) -> np.ndarray: """ Resize the image to the given size. Size can be `min_size` (scalar) or `(height, width)` tuple. If size is an int, smaller edge of the image will be matched to this number. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Dictionary containing the size to resize to. Can contain the keys `shortest_edge` and `longest_edge` or + `height` and `width`. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use if resizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if "max_size" in kwargs: logger.warning_once( @@ -817,7 +858,9 @@ def resize( max_size = None size = get_size_dict(size, max_size=max_size, default_to_square=False) if "shortest_edge" in size and "longest_edge" in size: - size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"]) + size = get_resize_output_image_size( + image, size["shortest_edge"], size["longest_edge"], input_data_format=input_data_format + ) elif "height" in size and "width" in size: size = (size["height"], size["width"]) else: @@ -825,7 +868,9 @@ def resize( "Size must contain 'height' and 'width' keys or 'shortest_edge' and 'longest_edge' keys. Got" f" {size.keys()}." ) - image = resize(image, size=size, resample=resample, data_format=data_format) + image = resize( + image, size=size, resample=resample, data_format=data_format, input_data_format=input_data_format, **kwargs + ) return image # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.resize_annotation @@ -844,7 +889,11 @@ def resize_annotation( # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.rescale def rescale( - self, image: np.ndarray, rescale_factor: float, data_format: Optional[Union[str, ChannelDimension]] = None + self, + image: np.ndarray, + rescale_factor: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Rescale the image by the given factor. image = image * rescale_factor. @@ -859,8 +908,13 @@ def rescale( image is used. Can be one of: - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the input image. If unset, is inferred from the input image. Can be + one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ - return rescale(image, rescale_factor, data_format=data_format) + return rescale(image, rescale_factor, data_format=data_format, input_data_format=input_data_format) # Copied from transformers.models.detr.image_processing_detr.DetrImageProcessor.normalize_annotation def normalize_annotation(self, annotation: Dict, image_size: Tuple[int, int]) -> Dict: @@ -877,18 +931,24 @@ def _pad_image( output_size: Tuple[int, int], constant_values: Union[float, Iterable[float]] = 0, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """ Pad an image with zeros to the given size. """ - input_height, input_width = get_image_size(image) + input_height, input_width = get_image_size(image, channel_dim=input_data_format) output_height, output_width = output_size pad_bottom = output_height - input_height pad_right = output_width - input_width padding = ((0, pad_bottom), (0, pad_right)) padded_image = pad( - image, padding, mode=PaddingMode.CONSTANT, constant_values=constant_values, data_format=data_format + image, + padding, + mode=PaddingMode.CONSTANT, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, ) return padded_image @@ -900,6 +960,7 @@ def pad( return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> BatchFeature: """ Pads a batch of images to the bottom and right of the image with zeros to the size of largest height and width @@ -921,17 +982,28 @@ def pad( - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ - pad_size = get_max_height_width(images) + pad_size = get_max_height_width(images, input_data_format=input_data_format) padded_images = [ - self._pad_image(image, pad_size, constant_values=constant_values, data_format=data_format) + self._pad_image( + image, + pad_size, + constant_values=constant_values, + data_format=data_format, + input_data_format=input_data_format, + ) for image in images ] data = {"pixel_values": padded_images} if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + masks = [ + make_pixel_mask(image=image, output_size=pad_size, input_data_format=input_data_format) + for image in images + ] data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -954,6 +1026,8 @@ def preprocess( format: Optional[Union[str, AnnotionFormat]] = None, return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1001,6 +1075,15 @@ def preprocess( Type of tensors to return. If `None`, will return the list of images. data_format (`str` or `ChannelDimension`, *optional*, defaults to self.data_format): The channel dimension format of the image. If not provided, it will be the same as the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + num_channels (`int`, *optional*): + The number of channels in the input image, used to infer the channel dimension format if + `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1085,13 +1168,22 @@ def preprocess( # All transformations expect numpy arrays images = [to_numpy_array(image) for image in images] + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: prepared_images = [] prepared_annotations = [] for image, target in zip(images, annotations): target = self.prepare_annotation( - image, target, format, return_segmentation_masks=return_segmentation_masks, masks_path=masks_path + image, + target, + format, + return_segmentation_masks=return_segmentation_masks, + masks_path=masks_path, + input_data_format=input_data_format, ) prepared_images.append(image) prepared_annotations.append(target) @@ -1105,7 +1197,9 @@ def preprocess( resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): orig_size = get_image_size(image) - resized_image = self.resize(image, size=size, max_size=max_size, resample=resample) + resized_image = self.resize( + image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format + ) resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) resized_images.append(resized_image) resized_annotations.append(resized_annotation) @@ -1113,13 +1207,18 @@ def preprocess( annotations = resized_annotations del resized_images, resized_annotations else: - images = [self.resize(image, size=size, resample=resample) for image in images] + images = [ + self.resize(image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] if do_rescale: - images = [self.rescale(image, rescale_factor) for image in images] + images = [self.rescale(image, rescale_factor, input_data_format=input_data_format) for image in images] if do_normalize: - images = [self.normalize(image, image_mean, image_std) for image in images] + images = [ + self.normalize(image, image_mean, image_std, input_data_format=input_data_format) for image in images + ] if annotations is not None: annotations = [ self.normalize_annotation(annotation, get_image_size(image)) @@ -1127,9 +1226,12 @@ def preprocess( ] if do_pad: - data = self.pad(images, data_format=data_format) + data = self.pad(images, data_format=data_format, input_data_format=input_data_format) else: - images = [to_channel_dimension_format(image, data_format) for image in images] + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + for image in images + ] data = {"pixel_values": images} encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) From 207019c766997fee95e1a4a21d3dec3761a14420 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Fri, 11 Aug 2023 14:48:51 +0000 Subject: [PATCH 7/9] Resolve tests and tidy up --- .../models/beit/image_processing_beit.py | 4 +- .../image_processing_bridgetower.py | 4 +- .../image_processing_conditional_detr.py | 10 +++-- .../convnext/image_processing_convnext.py | 9 +++- .../image_processing_deformable_detr.py | 10 +++-- .../models/deta/image_processing_deta.py | 10 +++-- .../models/detr/image_processing_detr.py | 10 +++-- .../models/dpt/image_processing_dpt.py | 4 +- .../image_processing_efficientnet.py | 2 +- .../layoutlmv2/image_processing_layoutlmv2.py | 11 +++-- .../layoutlmv3/image_processing_layoutlmv3.py | 11 +++-- .../image_processing_mask2former.py | 5 ++- .../pix2struct/image_processing_pix2struct.py | 2 +- .../models/sam/image_processing_sam.py | 19 ++++++--- .../swin2sr/image_processing_swin2sr.py | 2 +- .../models/vilt/image_processing_vilt.py | 9 ++-- .../models/yolos/image_processing_yolos.py | 8 ++-- .../models/blip/test_image_processing_blip.py | 10 ++++- .../test_image_processing_chinese_clip.py | 14 ++++++- .../flava/test_image_processing_flava.py | 5 +++ .../models/glpn/test_image_processing_glpn.py | 15 +++++++ .../test_image_processing_imagegpt.py | 4 ++ .../test_image_processing_pix2struct.py | 38 +++++++++++++++++ .../swin2sr/test_image_processing_swin2sr.py | 18 ++++++++ .../models/tvlt/test_image_processor_tvlt.py | 41 +++++++++++++++++++ .../test_image_processing_videomae.py | 27 ++++++++++++ .../vivit/test_image_processing_vivit.py | 27 ++++++++++++ tests/test_image_processing_common.py | 33 +++++++++++++++ 28 files changed, 312 insertions(+), 50 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index 59ea1d88dc8c26..b983a1442996a2 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -238,9 +238,7 @@ def _preprocess_image( # All transformations expect numpy arrays. image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format( - image, input_data_format=input_data_format, num_channels=num_channels - ) + input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) image = self._preprocess( image, do_reduce_labels=False, diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 81541e0de008eb..11db01bf569d90 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -95,7 +95,7 @@ def get_resize_output_image_size( size_divisor: int = 32, input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: - input_height, input_width = get_image_size(input_image, input_data_format=input_data_format) + input_height, input_width = get_image_size(input_image, input_data_format) min_size, max_size = shorter, longer scale = min_size / min(input_height, input_width) @@ -508,7 +508,7 @@ def preprocess( if do_pad: encoded_outputs = self.pad( - images, return_pixel_mask=True, return_tensors=return_tensors, input_data_format=input_data_format + images, return_pixel_mask=True, return_tensors=return_tensors, input_data_format=data_format ) else: encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index 3f973103be9a9d..39182d9f0993ac 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -144,7 +144,7 @@ def get_resize_output_image_size( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image, input_data_format=input_data_format) + image_size = get_image_size(input_image, input_data_format) if isinstance(size, (list, tuple)): return size @@ -1291,12 +1291,12 @@ def preprocess( if annotations is not None: resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): - orig_size = get_image_size(image) + orig_size = get_image_size(image, input_data_format) resized_image = self.resize( image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format ) resized_annotation = self.resize_annotation( - target, orig_size, get_image_size(resized_image), input_data_format=input_data_format + target, orig_size, get_image_size(resized_image, input_data_format) ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) @@ -1318,7 +1318,9 @@ def preprocess( ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image), input_data_format=input_data_format) + self.normalize_annotation( + annotation, get_image_size(image, input_data_format), input_data_format=input_data_format + ) for annotation, image in zip(annotations, images) ] diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index 4b687a48993ea4..5864c865996473 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -154,7 +154,14 @@ def resize( resize_size = get_resize_output_image_size( image, size=resize_shortest_edge, default_to_square=False, input_data_format=input_data_format ) - image = resize(image=image, size=resize_size, resample=resample, data_format=data_format, **kwargs) + image = resize( + image=image, + size=resize_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) # then crop to (shortest_edge, shortest_edge) return center_crop( image=image, diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index f2dc0609bc9156..ce2c3b31929456 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -143,7 +143,7 @@ def get_resize_output_image_size( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image, input_data_format=input_data_format) + image_size = get_image_size(input_image, input_data_format) if isinstance(size, (list, tuple)): return size @@ -1289,12 +1289,12 @@ def preprocess( if annotations is not None: resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): - orig_size = get_image_size(image) + orig_size = get_image_size(image, input_data_format) resized_image = self.resize( image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format ) resized_annotation = self.resize_annotation( - target, orig_size, get_image_size(resized_image), input_data_format=input_data_format + target, orig_size, get_image_size(resized_image, input_data_format) ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) @@ -1316,7 +1316,9 @@ def preprocess( ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image), input_data_format=input_data_format) + self.normalize_annotation( + annotation, get_image_size(image, input_data_format), input_data_format=input_data_format + ) for annotation, image in zip(annotations, images) ] diff --git a/src/transformers/models/deta/image_processing_deta.py b/src/transformers/models/deta/image_processing_deta.py index ae8805f01a6edd..195f627f16a26f 100644 --- a/src/transformers/models/deta/image_processing_deta.py +++ b/src/transformers/models/deta/image_processing_deta.py @@ -135,7 +135,7 @@ def get_resize_output_image_size( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image, input_data_format=input_data_format) + image_size = get_image_size(input_image, input_data_format) if isinstance(size, (list, tuple)): return size @@ -953,11 +953,13 @@ def preprocess( if annotations is not None: resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): - orig_size = get_image_size(image) + orig_size = get_image_size(image, input_data_format) resized_image = self.resize( image, size=size, resample=resample, input_data_format=input_data_format ) - resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) + resized_annotation = self.resize_annotation( + target, orig_size, get_image_size(resized_image, input_data_format) + ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) images = resized_images @@ -978,7 +980,7 @@ def preprocess( ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image)) + self.normalize_annotation(annotation, get_image_size(image, input_data_format)) for annotation, image in zip(annotations, images) ] diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 7115c3aa5a014b..6f4dc4ddf33e25 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -141,7 +141,7 @@ def get_resize_output_image_size( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image, input_data_format=input_data_format) + image_size = get_image_size(input_image, input_data_format) if isinstance(size, (list, tuple)): return size @@ -1261,12 +1261,12 @@ def preprocess( if annotations is not None: resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): - orig_size = get_image_size(image) + orig_size = get_image_size(image, input_data_format) resized_image = self.resize( image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format ) resized_annotation = self.resize_annotation( - target, orig_size, get_image_size(resized_image), input_data_format=input_data_format + target, orig_size, get_image_size(resized_image, input_data_format) ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) @@ -1288,7 +1288,9 @@ def preprocess( ] if annotations is not None: annotations = [ - self.normalize_annotation(annotation, get_image_size(image), input_data_format=input_data_format) + self.normalize_annotation( + annotation, get_image_size(image, input_data_format), input_data_format=input_data_format + ) for annotation, image in zip(annotations, images) ] diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index 00da6f5b4690d6..ac78fc6e14b8c7 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -68,7 +68,7 @@ def constraint_to_multiple_of(val, multiple, min_val=0, max_val=None): output_size = (output_size, output_size) if isinstance(output_size, int) else output_size - input_height, input_width = get_image_size(input_image, input_data_format=input_data_format) + input_height, input_width = get_image_size(input_image, input_data_format) output_height, output_width = output_size # determine new height and width @@ -188,7 +188,7 @@ def resize( input_data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. """ - size = get_size_dict(size, input_data_format=input_data_format) + size = get_size_dict(size) if "height" not in size or "width" not in size: raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}") output_size = get_resize_output_image_size( diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index 7f445be979601f..19c019bf9a1122 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -350,7 +350,7 @@ def preprocess( if include_top: images = [ - self.normalize(image=image, mean=[0, 0, 0], std=image_std, input_data_format=input_data_format) + self.normalize(image=image, mean=0, std=image_std, input_data_format=input_data_format) for image in images ] diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index e816ba36bd4020..4a46f26ae7ba21 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -51,12 +51,17 @@ def normalize_box(box, width, height): ] -def apply_tesseract(image: np.ndarray, lang: Optional[str], tesseract_config: Optional[str] = None): +def apply_tesseract( + image: np.ndarray, + lang: Optional[str], + tesseract_config: Optional[str] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, +): """Applies Tesseract OCR on a document image, and returns recognized words + normalized bounding boxes.""" tesseract_config = tesseract_config if tesseract_config is not None else "" # apply OCR - pil_image = to_pil_image(image) + pil_image = to_pil_image(image, input_data_format=input_data_format) image_width, image_height = pil_image.size data = pytesseract.image_to_data(pil_image, lang=lang, output_type="dict", config=tesseract_config) words, left, top, width, height = data["text"], data["left"], data["top"], data["width"], data["height"] @@ -260,7 +265,7 @@ def preprocess( words_batch = [] boxes_batch = [] for image in images: - words, boxes = apply_tesseract(image, ocr_lang, tesseract_config) + words, boxes = apply_tesseract(image, ocr_lang, tesseract_config, input_data_format=input_data_format) words_batch.append(words) boxes_batch.append(boxes) diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index f770011d1b742c..55d4f5824b9439 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -53,11 +53,16 @@ def normalize_box(box, width, height): ] -def apply_tesseract(image: np.ndarray, lang: Optional[str], tesseract_config: Optional[str]): +def apply_tesseract( + image: np.ndarray, + lang: Optional[str], + tesseract_config: Optional[str], + input_data_format: Optional[Union[ChannelDimension, str]] = None, +): """Applies Tesseract OCR on a document image, and returns recognized words + normalized bounding boxes.""" # apply OCR - pil_image = to_pil_image(image) + pil_image = to_pil_image(image, input_data_format=input_data_format) image_width, image_height = pil_image.size data = pytesseract.image_to_data(pil_image, lang=lang, output_type="dict", config=tesseract_config) words, left, top, width, height = data["text"], data["left"], data["top"], data["width"], data["height"] @@ -323,7 +328,7 @@ def preprocess( words_batch = [] boxes_batch = [] for image in images: - words, boxes = apply_tesseract(image, ocr_lang, tesseract_config) + words, boxes = apply_tesseract(image, ocr_lang, tesseract_config, input_data_format=input_data_format) words_batch.append(words) boxes_batch.append(boxes) diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index 5db1387460524b..cfca5cd1806b2e 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -632,6 +632,7 @@ def _preprocess_mask( do_resize: bool = None, size: Dict[str, int] = None, size_divisor: int = 0, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> np.ndarray: """Preprocesses a single mask.""" segmentation_map = to_numpy_array(segmentation_map) @@ -920,7 +921,9 @@ def encode_inputs( if input_data_format is None: input_data_format = infer_channel_dimension_format(pixel_values_list[0]) - encoded_inputs = self.pad(pixel_values_list, return_tensors=return_tensors) + encoded_inputs = self.pad( + pixel_values_list, return_tensors=return_tensors, input_data_format=input_data_format + ) if segmentation_maps is not None: mask_labels = [] diff --git a/src/transformers/models/pix2struct/image_processing_pix2struct.py b/src/transformers/models/pix2struct/image_processing_pix2struct.py index 789666bef6ee5e..13efd02ab78115 100644 --- a/src/transformers/models/pix2struct/image_processing_pix2struct.py +++ b/src/transformers/models/pix2struct/image_processing_pix2struct.py @@ -269,7 +269,7 @@ def extract_flattened_patches( image = torch.from_numpy(image) patch_height, patch_width = patch_size["height"], patch_size["width"] - image_height, image_width = get_image_size(image, input_data_format=ChannelDimension.FIRST) + image_height, image_width = get_image_size(image, ChannelDimension.FIRST) # maximize scale s.t. scale = math.sqrt(max_patches * (patch_height / image_height) * (patch_width / image_width)) diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 5fe7d9ec0a7eb2..84b9dc8be58193 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -568,6 +568,7 @@ def generate_crop_boxes( points_per_crop: Optional[int] = 32, crop_n_points_downscale_factor: Optional[List[int]] = 1, device: Optional["torch.device"] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, return_tensors: str = "pt", ): """ @@ -590,6 +591,8 @@ def generate_crop_boxes( The number of points-per-side sampled in layer n is scaled down by crop_n_points_downscale_factor**n. device (`torch.device`, *optional*, defaults to None): Device to use for the computation. If None, cpu will be used. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. return_tensors (`str`, *optional*, defaults to `pt`): If `pt`, returns `torch.Tensor`. If `tf`, returns `tf.Tensor`. """ @@ -600,6 +603,7 @@ def generate_crop_boxes( overlap_ratio, points_per_crop, crop_n_points_downscale_factor, + input_data_format, ) if return_tensors == "pt": if device is None: @@ -906,6 +910,7 @@ def _generate_crop_boxes( overlap_ratio: float = 512 / 1500, points_per_crop: Optional[int] = 32, crop_n_points_downscale_factor: Optional[List[int]] = 1, + input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[List[List[int]], List[int]]: """ Generates a list of crop boxes of different sizes. Each layer has (2**i)**2 boxes for the ith layer. @@ -925,12 +930,14 @@ def _generate_crop_boxes( Number of points to sample per crop. crop_n_points_downscale_factor (`int`, *optional*): The number of points-per-side sampled in layer n is scaled down by crop_n_points_downscale_factor**n. + input_data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the input image. If not provided, it will be inferred. """ if isinstance(image, list): raise ValueError("Only one image is allowed for crop generation.") image = to_numpy_array(image) - original_size = get_image_size(image) + original_size = get_image_size(image, input_data_format) points_grid = [] for i in range(crop_n_layers + 1): @@ -940,7 +947,7 @@ def _generate_crop_boxes( crop_boxes, layer_idxs = _generate_per_layer_crops(crop_n_layers, overlap_ratio, original_size) cropped_images, point_grid_per_crop = _generate_crop_images( - crop_boxes, image, points_grid, layer_idxs, target_size, original_size + crop_boxes, image, points_grid, layer_idxs, target_size, original_size, input_data_format ) crop_boxes = np.array(crop_boxes) crop_boxes = crop_boxes.astype(np.float32) @@ -986,7 +993,9 @@ def _generate_per_layer_crops(crop_n_layers, overlap_ratio, original_size): return crop_boxes, layer_idxs -def _generate_crop_images(crop_boxes, image, points_grid, layer_idxs, target_size, original_size): +def _generate_crop_images( + crop_boxes, image, points_grid, layer_idxs, target_size, original_size, input_data_format=None +): """ Takes as an input bounding boxes that are used to crop the image. Based in the crops, the corresponding points are also passed. @@ -996,7 +1005,7 @@ def _generate_crop_images(crop_boxes, image, points_grid, layer_idxs, target_siz for i, crop_box in enumerate(crop_boxes): left, top, right, bottom = crop_box - channel_dim = infer_channel_dimension_format(image) + channel_dim = infer_channel_dimension_format(image, input_data_format) if channel_dim == ChannelDimension.LAST: cropped_im = image[top:bottom, left:right, :] else: @@ -1004,7 +1013,7 @@ def _generate_crop_images(crop_boxes, image, points_grid, layer_idxs, target_siz cropped_images.append(cropped_im) - cropped_im_size = get_image_size(cropped_im) + cropped_im_size = get_image_size(cropped_im, channel_dim) points_scale = np.array(cropped_im_size)[None, ::-1] points = points_grid[layer_idxs[i]] * points_scale diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 0380d5b40e6242..5ebf07889e6bbe 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -93,7 +93,7 @@ def pad( Returns: `np.ndarray`: The padded image. """ - old_height, old_width = get_image_size(image) + old_height, old_width = get_image_size(image, input_data_format) pad_height = (old_height // size + 1) * size - old_height pad_width = (old_width // size + 1) * size - old_width diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index 08c2c8367c0e4e..2e16f84e16c190 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -92,7 +92,7 @@ def get_resize_output_image_size( size_divisor: int = 32, input_data_format: Optional[Union[str, ChannelDimension]] = None, ) -> Tuple[int, int]: - input_height, input_width = get_image_size(input_image, input_data_format=input_data_format) + input_height, input_width = get_image_size(input_image, input_data_format) min_size, max_size = shorter, longer scale = min_size / min(input_height, input_width) @@ -466,15 +466,12 @@ def preprocess( ] images = [ - to_channel_dimension_format( - image, data_format, input_channel_dim=input_data_format, input_data_format=input_data_format - ) - for image in images + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images ] if do_pad: encoded_outputs = self.pad( - images, return_pixel_mask=True, return_tensors=return_tensors, input_data_format=input_data_format + images, return_pixel_mask=True, return_tensors=return_tensors, input_data_format=data_format ) else: encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index 6776e8d8f14821..a0a35ca3f26df2 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -160,7 +160,7 @@ def get_resize_output_image_size( input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred from the input image. """ - image_size = get_image_size(input_image, input_data_format=input_data_format) + image_size = get_image_size(input_image, input_data_format) if isinstance(size, (list, tuple)): return size @@ -1196,11 +1196,13 @@ def preprocess( if annotations is not None: resized_images, resized_annotations = [], [] for image, target in zip(images, annotations): - orig_size = get_image_size(image) + orig_size = get_image_size(image, input_data_format) resized_image = self.resize( image, size=size, max_size=max_size, resample=resample, input_data_format=input_data_format ) - resized_annotation = self.resize_annotation(target, orig_size, get_image_size(resized_image)) + resized_annotation = self.resize_annotation( + target, orig_size, get_image_size(resized_image, input_data_format) + ) resized_images.append(resized_image) resized_annotations.append(resized_annotation) images = resized_images diff --git a/tests/models/blip/test_image_processing_blip.py b/tests/models/blip/test_image_processing_blip.py index 398a8c02b9997f..1d7e7f12ee9bfa 100644 --- a/tests/models/blip/test_image_processing_blip.py +++ b/tests/models/blip/test_image_processing_blip.py @@ -70,7 +70,7 @@ def prepare_image_processor_dict(self): } def expected_output_image_shape(self, images): - return 3, self.size["height"], self.size["width"] + return self.num_channels, self.size["height"], self.size["width"] def prepare_image_inputs(self, equal_resolution=False, numpify=False, torchify=False): return prepare_image_inputs( @@ -135,3 +135,11 @@ def test_call_numpy(self): @unittest.skip("BlipImageProcessor does not support 4 channels yet") # FIXME Amy def test_call_pytorch(self): return super().test_call_torch() + + @unittest.skip("BLIP doesn't treat 4 channel PIL and numpy consistently yet") # FIXME Amy + def test_call_pil(self): + pass + + @unittest.skip("BLIP doesn't treat 4 channel PIL and numpy consistently yet") # FIXME Amy + def test_call_numpy_4_channels(self): + pass diff --git a/tests/models/chinese_clip/test_image_processing_chinese_clip.py b/tests/models/chinese_clip/test_image_processing_chinese_clip.py index e8f51b29fbc0a4..7eea00f885201c 100644 --- a/tests/models/chinese_clip/test_image_processing_chinese_clip.py +++ b/tests/models/chinese_clip/test_image_processing_chinese_clip.py @@ -17,7 +17,7 @@ import unittest from transformers.testing_utils import require_torch, require_vision -from transformers.utils import is_vision_available +from transformers.utils import is_torch_available, is_vision_available from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs @@ -26,6 +26,10 @@ from transformers import ChineseCLIPImageProcessor +if is_torch_available(): + pass + + class ChineseCLIPImageProcessingTester(unittest.TestCase): def __init__( self, @@ -120,6 +124,10 @@ def test_image_processor_from_dict_with_kwargs(self): self.assertEqual(image_processor.size, {"shortest_edge": 42}) self.assertEqual(image_processor.crop_size, {"height": 84, "width": 84}) + @unittest.skip("ChineseCLIPImageProcessor doesn't treat 4 channel PIL and numpy consistently yet") # FIXME Amy + def test_call_numpy_4_channels(self): + pass + @require_torch @require_vision @@ -152,3 +160,7 @@ def test_call_numpy(self): @unittest.skip("ChineseCLIPImageProcessor does not support 4 channels yet") # FIXME Amy def test_call_pytorch(self): return super().test_call_torch() + + @unittest.skip("ChineseCLIPImageProcessor doesn't treat 4 channel PIL and numpy consistently yet") # FIXME Amy + def test_call_numpy_4_channels(self): + pass diff --git a/tests/models/flava/test_image_processing_flava.py b/tests/models/flava/test_image_processing_flava.py index 8e04abafe1f6aa..d89a1a6f6bfb58 100644 --- a/tests/models/flava/test_image_processing_flava.py +++ b/tests/models/flava/test_image_processing_flava.py @@ -337,6 +337,11 @@ def _test_call_framework(self, instance_class, prepare_kwargs): def test_call_numpy(self): self._test_call_framework(np.ndarray, prepare_kwargs={"numpify": True}) + def test_call_numpy_4_channels(self): + self.image_processing_class.num_channels = 4 + self._test_call_framework(np.ndarray, prepare_kwargs={"numpify": True}) + self.image_processing_class.num_channels = 3 + def test_call_pytorch(self): self._test_call_framework(torch.Tensor, prepare_kwargs={"torchify": True}) diff --git a/tests/models/glpn/test_image_processing_glpn.py b/tests/models/glpn/test_image_processing_glpn.py index e9e6210252c00d..f9cadb33137843 100644 --- a/tests/models/glpn/test_image_processing_glpn.py +++ b/tests/models/glpn/test_image_processing_glpn.py @@ -144,3 +144,18 @@ def test_call_pytorch(self): encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) self.assertTrue(tuple(encoded_images.shape) == (1, *expected_output_image_shape)) + + def test_call_numpy_4_channels(self): + # Initialize image_processing + image_processing = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + self.image_processing_class.num_channels = 4 + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + # Test not batched input (GLPNImageProcessor doesn't support batching) + encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + self.assertTrue(tuple(encoded_images.shape) == (1, *expected_output_image_shape)) + self.image_processing_class.num_channels = 3 diff --git a/tests/models/imagegpt/test_image_processing_imagegpt.py b/tests/models/imagegpt/test_image_processing_imagegpt.py index dd141be49bf582..a806f032435cbb 100644 --- a/tests/models/imagegpt/test_image_processing_imagegpt.py +++ b/tests/models/imagegpt/test_image_processing_imagegpt.py @@ -198,6 +198,10 @@ def test_call_numpy(self): tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape) ) + @unittest.skip("ImageGPT assumes clusters for 3 channels") + def test_call_numpy_4_channels(self): + pass + # Override the test from ImageProcessingTestMixin as ImageGPT model takes input_ids as input def test_call_pytorch(self): # Initialize image_processing diff --git a/tests/models/pix2struct/test_image_processing_pix2struct.py b/tests/models/pix2struct/test_image_processing_pix2struct.py index 5bf729a502eebc..4b06573fe61ff6 100644 --- a/tests/models/pix2struct/test_image_processing_pix2struct.py +++ b/tests/models/pix2struct/test_image_processing_pix2struct.py @@ -222,6 +222,40 @@ def test_call_numpy(self): (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), ) + def test_call_numpy_4_channels(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + expected_hidden_dim = ( + (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"]) + * self.image_processor_tester.num_channels + ) + 2 + + for max_patch in self.image_processor_tester.max_patches: + # Test not batched input + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch, input_data_format="channels_first" + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (1, max_patch, expected_hidden_dim), + ) + + # Test batched + encoded_images = image_processor( + image_inputs, return_tensors="pt", max_patches=max_patch, input_data_format="channels_first" + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), + ) + self.image_processor_tester.num_channels = 3 + def test_call_pytorch(self): # Initialize image_processor image_processor = self.image_processing_class(**self.image_processor_dict) @@ -318,3 +352,7 @@ def test_call_numpy(self): @unittest.skip("Pix2StructImageProcessor does not support 4 channels yet") # FIXME Amy def test_call_pytorch(self): return super().test_call_torch() + + @unittest.skip("Pix2StructImageProcessor does treat numpy and PIL 4 channel images consistently") # FIXME Amy + def test_call_numpy_4_channels(self): + return super().test_call_torch() diff --git a/tests/models/swin2sr/test_image_processing_swin2sr.py b/tests/models/swin2sr/test_image_processing_swin2sr.py index 8448062132d9e9..719ac79d09db23 100644 --- a/tests/models/swin2sr/test_image_processing_swin2sr.py +++ b/tests/models/swin2sr/test_image_processing_swin2sr.py @@ -147,6 +147,24 @@ def test_call_numpy(self): expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + # Swin2SRImageProcessor does not support batched input + def test_call_numpy_4_channels(self): + # Initialize image_processing + image_processing = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + # Test not batched input + encoded_images = image_processing( + image_inputs[0], return_tensors="pt", input_data_format="channels_first" + ).pixel_values + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + self.image_processor_tester.num_channels = 3 + # Swin2SRImageProcessor does not support batched input def test_call_pytorch(self): # Initialize image_processing diff --git a/tests/models/tvlt/test_image_processor_tvlt.py b/tests/models/tvlt/test_image_processor_tvlt.py index 6e5c1c4c868c0a..8677a86665f326 100644 --- a/tests/models/tvlt/test_image_processor_tvlt.py +++ b/tests/models/tvlt/test_image_processor_tvlt.py @@ -217,6 +217,47 @@ def test_call_numpy(self): ), ) + def test_call_numpy_4_channels(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + video_inputs = prepare_video_inputs(self.image_processor_tester, equal_resolution=False, numpify=True) + for video in video_inputs: + self.assertIsInstance(video, list) + self.assertIsInstance(video[0], np.ndarray) + + # Test not batched input + encoded_videos = image_processor( + video_inputs[0], return_tensors="pt", input_data_format="channels_first", image_mean=0, image_std=1 + ).pixel_values + self.assertEqual( + encoded_videos.shape, + ( + 1, + self.image_processor_tester.num_frames, + self.image_processor_tester.num_channels, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + + # Test batched + encoded_videos = image_processor( + video_inputs, return_tensors="pt", input_data_format="channels_first", image_mean=0, image_std=1 + ).pixel_values + self.assertEqual( + encoded_videos.shape, + ( + self.image_processor_tester.batch_size, + self.image_processor_tester.num_frames, + self.image_processor_tester.num_channels, + self.image_processor_tester.crop_size["height"], + self.image_processor_tester.crop_size["width"], + ), + ) + self.image_processor_tester.num_channels = 3 + def test_call_pytorch(self): # Initialize image_processor image_processor = self.image_processing_class(**self.image_processor_dict) diff --git a/tests/models/videomae/test_image_processing_videomae.py b/tests/models/videomae/test_image_processing_videomae.py index 140942fd135bcd..4a6f0b93c4dde9 100644 --- a/tests/models/videomae/test_image_processing_videomae.py +++ b/tests/models/videomae/test_image_processing_videomae.py @@ -165,6 +165,33 @@ def test_call_numpy(self): tuple(encoded_videos.shape), (self.image_processor_tester.batch_size, *expected_output_video_shape) ) + def test_call_numpy_4_channels(self): + # Initialize image_processing + image_processing = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + video_inputs = self.image_processor_tester.prepare_video_inputs(equal_resolution=False, numpify=True) + for video in video_inputs: + self.assertIsInstance(video, list) + self.assertIsInstance(video[0], np.ndarray) + + # Test not batched input + encoded_videos = image_processing( + video_inputs[0], return_tensors="pt", image_mean=0, image_std=1, input_data_format="channels_first" + ).pixel_values + expected_output_video_shape = self.image_processor_tester.expected_output_image_shape([encoded_videos[0]]) + self.assertEqual(tuple(encoded_videos.shape), (1, *expected_output_video_shape)) + + # Test batched + encoded_videos = image_processing( + video_inputs, return_tensors="pt", image_mean=0, image_std=1, input_data_format="channels_first" + ).pixel_values + expected_output_video_shape = self.image_processor_tester.expected_output_image_shape(encoded_videos) + self.assertEqual( + tuple(encoded_videos.shape), (self.image_processor_tester.batch_size, *expected_output_video_shape) + ) + self.image_processor_tester.num_channels = 3 + def test_call_pytorch(self): # Initialize image_processing image_processing = self.image_processing_class(**self.image_processor_dict) diff --git a/tests/models/vivit/test_image_processing_vivit.py b/tests/models/vivit/test_image_processing_vivit.py index d901a86198a972..dad120ef818e9b 100644 --- a/tests/models/vivit/test_image_processing_vivit.py +++ b/tests/models/vivit/test_image_processing_vivit.py @@ -179,6 +179,33 @@ def test_call_numpy(self): tuple(encoded_videos.shape), (self.image_processor_tester.batch_size, *expected_output_video_shape) ) + def test_call_numpy_4_channels(self): + # Initialize image_processing + image_processing = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + video_inputs = self.image_processor_tester.prepare_video_inputs(equal_resolution=False, numpify=True) + for video in video_inputs: + self.assertIsInstance(video, list) + self.assertIsInstance(video[0], np.ndarray) + + # Test not batched input + encoded_videos = image_processing( + video_inputs[0], return_tensors="pt", image_mean=0, image_std=1, input_data_format="channels_first" + ).pixel_values + expected_output_video_shape = self.image_processor_tester.expected_output_image_shape([encoded_videos[0]]) + self.assertEqual(tuple(encoded_videos.shape), (1, *expected_output_video_shape)) + + # Test batched + encoded_videos = image_processing( + video_inputs, return_tensors="pt", image_mean=0, image_std=1, input_data_format="channels_first" + ).pixel_values + expected_output_video_shape = self.image_processor_tester.expected_output_image_shape(encoded_videos) + self.assertEqual( + tuple(encoded_videos.shape), (self.image_processor_tester.batch_size, *expected_output_video_shape) + ) + self.image_processor_tester.num_channels = 3 + def test_call_pytorch(self): # Initialize image_processing image_processing = self.image_processing_class(**self.image_processor_dict) diff --git a/tests/test_image_processing_common.py b/tests/test_image_processing_common.py index 2eb360c4a143ca..cb78b33375568e 100644 --- a/tests/test_image_processing_common.py +++ b/tests/test_image_processing_common.py @@ -252,3 +252,36 @@ def test_call_pytorch(self): tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape), ) + + def test_call_numpy_4_channels(self): + # Test that can process images which have an arbitrary number of channels + # Initialize image_processing + image_processor = self.image_processing_class(**self.image_processor_dict) + + # create random numpy tensors + self.image_processor_tester.num_channels = 4 + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + + # Test not batched input + encoded_images = image_processor( + image_inputs[0], + return_tensors="pt", + input_data_format="channels_first", + image_mean=0, + image_std=1, + ).pixel_values + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + + # Test batched + encoded_images = image_processor( + image_inputs, + return_tensors="pt", + input_data_format="channels_first", + image_mean=0, + image_std=1, + ).pixel_values + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + self.assertEqual( + tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape) + ) From 3c5a39d5c1f7075c9aa515e9a3b218f3133228ce Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:34:11 +0000 Subject: [PATCH 8/9] Remove num_channels argument --- src/transformers/models/beit/image_processing_beit.py | 8 +------- src/transformers/models/bit/image_processing_bit.py | 6 +----- src/transformers/models/blip/image_processing_blip.py | 6 +----- .../models/bridgetower/image_processing_bridgetower.py | 4 ---- .../models/chinese_clip/image_processing_chinese_clip.py | 6 +----- src/transformers/models/clip/image_processing_clip.py | 6 +----- .../image_processing_conditional_detr.py | 6 +----- .../models/convnext/image_processing_convnext.py | 6 +----- .../deformable_detr/image_processing_deformable_detr.py | 6 +----- src/transformers/models/deit/image_processing_deit.py | 6 +----- src/transformers/models/deta/image_processing_deta.py | 6 +----- src/transformers/models/detr/image_processing_detr.py | 6 +----- src/transformers/models/donut/image_processing_donut.py | 6 +----- src/transformers/models/dpt/image_processing_dpt.py | 6 +----- .../efficientformer/image_processing_efficientformer.py | 6 +----- .../models/efficientnet/image_processing_efficientnet.py | 6 +----- src/transformers/models/flava/image_processing_flava.py | 9 +-------- src/transformers/models/glpn/image_processing_glpn.py | 6 +----- .../models/imagegpt/image_processing_imagegpt.py | 6 +----- .../models/layoutlmv2/image_processing_layoutlmv2.py | 3 +-- .../models/layoutlmv3/image_processing_layoutlmv3.py | 6 +----- src/transformers/models/levit/image_processing_levit.py | 6 +----- .../models/mask2former/image_processing_mask2former.py | 5 +---- .../models/maskformer/image_processing_maskformer.py | 5 +---- .../models/mobilenet_v1/image_processing_mobilenet_v1.py | 6 +----- .../models/mobilenet_v2/image_processing_mobilenet_v2.py | 6 +----- .../models/mobilevit/image_processing_mobilevit.py | 6 +----- .../models/oneformer/image_processing_oneformer.py | 5 +---- .../models/owlvit/image_processing_owlvit.py | 6 +----- .../models/perceiver/image_processing_perceiver.py | 6 +----- .../models/pix2struct/image_processing_pix2struct.py | 6 +----- .../models/poolformer/image_processing_poolformer.py | 6 +----- src/transformers/models/pvt/image_processing_pvt.py | 6 +----- src/transformers/models/sam/image_processing_sam.py | 6 +----- .../models/segformer/image_processing_segformer.py | 8 +------- .../models/swin2sr/image_processing_swin2sr.py | 6 +----- src/transformers/models/tvlt/image_processing_tvlt.py | 8 +------- .../models/videomae/image_processing_videomae.py | 8 +------- src/transformers/models/vilt/image_processing_vilt.py | 6 +----- src/transformers/models/vit/image_processing_vit.py | 6 +----- .../models/vit_hybrid/image_processing_vit_hybrid.py | 6 +----- src/transformers/models/vivit/image_processing_vivit.py | 8 +------- src/transformers/models/yolos/image_processing_yolos.py | 6 +----- 43 files changed, 42 insertions(+), 221 deletions(-) diff --git a/src/transformers/models/beit/image_processing_beit.py b/src/transformers/models/beit/image_processing_beit.py index b983a1442996a2..040cca8984fe30 100644 --- a/src/transformers/models/beit/image_processing_beit.py +++ b/src/transformers/models/beit/image_processing_beit.py @@ -232,13 +232,12 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) image = self._preprocess( image, do_reduce_labels=False, @@ -322,7 +321,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -375,9 +373,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -436,7 +431,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in images ] diff --git a/src/transformers/models/bit/image_processing_bit.py b/src/transformers/models/bit/image_processing_bit.py index 6f21ca3f169f52..938a2b71fd8c26 100644 --- a/src/transformers/models/bit/image_processing_bit.py +++ b/src/transformers/models/bit/image_processing_bit.py @@ -177,7 +177,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -229,9 +228,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -276,7 +272,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 294c3139513d3f..0f6c0a9e7876f7 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -169,7 +169,6 @@ def preprocess( do_convert_rgb: bool = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -217,9 +216,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -259,7 +255,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 11db01bf569d90..06a9ef4805f2a7 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -380,7 +380,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -434,9 +433,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size_divisor = size_divisor if size_divisor is not None else self.size_divisor diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py index 14ce13e26c95e5..32b90eb39f4e06 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip.py @@ -175,7 +175,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -227,9 +226,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -274,7 +270,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/clip/image_processing_clip.py b/src/transformers/models/clip/image_processing_clip.py index 9e6f18f5db287a..7ee5cd65d22e02 100644 --- a/src/transformers/models/clip/image_processing_clip.py +++ b/src/transformers/models/clip/image_processing_clip.py @@ -176,7 +176,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -228,9 +227,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -275,7 +271,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index 39182d9f0993ac..e69316d574ed5d 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -1119,7 +1119,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1176,9 +1175,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1265,7 +1261,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index 5864c865996473..408766673e4ad1 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -196,7 +196,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -245,9 +244,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize crop_pct = crop_pct if crop_pct is not None else self.crop_pct @@ -286,7 +282,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index ce2c3b31929456..1f0b209ba898c5 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -1117,7 +1117,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1174,9 +1173,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1263,7 +1259,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: diff --git a/src/transformers/models/deit/image_processing_deit.py b/src/transformers/models/deit/image_processing_deit.py index d786d7c30f6ce8..fc92a4d2577445 100644 --- a/src/transformers/models/deit/image_processing_deit.py +++ b/src/transformers/models/deit/image_processing_deit.py @@ -173,7 +173,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -221,9 +220,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -264,7 +260,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/deta/image_processing_deta.py b/src/transformers/models/deta/image_processing_deta.py index 195f627f16a26f..22e73d032305ac 100644 --- a/src/transformers/models/deta/image_processing_deta.py +++ b/src/transformers/models/deta/image_processing_deta.py @@ -789,7 +789,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -846,9 +845,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -927,7 +923,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: diff --git a/src/transformers/models/detr/image_processing_detr.py b/src/transformers/models/detr/image_processing_detr.py index 6f4dc4ddf33e25..09c11d91540a00 100644 --- a/src/transformers/models/detr/image_processing_detr.py +++ b/src/transformers/models/detr/image_processing_detr.py @@ -1089,7 +1089,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1146,9 +1145,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1235,7 +1231,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index c435ef0875a573..a1cd1c084a1ca2 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -312,7 +312,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -368,9 +367,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. if `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -413,7 +409,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_align_long_axis: images = [self.align_long_axis(image, size=size, input_data_format=input_data_format) for image in images] diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index ac78fc6e14b8c7..fd0bdada6d7b06 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -223,7 +223,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -273,9 +272,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -311,7 +307,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/efficientformer/image_processing_efficientformer.py b/src/transformers/models/efficientformer/image_processing_efficientformer.py index eda2d6f492689d..8e1b81c2846152 100644 --- a/src/transformers/models/efficientformer/image_processing_efficientformer.py +++ b/src/transformers/models/efficientformer/image_processing_efficientformer.py @@ -172,7 +172,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -221,9 +220,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -262,7 +258,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet.py b/src/transformers/models/efficientnet/image_processing_efficientnet.py index 19c019bf9a1122..0957f60ecaeba3 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet.py @@ -224,7 +224,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -276,9 +275,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -321,7 +317,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/flava/image_processing_flava.py b/src/transformers/models/flava/image_processing_flava.py index 1fea5b861b4e86..b1681e9d9f8a91 100644 --- a/src/transformers/models/flava/image_processing_flava.py +++ b/src/transformers/models/flava/image_processing_flava.py @@ -400,7 +400,6 @@ def _preprocess_image( do_map_pixels: bool = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[ChannelDimension] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -417,7 +416,7 @@ def _preprocess_image( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) if do_resize: image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) @@ -475,7 +474,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -563,9 +561,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -649,7 +644,6 @@ def preprocess( do_map_pixels=False, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in images ] @@ -672,7 +666,6 @@ def preprocess( do_map_pixels=codebook_do_map_pixels, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in images ] diff --git a/src/transformers/models/glpn/image_processing_glpn.py b/src/transformers/models/glpn/image_processing_glpn.py index 7be723cc1effc9..a0c341e4762907 100644 --- a/src/transformers/models/glpn/image_processing_glpn.py +++ b/src/transformers/models/glpn/image_processing_glpn.py @@ -130,7 +130,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -166,9 +165,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -188,7 +184,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/imagegpt/image_processing_imagegpt.py b/src/transformers/models/imagegpt/image_processing_imagegpt.py index 0fc0b545b981f2..c75a0e890f40ce 100644 --- a/src/transformers/models/imagegpt/image_processing_imagegpt.py +++ b/src/transformers/models/imagegpt/image_processing_imagegpt.py @@ -183,7 +183,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -224,9 +223,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -256,7 +252,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py index 4a46f26ae7ba21..a5f8d7c2ce4d60 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py @@ -198,7 +198,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -258,7 +257,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if apply_ocr: requires_backends(self, "pytesseract") diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py index 55d4f5824b9439..7eaa8b8373cbe3 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py @@ -229,7 +229,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -281,9 +280,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -320,7 +316,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) # Tesseract OCR to get words + normalized bounding boxes if apply_ocr: diff --git a/src/transformers/models/levit/image_processing_levit.py b/src/transformers/models/levit/image_processing_levit.py index 37240235e3c560..b00a3f42eb2a67 100644 --- a/src/transformers/models/levit/image_processing_levit.py +++ b/src/transformers/models/levit/image_processing_levit.py @@ -185,7 +185,6 @@ def preprocess( return_tensors: Optional[TensorType] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -236,9 +235,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. if `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -279,7 +275,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [self.resize(image, size, resample, input_data_format=input_data_format) for image in images] diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index cfca5cd1806b2e..6d838ec2b96219 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -602,13 +602,12 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) image = self._preprocess( image=image, do_resize=do_resize, @@ -682,7 +681,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: if "pad_and_return_pixel_mask" in kwargs: @@ -746,7 +744,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for image in images ] diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index 5a00233124298d..61f1d1ab17ef1f 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -609,13 +609,12 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) image = self._preprocess( image=image, do_resize=do_resize, @@ -689,7 +688,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: if "pad_and_return_pixel_mask" in kwargs: @@ -763,7 +761,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for image in images ] diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py index 1996539cb10c20..e703ad538fb7f1 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py @@ -169,7 +169,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ): """ @@ -218,9 +217,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -260,7 +256,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py index 907aebafd71285..26319cf1c4d860 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py @@ -173,7 +173,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ): """ @@ -222,9 +221,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -264,7 +260,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/mobilevit/image_processing_mobilevit.py b/src/transformers/models/mobilevit/image_processing_mobilevit.py index e8fc3e20939c56..f5208812f1a1dd 100644 --- a/src/transformers/models/mobilevit/image_processing_mobilevit.py +++ b/src/transformers/models/mobilevit/image_processing_mobilevit.py @@ -182,7 +182,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -225,9 +224,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -265,7 +261,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 842414fe6cf87f..88250996a23194 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -570,13 +570,12 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) image = self._preprocess( image=image, do_resize=do_resize, @@ -647,7 +646,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: if "pad_and_return_pixel_mask" in kwargs: @@ -725,7 +723,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for image in images ] diff --git a/src/transformers/models/owlvit/image_processing_owlvit.py b/src/transformers/models/owlvit/image_processing_owlvit.py index f5ebf61166cd62..e75eb88a2ac835 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit.py +++ b/src/transformers/models/owlvit/image_processing_owlvit.py @@ -281,7 +281,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -334,9 +333,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -374,7 +370,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/perceiver/image_processing_perceiver.py b/src/transformers/models/perceiver/image_processing_perceiver.py index 0ca916fe16c0e5..06544ef0de10e1 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver.py +++ b/src/transformers/models/perceiver/image_processing_perceiver.py @@ -221,7 +221,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -268,9 +267,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop crop_size = crop_size if crop_size is not None else self.crop_size @@ -310,7 +306,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_center_crop: images = [ diff --git a/src/transformers/models/pix2struct/image_processing_pix2struct.py b/src/transformers/models/pix2struct/image_processing_pix2struct.py index 13efd02ab78115..3833923455d969 100644 --- a/src/transformers/models/pix2struct/image_processing_pix2struct.py +++ b/src/transformers/models/pix2struct/image_processing_pix2struct.py @@ -370,7 +370,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> ImageInput: """ @@ -412,9 +411,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. if `input_data_format` is unset. """ do_normalize = do_normalize if do_normalize is not None else self.do_normalize do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb @@ -442,7 +438,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if is_vqa: if header_text is None: diff --git a/src/transformers/models/poolformer/image_processing_poolformer.py b/src/transformers/models/poolformer/image_processing_poolformer.py index d51f8304cf88b4..afa8b1925e9f94 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer.py +++ b/src/transformers/models/poolformer/image_processing_poolformer.py @@ -224,7 +224,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -273,9 +272,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize crop_pct = crop_pct if crop_pct is not None else self.crop_pct @@ -317,7 +313,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/pvt/image_processing_pvt.py b/src/transformers/models/pvt/image_processing_pvt.py index 69a7cf6059984c..7a15407ae20737 100644 --- a/src/transformers/models/pvt/image_processing_pvt.py +++ b/src/transformers/models/pvt/image_processing_pvt.py @@ -157,7 +157,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ): """ @@ -202,9 +201,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -236,7 +232,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/sam/image_processing_sam.py b/src/transformers/models/sam/image_processing_sam.py index 84b9dc8be58193..d7e0976b9f4e62 100644 --- a/src/transformers/models/sam/image_processing_sam.py +++ b/src/transformers/models/sam/image_processing_sam.py @@ -252,7 +252,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ): """ @@ -303,9 +302,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -350,7 +346,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) original_sizes = [get_image_size(image, channel_dim=input_data_format) for image in images] diff --git a/src/transformers/models/segformer/image_processing_segformer.py b/src/transformers/models/segformer/image_processing_segformer.py index 9697a70119b3e1..177889955614d8 100644 --- a/src/transformers/models/segformer/image_processing_segformer.py +++ b/src/transformers/models/segformer/image_processing_segformer.py @@ -228,13 +228,12 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[Union[str, ChannelDimension]] = None, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" # All transformations expect numpy arrays. image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) image = self._preprocess( image=image, do_reduce_labels=False, @@ -313,7 +312,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -362,9 +360,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -414,7 +409,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in images ] diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 5ebf07889e6bbe..5ebce752359ac5 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -115,7 +115,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ): """ @@ -151,9 +150,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. if `input_data_format` is unset. """ do_rescale = do_rescale if do_rescale is not None else self.do_rescale rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor @@ -176,7 +172,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_rescale: images = [ diff --git a/src/transformers/models/tvlt/image_processing_tvlt.py b/src/transformers/models/tvlt/image_processing_tvlt.py index 9ae4075e709baa..2d0a0202a444fd 100644 --- a/src/transformers/models/tvlt/image_processing_tvlt.py +++ b/src/transformers/models/tvlt/image_processing_tvlt.py @@ -209,7 +209,6 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -228,7 +227,7 @@ def _preprocess_image( image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) if do_resize: image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) @@ -263,7 +262,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -317,9 +315,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. if `input_data_format` is unset. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: @@ -391,7 +386,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in video ] diff --git a/src/transformers/models/videomae/image_processing_videomae.py b/src/transformers/models/videomae/image_processing_videomae.py index 6e63998e2a2545..cbe95eb89ab5c1 100644 --- a/src/transformers/models/videomae/image_processing_videomae.py +++ b/src/transformers/models/videomae/image_processing_videomae.py @@ -188,7 +188,6 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -207,7 +206,7 @@ def _preprocess_image( image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) if do_resize: image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) @@ -240,7 +239,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -288,9 +286,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -330,7 +325,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in video ] diff --git a/src/transformers/models/vilt/image_processing_vilt.py b/src/transformers/models/vilt/image_processing_vilt.py index 2e16f84e16c190..934ce5be2e9064 100644 --- a/src/transformers/models/vilt/image_processing_vilt.py +++ b/src/transformers/models/vilt/image_processing_vilt.py @@ -350,7 +350,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -400,9 +399,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size_divisor = size_divisor if size_divisor is not None else self.size_divisor @@ -439,7 +435,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/vit/image_processing_vit.py b/src/transformers/models/vit/image_processing_vit.py index 926cd4be847f9d..60c316a0bbf732 100644 --- a/src/transformers/models/vit/image_processing_vit.py +++ b/src/transformers/models/vit/image_processing_vit.py @@ -156,7 +156,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ): """ @@ -201,9 +200,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize do_rescale = do_rescale if do_rescale is not None else self.do_rescale @@ -235,7 +231,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py index d1484cd9717df0..77b0f629097a57 100644 --- a/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py +++ b/src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py @@ -177,7 +177,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -229,9 +228,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. if `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize size = size if size is not None else self.size @@ -276,7 +272,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) if do_resize: images = [ diff --git a/src/transformers/models/vivit/image_processing_vivit.py b/src/transformers/models/vivit/image_processing_vivit.py index 83c782e5f53fea..156c3bf3e7de85 100644 --- a/src/transformers/models/vivit/image_processing_vivit.py +++ b/src/transformers/models/vivit/image_processing_vivit.py @@ -237,7 +237,6 @@ def _preprocess_image( image_std: Optional[Union[float, List[float]]] = None, data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, ) -> np.ndarray: """Preprocesses a single image.""" if do_resize and size is None or resample is None: @@ -259,7 +258,7 @@ def _preprocess_image( image = to_numpy_array(image) if input_data_format is None: - input_data_format = infer_channel_dimension_format(image, num_channels=num_channels) + input_data_format = infer_channel_dimension_format(image) if do_resize: image = self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) @@ -293,7 +292,6 @@ def preprocess( return_tensors: Optional[Union[str, TensorType]] = None, data_format: ChannelDimension = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> PIL.Image.Image: """ @@ -343,9 +341,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ do_resize = do_resize if do_resize is not None else self.do_resize resample = resample if resample is not None else self.resample @@ -387,7 +382,6 @@ def preprocess( image_std=image_std, data_format=data_format, input_data_format=input_data_format, - num_channels=num_channels, ) for img in video ] diff --git a/src/transformers/models/yolos/image_processing_yolos.py b/src/transformers/models/yolos/image_processing_yolos.py index a0a35ca3f26df2..0af4f12cbb195f 100644 --- a/src/transformers/models/yolos/image_processing_yolos.py +++ b/src/transformers/models/yolos/image_processing_yolos.py @@ -1027,7 +1027,6 @@ def preprocess( return_tensors: Optional[Union[TensorType, str]] = None, data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, input_data_format: Optional[Union[str, ChannelDimension]] = None, - num_channels: Optional[int] = None, **kwargs, ) -> BatchFeature: """ @@ -1081,9 +1080,6 @@ def preprocess( - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - num_channels (`int`, *optional*): - The number of channels in the input image, used to infer the channel dimension format if - `input_data_format` is unset. """ if "pad_and_return_pixel_mask" in kwargs: logger.warning_once( @@ -1170,7 +1166,7 @@ def preprocess( if input_data_format is None: # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0], num_channels=num_channels) + input_data_format = infer_channel_dimension_format(images[0]) # prepare (COCO annotations as a list of Dict -> DETR target as a single Dict per image) if annotations is not None: From 24c9bd8c8eb03d25aa0a38527737da93cafec892 Mon Sep 17 00:00:00 2001 From: Amy Roberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:08:32 +0000 Subject: [PATCH 9/9] Update doc strings -> default ints not in code formatting --- .../models/bridgetower/image_processing_bridgetower.py | 2 +- src/transformers/models/dpt/image_processing_dpt.py | 4 ++-- .../models/mask2former/image_processing_mask2former.py | 4 ++-- .../models/maskformer/image_processing_maskformer.py | 4 ++-- src/transformers/models/swin2sr/image_processing_swin2sr.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower.py b/src/transformers/models/bridgetower/image_processing_bridgetower.py index 06a9ef4805f2a7..899fe062856129 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower.py @@ -131,7 +131,7 @@ class BridgeTowerImageProcessor(BaseImageProcessor): Resize the shorter side of the input to `size["shortest_edge"]`. The longer side will be limited to under `int((1333 / 800) * size["shortest_edge"])` while preserving the aspect ratio. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `size` parameter in the `preprocess` method. - size_divisor (`int`, *optional*, defaults to `32`): + size_divisor (`int`, *optional*, defaults to 32): The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): diff --git a/src/transformers/models/dpt/image_processing_dpt.py b/src/transformers/models/dpt/image_processing_dpt.py index fd0bdada6d7b06..31092259e7d3aa 100644 --- a/src/transformers/models/dpt/image_processing_dpt.py +++ b/src/transformers/models/dpt/image_processing_dpt.py @@ -102,7 +102,7 @@ class DPTImageProcessor(BaseImageProcessor): keep_aspect_ratio (`bool`, *optional*, defaults to `False`): If `True`, the image is resized to the largest possible size such that the aspect ratio is preserved. Can be overidden by `keep_aspect_ratio` in `preprocess`. - ensure_multiple_of (`int`, *optional*, defaults to `1`): + ensure_multiple_of (`int`, *optional*, defaults to 1): If `do_resize` is `True`, the image is resized to a size that is a multiple of this value. Can be overidden by `ensure_multiple_of` in `preprocess`. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): @@ -176,7 +176,7 @@ def resize( Target size of the output image. keep_aspect_ratio (`bool`, *optional*, defaults to `False`): If `True`, the image is resized to the largest possible size such that the aspect ratio is preserved. - ensure_multiple_of (`int`, *optional*, defaults to `1`): + ensure_multiple_of (`int`, *optional*, defaults to 1): The image is resized to a size that is a multiple of this value. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): Defines the resampling filter to use if resizing the image. Otherwise, the image is resized to size diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index 6d838ec2b96219..90917d12ec294a 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -316,7 +316,7 @@ def get_mask2former_resize_output_image_size( Whether to default to square if no size is provided. max_size (`int`, *optional*): The maximum size of the output image. - size_divisible (`int`, *optional*, defaults to `0`): + size_divisible (`int`, *optional*, defaults to 0): If size_divisible is given, the output image size will be divisible by the number. Returns: @@ -472,7 +472,7 @@ def resize( Image to resize. size (`Dict[str, int]`): The size of the output image. - size_divisor (`int`, *optional*, defaults to `0`): + size_divisor (`int`, *optional*, defaults to 0): If size_divisor is given, the output image size will be divisible by the number. resample (`PILImageResampling` resampling filter, *optional*, defaults to `PILImageResampling.BILINEAR`): Resampling filter to use when resizing the image. diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index 61f1d1ab17ef1f..bd0a248e11d90d 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -318,7 +318,7 @@ def get_maskformer_resize_output_image_size( Whether to default to square if no size is provided. max_size (`int`, *optional*): The maximum size of the output image. - size_divisible (`int`, *optional*, defaults to `0`): + size_divisible (`int`, *optional*, defaults to 0): If size_divisible is given, the output image size will be divisible by the number. Returns: @@ -480,7 +480,7 @@ def resize( Image to resize. size (`Dict[str, int]`): The size of the output image. - size_divisor (`int`, *optional*, defaults to `0`): + size_divisor (`int`, *optional*, defaults to 0): If size_divisor is given, the output image size will be divisible by the number. resample (`PILImageResampling` resampling filter, *optional*, defaults to `PILImageResampling.BILINEAR`): Resampling filter to use when resizing the image. diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr.py b/src/transformers/models/swin2sr/image_processing_swin2sr.py index 5ebce752359ac5..0944a3c67894c4 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr.py @@ -129,7 +129,7 @@ def preprocess( Rescale factor to rescale the image by if `do_rescale` is set to `True`. do_pad (`bool`, *optional*, defaults to `True`): Whether to pad the image to make the height and width divisible by `window_size`. - pad_size (`int`, *optional*, defaults to `32`): + pad_size (`int`, *optional*, defaults to 32): The size of the sliding window for the local attention. return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be one of: