o eip@sddlmZddlmZddlmZmZddlmZm Z m Z ddl Z ddl mZddlmZmZmZdd lmZmZmZmZmZdd lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1e-rddlm2Z2e+rddl3Z3e,rddl4m5m6m7Z8ddlm9Z9ndZ9e.:e;ZdBde?dBde>dBde?e@e?BdBde?e@e?BdBde>dBdedBde>dBdedBde ddeAe)BdBdefdd ZBd4d!d"d#eCdBd$d"fd%d&ZDd'eed$e@efd(d)ZEd*e@d"d$eFeCd+ffd,d-ZGd.e e jHd"fd/eCd$e@e e jHd"ffd0d1ZIe*Gd2d3d3eZJdS)5)Iterable)deepcopy) lru_cachepartial)AnyOptionalUnionN)validate_typed_dict)BaseImageProcessor BatchFeature get_size_dict)convert_to_rgbget_resize_output_image_sizeget_size_with_aspect_ratiogroup_images_by_shapereorder_images) ChannelDimension ImageInput ImageTypeSizeDictget_image_size#get_image_size_for_max_height_widthget_image_typeinfer_channel_dimension_formatmake_flat_list_of_imagesvalidate_kwargsvalidate_preprocess_arguments) ImagesKwargsUnpack) TensorTypeauto_docstringis_torch_availableis_torchvision_availableis_vision_availablelogging)is_rocm_platformis_torchdynamo_compiling)PILImageResampling)pil_torch_interpolation_mapping maxsize do_rescalerescale_factor do_normalize image_mean image_stddo_center_crop crop_size do_resizesize interpolationtvF.InterpolationModereturn_tensors data_formatc CsJt|||||||||| d | dur| dkrtd| tjkr#tddS)z Checks validity of typically used arguments in an `ImageProcessorFast` `preprocess` method. Raises `ValueError` if arguments incompatibility is caught. ) r-r.r/r0r1r2r3r4r5r6Nptz6Only returning PyTorch tensors is currently supported.z6Only channel first data format is currently supported.)r ValueErrorrFIRST) r-r.r/r0r1r2r3r4r5r6r8r9r=f/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/image_processing_utils_fast.py"validate_fast_preprocess_argumentsIs"  r?tensor torch.TensoraxisreturncCs6|dur|Sz|j|dWSty|YSw)zF Squeezes a tensor, but only if the axis specified has dim 1. N)rB)squeezer;)r@rBr=r=r> safe_squeezeps rEvaluescCsddt|DS)zO Return the maximum value across all indices of an iterable of values. cSsg|]}t|qSr=)max).0values_ir=r=r> z&max_across_indices..)zip)rFr=r=r>max_across_indices}srMimages.cCs tdd|D\}}}||fS)zH Get the maximum height and width across all images in a batch. cSsg|]}|jqSr=)shaperHimgr=r=r>rJsz(get_max_height_width..)rM)rN_ max_height max_widthr=r=r>get_max_height_widthsrUimage patch_sizecCsjg}t|tjd\}}td||D]!}td||D]}|dd||||||f}||qq|S)a6 Divides an image into patches of a specified size. Args: image (`Union[np.array, "torch.Tensor"]`): The input image. patch_size (`int`): The size of each patch. Returns: list: A list of Union[np.array, "torch.Tensor"] representing the patches. ) channel_dimrN)rrr<rangeappend)rVrWpatchesheightwidthijpatchr=r=r>divide_to_patchess" rac seZdZdZdZdZdZdZdZdZ dZ dZ dZ dZ dZdZdZdZdZejZdZdZdgZdZeZdZdeeffdd Zed e fd d Z!    d\de"dde#de$dBde%dBde de dBde dBd e&e'ddffddZ(  d]ddde#de)dde d df dd Z*e+  d]ddd!e'e$e$fde)dde d df d"d#Z,ddd$e-d dfd%d&Z.ddd'e-e/e-Bd(e-e/e-Bd dfd)d*Z0e1d+d,      d^d-e dBd.e-e"e-BdBd/e-e"e-BdBd0e dBd1e-dBd2e)d3d e'fd4d5Z2ddd0e d1e-d-e d.e-e"e-Bd/e-e"e-Bd dfd6d7Z3ddde#d dfd8d9Z4de5d e5fd:d;Z6de7fdd_de5d?e$d e5fd@dAZ9   d`de5dBe dBdCe%eBdBd2e)d3d df dDdEZ:    >dade5dBe dBdCe%eBdBd2e)d3d?e$d e"df dFdGZ;       dbde#dBdHe#dBde#dBdIe dBd.e-e"e-BdBd/e-e"e-BdBdJedBd e7fdKdLZ<            dcd0e dBd1e-dBd-e dBd.e-e'e-BdBd/e-e'e-BdBdMe dBde#dBdNe dBdHe#dBde)ddOe%e=BdBdJedBfdPdQZ>e?de5deed e@fdRdSZAddTde5dBe dCed2e&e%d3fdBdeed e@f dUdVZBde"ddMe de#de)ddNe dHe#d0e d1e-d-e d.e-e"e-BdBd/e-e"e-BdBdWe dBde#dBde dBdOe%e=BdBd e@f dXdYZCfdZd[ZDZES)dBaseImageProcessorFasta3 Base class for fast image processors using PyTorch and TorchVision for image transformations. This class provides a complete implementation for standard image preprocessing operations (resize, crop, rescale, normalize) with GPU support and batch processing optimizations. Most image processors can be implemented by simply setting class attributes; only processors requiring custom logic need to override methods. Basic Implementation -------------------- For processors that only need standard operations (resize, center crop, rescale, normalize), define class attributes: class MyImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_DEFAULT_MEAN image_std = IMAGENET_DEFAULT_STD size = {"height": 224, "width": 224} do_resize = True do_rescale = True do_normalize = True Custom Processing ----------------- Override `_preprocess` (most common): For custom image processing logic, override `_preprocess`. This method receives a list of torch tensors with channel dimension first and should return a BatchFeature. Use `group_images_by_shape` and `reorder_images` for efficient batch processing: def _preprocess( self, images: list[torch.Tensor], do_resize: bool, size: SizeDict, # ... other parameters **kwargs, ) -> BatchFeature: # Group images by shape for batched operations grouped_images, indices = group_images_by_shape(images) processed_groups = {} for shape, stacked_images in grouped_images.items(): if do_resize: stacked_images = self.resize(stacked_images, size) # Custom processing here processed_groups[shape] = stacked_images processed_images = reorder_images(processed_groups, indices) return BatchFeature(data={"pixel_values": torch.stack(processed_images)}) Override `_preprocess_image_like_inputs` (for additional inputs): For processors handling multiple input types (e.g., images + segmentation maps), override this method: def _preprocess_image_like_inputs( self, images: ImageInput, segmentation_maps: Optional[ImageInput] = None, do_convert_rgb: bool, input_data_format: ChannelDimension, device: Optional[torch.device] = None, **kwargs, ) -> BatchFeature: images = self._prepare_image_like_inputs(images, do_convert_rgb, input_data_format, device) batch_feature = self._preprocess(images, **kwargs) if segmentation_maps is not None: # Process segmentation maps separately maps = self._prepare_image_like_inputs(segmentation_maps, ...) batch_feature["labels"] = self._preprocess(maps, ...) return batch_feature Override `_further_process_kwargs` (for custom kwargs formatting): To format custom kwargs before validation: def _further_process_kwargs(self, custom_param=None, **kwargs): kwargs = super()._further_process_kwargs(**kwargs) if custom_param is not None: kwargs["custom_param"] = self._format_custom_param(custom_param) return kwargs Override `_validate_preprocess_kwargs` (for custom validation): To add custom validation logic: def _validate_preprocess_kwargs(self, custom_param=None, **kwargs): super()._validate_preprocess_kwargs(**kwargs) if custom_param is not None and custom_param < 0: raise ValueError("custom_param must be non-negative") Override `_prepare_images_structure` (for nested inputs): By default, nested image lists are flattened. Override to preserve structure: def _prepare_images_structure(self, images, expected_ndims=3): # Custom logic to handle nested structure return images # Return as-is or with custom processing Custom Parameters ----------------- To add parameters beyond `ImagesKwargs`, create a custom kwargs class and set it as `valid_kwargs`: class MyImageProcessorKwargs(ImagesKwargs): custom_param: Optional[int] = None another_param: Optional[bool] = None class MyImageProcessorFast(BaseImageProcessorFast): valid_kwargs = MyImageProcessorKwargs custom_param = 10 # default value def _preprocess(self, images, custom_param, **kwargs): # Use custom_param in processing ... Key Notes --------- - Images in `_preprocess` are always torch tensors with channel dimension first, regardless of input format - Arguments not provided by users default to class attribute values - Use batch processing utilities (`group_images_by_shape`, `reorder_images`) for GPU efficiency - Image loading, format conversion, and argument handling are automatic - focus only on processing logic NTgp? pixel_valueskwargsc stjdi|||}|d|j}|dur$t||d|jdnd|_|d|j}|dur8t|ddnd|_|d|j}|durLt|ddnd|_|j j D]}||d}|durft |||qSt ||t t ||dqSt|j j |_dS) Nr5default_to_squarer5rer3 param_namepad_sizer5rhr=)super__init__filter_out_unused_kwargspopr5r rer3ri valid_kwargs__annotations__setattrrgetattrlistkeys_valid_kwargs_names)selfrdr5r3rikeykwarg __class__r=r>rl:s"   zBaseImageProcessorFast.__init__rCcCsdS)zv `bool`: Whether or not this image processor is a fast processor (backed by PyTorch and TorchVision). Tr=)rvr=r=r>is_fastRszBaseImageProcessorFast.is_fastrconstantFrNrAri fill_value padding_mode return_maskdisable_grouping is_nested)rArAcKsb|dur|jr |jstd|d|j|jf}nt|}t|||d\} } i} i} | D]l\} }|jdd}|d|d}|d|d}|dksQ|dkr\td|d |d||krodd||f}tj||||d }|| | <|rt j |t j d d dddddf}d|d d|dd|df<|| | <q.t | | |d }|rt | | |d }||fS|S)ax Pads images to `(pad_size["height"], pad_size["width"])` or to the largest size in the batch. Args: images (`list[torch.Tensor]`): Images to pad. pad_size (`SizeDict`, *optional*): Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. fill_value (`int`, *optional*, defaults to `0`): The constant value used to fill the padded area. padding_mode (`str`, *optional*, defaults to "constant"): The padding mode to use. Can be any of the modes supported by `torch.nn.functional.pad` (e.g. constant, reflection, replication). return_mask (`bool`, *optional*, defaults to `False`): Whether to return a pixel mask to denote padded regions. disable_grouping (`bool`, *optional*, defaults to `False`): Whether to disable grouping of images by size. Returns: `Union[tuple[torch.Tensor, torch.Tensor], torch.Tensor]`: The padded images and pixel masks if `return_mask` is `True`. NzCPad size must contain 'height' and 'width' keys only. Got pad_size=.)rrrr zrPadding dimensions are negative. Please make sure that the `pad_size` is larger than the image size. Got pad_size=z , image_size=)fillr~dtype.)r) r\r]r;rUritemsrOtvFpadtorch zeros_likeint64r)rvrNrir}r~rrrrdgrouped_imagesgrouped_images_indexprocessed_images_groupedprocessed_masks_groupedrOstacked_images image_sizepadding_height padding_widthpadding stacked_masksprocessed_imagesprocessed_masksr=r=r>rYsH   $zBaseImageProcessorFast.padrVr5r6r7 antialiascKs|dur|ntjj}|jr|jrt|dd|j|j}n8|jr-t||jdtj d}n*|j rB|j rBt |dd|j |j }n|j rO|jrO|j |jf}ntd|dtretre|||||Stj||||dS)a Resize an image to `(size["height"], size["width"])`. Args: image (`torch.Tensor`): Image to resize. size (`SizeDict`): Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. interpolation (`InterpolationMode`, *optional*, defaults to `InterpolationMode.BILINEAR`): `InterpolationMode` filter to use when resizing the image e.g. `InterpolationMode.BICUBIC`. antialias (`bool`, *optional*, defaults to `True`): Whether to use antialiasing. Returns: `torch.Tensor`: The resized image. NrF)r5reinput_data_formatzjSize must contain 'height' and 'width' keys, or 'max_height' and 'max_width', or 'shortest_edge' key. Got rr6r)rInterpolationModeBILINEAR shortest_edge longest_edgerr5rrr<rSrTrr\r]r;r'r&compile_friendly_resizeresize)rvrVr5r6rrdnew_sizer=r=r>rs4    zBaseImageProcessorFast.resizercCs|jtjkr5|d}tj||||d}|d}t|dkd|}t|dkd|}|tj}|Stj||||d}|S)z} A wrapper around `tvF.resize` so that it is compatible with torch.compile when the image is a uint8 tensor. rr) rruint8floatrrwhereroundto)rVrr6rr=r=r>rs z.BaseImageProcessorFast.compile_friendly_resizescalecKs||S)a? Rescale an image by a scale factor. image = image * scale. Args: image (`torch.Tensor`): Image to rescale. scale (`float`): The scaling factor to rescale pixel values by. Returns: `torch.Tensor`: The rescaled image. r=)rvrVrrdr=r=r>rescaleszBaseImageProcessorFast.rescalemeanstdcKst|||S)a Normalize an image. image = (image - image_mean) / image_std. Args: image (`torch.Tensor`): Image to normalize. mean (`torch.Tensor`, `float` or `Iterable[float]`): Image mean to use for normalization. std (`torch.Tensor`, `float` or `Iterable[float]`): Image standard deviation to use for normalization. Returns: `torch.Tensor`: The normalized image. )r normalize)rvrVrrrdr=r=r>rsz BaseImageProcessorFast.normalizer*r+r/r0r1r-r.devicez torch.devicecCsB|r|rtj||dd|}tj||dd|}d}|||fS)Nrg?F)rr@)rvr/r0r1r-r.rr=r=r>!_fuse_mean_std_and_rescale_factors   z8BaseImageProcessorFast._fuse_mean_std_and_rescale_factorcCsR|j||||||jd\}}}|r||jtjd||}|S|r'|||}|S)z/ Rescale and normalize images. )r/r0r1r-r.rr)rrrrrfloat32r)rvrNr-r.r/r0r1r=r=r>rescale_and_normalize-s    z,BaseImageProcessorFast.rescale_and_normalizec Ks"|jdus |jdurtd||jdd\}}|j|j}}||ks+||krx||kr5||dnd||kr@||dnd||krM||ddnd||krZ||ddndg}tj||dd}|jdd\}}||krx||krx|St||d} t||d} t|| | ||S) a Note: override torchvision's center_crop to have the same behavior as the slow processor. Center crop an image to `(size["height"], size["width"])`. If the input size is smaller than `crop_size` along any edge, the image is padded with 0's and then center cropped. Args: image (`"torch.Tensor"`): Image to center crop. size (`dict[str, int]`): Size of the output image. Returns: `torch.Tensor`: The center cropped image. Nz=The size dictionary must have keys 'height' and 'width'. Got rrr )rg@) r\r]r;rtrOrrintcrop) rvrVr5rd image_height image_width crop_height crop_width padding_ltrbcrop_top crop_leftr=r=r> center_cropIs"z"BaseImageProcessorFast.center_cropcCst|S)a' Converts an image to RGB format. Only converts if the image is of type PIL.Image.Image, otherwise returns the image as is. Args: image (ImageInput): The image to convert. Returns: ImageInput: The converted image. )r)rvrVr=r=r>rrsz%BaseImageProcessorFast.convert_to_rgbcCsB|jdur|S|jD]}||vrtd|d||q |S)zJ Filter out the unused kwargs from the kwargs dictionary. Nz!This processor does not use the `z ` parameter. It will be ignored.) unused_kwargslogger warning_oncern)rvrd kwarg_namer=r=r>rms   z/BaseImageProcessorFast.filter_out_unused_kwargsexpected_ndimscCs||}t||dS)z Prepare the images structure for processing. Args: images (`ImageInput`): The input images to process. Returns: `ImageInput`: The images with a valid nesting. r) fetch_imagesr)rvrNrr=r=r>_prepare_images_structures  z0BaseImageProcessorFast._prepare_images_structuredo_convert_rgbrcCst|}|tjtjtjfvrtd||r||}|tjkr't|}n |tjkr3t | }|j dkr=| d}|durEt|}|tjkrS|ddd }|dur\||}|S)NzUnsupported input image type rrr )rrPILTORCHNUMPYr;rr pil_to_tensorr from_numpy contiguousndim unsqueezerrLASTpermuter)rvrVrrr image_typer=r=r>_process_images$        z%BaseImageProcessorFast._process_imagecsn|j||d}t|j|||dt|dkot|dttf}|r,fdd|D}|Sfdd|D}|S)a Prepare image-like inputs for processing. Args: images (`ImageInput`): The image-like inputs to process. do_convert_rgb (`bool`, *optional*): Whether to convert the images to RGB. input_data_format (`str` or `ChannelDimension`, *optional*): The input data format of the images. device (`torch.device`, *optional*): The device to put the processed images on. expected_ndims (`int`, *optional*): The expected number of dimensions for the images. (can be 2 for segmentation maps etc.) Returns: List[`torch.Tensor`]: The processed images. rrrrrcsg|] }fdd|DqS)cg|]}|qSr=r=rPprocess_image_partialr=r>rJrKzPBaseImageProcessorFast._prepare_image_like_inputs...r=)rH nested_listrr=r>rJszEBaseImageProcessorFast._prepare_image_like_inputs..crr=r=rPrr=r>rJrK)rrrlen isinstancerstuple)rvrNrrrrhas_nested_structurerr=rr>_prepare_image_like_inputss z1BaseImageProcessorFast._prepare_image_like_inputsr3rer9c Ks|duri}|durtd it||d}|dur$td it|dd}|dur3td it|dd}t|tr_further_process_kwargss0   z.BaseImageProcessorFast._further_process_kwargsr4r2r8c Ks$t||||||||| | | | d dS)z@ validate the kwargs for the preprocess method. ) r-r.r/r0r1r4r5r2r3r6r8r9N)r?)rvr-r.r/r0r1r4r5r2r3r6r8r9rdr=r=r>_validate_preprocess_kwargs$s z2BaseImageProcessorFast._validate_preprocess_kwargscOst||jdt|j||jD] }||t||dq|d}|d}|d}|jdi|}|j di||d|j |g|R|||d|S)N)captured_kwargsvalid_processor_keysrrrr9rr=) rrtrur ro setdefaultrrrnrr_preprocess_image_like_inputs)rvrNargsrdrrrrr=r=r> preprocessFs&      z!BaseImageProcessorFast.preprocessrcOs*|j||||d}|j|g|Ri|S)z Preprocess image-like inputs. To be overridden by subclasses when image-like inputs other than images should be processed. It can be used for segmentation maps, depth maps, etc. )rNrrr)r _preprocess)rvrNrrrrrdr=r=r>resz4BaseImageProcessorFast._preprocess_image_like_inputsdo_padc Kst||d\}}i}|D]\}}|r|j|||d}|||<qt||}t||d\}}i}|D]\}}|r@|||}||||| | | }|||<q4t||}| r^|j|| |d}td|i|dS)N)r)rVr5r6)rirrc)data tensor_type)rrrrrrrr )rvrNr4r5r6r2r3r-r.r/r0r1rrirr8rdrrresized_images_groupedrOrresized_imagesrrr=r=r>rys(      z"BaseImageProcessorFast._preprocesscsvt}i}|D]!\}}|dur(tt||d}|dkr'|dur'|||<q |||<q |dd|dd|S)N NOT_FOUND_valid_processor_keysru)rkto_dictrrrtypern)rv encoder_dict filtered_dictrwvalue class_defaultryr=r>rs    zBaseImageProcessorFast.to_dict)Nrr|FFF)NT)NNNNNN)r)NNN)NNNr)NNNNNNN) NNNNNNNNNNNN)F__name__ __module__ __qualname____doc__rr0r1r5rer3r4r2rrir-r.r/rr8rr<r9rrmodel_input_namesimage_seq_lengthrrorrrlpropertyboolr{rsrrstrrrrrr staticmethodrrrrrrrrrrrdictrmrrrrr rr!r rrrr __classcell__r=r=ryr>rbsJ{  K 8           )    )  . 2     "$        /rb)N)Kcollections.abcrcopyr functoolsrrtypingrrrnumpynphuggingface_hub.dataclassesr image_processing_utilsr r r image_transformsrrrrr image_utilsrrrrrrrrrrrprocessing_utilsrrutilsr r!r"r#r$r%utils.import_utilsr&r'r(r$torchvision.transforms.v2.functional transformsv2 functionalrr) get_loggerrrr<rrrsrr?rrErMrrUndarrayrarbr=r=r=r>s   4       &