o
    wi?                     @   s   d dl mZmZ d dlZddlmZ ddlmZm	Z	m
Z
 ddlmZmZmZmZmZ ddlmZmZ ddlmZmZ G d	d
 d
eddZG dd deddZG dd deZdgZdS )    )OptionalUnionN   )BatchFeature)
ImageInputconcatenate_listmake_flat_list_of_images)ImagesKwargsMultiModalDataProcessingKwargsProcessorMixinUnpack)PreTokenizedInput	TextInput)
VideoInputmake_batched_videosc                   @   s2   e Zd ZU ee ed< ee ed< ee ed< dS )InternVLImagesKwargscrop_to_patchesmin_patchesmax_patchesN)__name__
__module____qualname__r   bool__annotations__int r   r   m/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/internvl/processing_internvl.pyr      s   
 r   F)totalc                   @   s,   e Zd ZU eed< dddddii dZdS )	InternVLProcessorKwargsimages_kwargsleftF)padding_sidereturn_mm_token_type_idsr   T)text_kwargsr    videos_kwargsN)r   r   r   r   r   	_defaultsr   r   r   r   r   !   s   
 
r   c                       s   e Zd ZdZg dZdZdZdZ					d!def fd	d
Z	de
e de
e de
e dejdejdejfddZ				d"dee deeeee
e e
e f  dee dee def
ddZd#ddZdd Zdd Zedd  Z  ZS )$InternVLProcessoraM  
    Constructs a InternVL processor which wraps a [`AutoImageProcessor`] and
    [`PretrainedTokenizerFast`] tokenizer into a single processor that inherits both the image processor and
    tokenizer functionalities. See the [`~InternVLProcessor.__call__`] and [`~InternVLProcessor.decode`] for more information.
    Args:
        image_processor ([`AutoImageProcessor`], *optional*):
            The image processor is a required input.
        tokenizer ([`PreTrainedTokenizer`, `PreTrainedTokenizerFast`], *optional*):
            The tokenizer is a required input.
        video_processor ([`AutoVideoProcessor`], *optional*):
            The video processor is a required input.
        image_seq_length (`int`, *optional*, defaults to 256):
            The number of image token to use per image patch. it should be set so that:
            image_seq_length = (config.image_size // config.patch_size) ** 2 * (config.scale_factor**2)
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    )image_processor	tokenizervideo_processorAutoImageProcessorAutoVideoProcessorAutoTokenizerN   image_seq_lengthc                    sp   || _ |j| _|j| _|j| _|j| _|j| _|j| _|j| _	| j	| j| jg| _
t j|||fd|i| d S )Nchat_template)r/   start_image_tokenend_image_tokenstart_image_token_idend_image_token_idcontext_image_tokenimage_tokenvideo_tokencontext_image_token_idimage_token_id	image_idssuper__init__)selfr(   r)   r*   r/   r0   kwargs	__class__r   r   r<   G   s   	 zInternVLProcessor.__init__textimage_num_patchesvideo_num_patchesimage_num_patches_indicesvideo_num_patches_indicesvideo_patch_indicesc	                    s  d}	d}
g }g }g }|D ]}|}j |v sj|v rȈj |v rkj|vs0|j |jk rk|	dkr:||	d  nd}||	 }||||  |j dd}|j j j ||	   j  |	d7 }	nS|
dkru||
d  nd}||
 }|
dkr|| nd}||d  }||||  t|||  d	 fddt
t D }|| |jdd}|
d7 }
j |v sj|v sd|v r|d}|d|d}d|v s|| q|||	|
fS )z
        Processes interleaved text with <image> and <video> placeholders, replacing them with appropriate
        image and video tokens while keeping track of the patches used.
        r      z<placeholder>
c                 3   s@    | ]}d |d  dj  jj  |   j V  qdS )FramerG   z: N)r1   r6   r/   r2   ).0inum_patchesr=   r   r   	<genexpr>   s
    .
z?InternVLProcessor._insert_media_placeholders.<locals>.<genexpr>)r6   r7   indexappendreplacer1   r/   r2   listjoinrangelenpop)r=   rA   image_pixel_valuesvideo_pixel_valuesrB   rC   rD   rE   rF   image_indexvideo_indexprocessed_textimage_video_patchesreplace_stringsprompt
new_promptstart_index	end_indexcurrent_patch_indexend_patch_indexvideo_promptreplace_strr   rL   r   _insert_media_placeholders\   sL   

"


!
z,InternVLProcessor._insert_media_placeholdersimagesvideosr>   returnc              
   K   s:  |du rt d| jtfd| jji|}t|ttfs|g}g }g }i }	d}
d}t	dg}t	dg}t	dg}|dur^t
|}| jdd|i|d }|d}|d}
t|}|durt|}| jdd	|i|d
 }|d}dd |D }dd |D }t|}t|}|dd}|dus|dur| ||
||||||\}}}}|dur|t|krt d|dur|t|krt ddt|i}	|d dd}|d dd}| j|fi |d }| j||dgd |rt	|d }t|d }d|t|| j< | |d< ti ||	|dS )a	  
        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
        and `kwargs` arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizerFast.__call__`] to encode the text if `text`
        is not `None`, otherwise encode default OCR queries which depends on the `format`, `box`, `color`, `multi_page` and
        `crop_to_patches` arguments. To prepare the vision inputs, this method forwards the `images` and `kwrags` arguments to
        GotOcr2ImageProcessor's [`~GotOcr2ImageProcessor.__call__`] if `images` is not `None`.

        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. Both channels-first and channels-last formats are supported.
            text (`str`, `list[str]`, `list[list[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
                The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors of a particular framework. Acceptable values are:
                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return NumPy `np.ndarray` objects.
                - `'jax'`: Return JAX `jnp.ndarray` objects.

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
        NzYou have to specify text.tokenizer_init_kwargsr   rg   r    rM   pixel_valuesrh   r%   pixel_values_videosc                 S   s   g | ]}t |qS r   )rU   )rJ   videor   r   r   
<listcomp>   s    z.InternVLProcessor.__call__.<locals>.<listcomp>c                 S   s   g | ]}t |D ]}d qqS )rG   )rT   )rJ   frames_r   r   r   rn      s    rG   zONumber of image placeholders in the prompt does not match the number of images.zONumber of video placeholders in the prompt does not match the number of videos.r$   return_tensorsr#   image)
modalities	input_idsmm_token_type_ids)datatensor_typer   )
ValueError_merge_kwargsr   r)   init_kwargs
isinstancerR   tuplenparrayr   r(   rV   cumsumr   r*   flattenrf   rU   r   _check_special_mm_tokens
zeros_likeisinr:   tolistr   )r=   rg   rA   audiorh   r>   output_kwargsrB   rC   image_videos_inputsrW   rX   rD   rF   rE   image_inputsvideo_inputsnum_frames_per_videor\   rY   rZ   rq   r#   text_inputs	array_idsru   r   r   r   __call__   sx   )






zInternVLProcessor.__call__c                    sh   i }|dur-t jdi   |  fdd|D }fdd|D }|||d tdi |S )a  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr    c                    s"   g | ]}j jg | R  qS r   )r(   get_number_of_image_tokens)rJ   
image_sizer    r=   r   r   rn     s    z@InternVLProcessor._get_num_multimodal_tokens.<locals>.<listcomp>c                    s   g | ]	}d  j |  qS )   )r/   )rJ   rM   )r=   r   r   rn   #  s    )num_image_tokensnum_image_patchesr   )r   r&   getupdater
   )r=   image_sizesr>   vision_datar   r   r   r   r   _get_num_multimodal_tokens  s   
z,InternVLProcessor._get_num_multimodal_tokensc                 O      | j j|i |S )z
        This method forwards all its arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
        refer to the docstring of this method for more information.
        )r)   batch_decoder=   argsr>   r   r   r   r   (     zInternVLProcessor.batch_decodec                 O   r   )z
        This method forwards all its arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
        the docstring of this method for more information.
        )r)   decoder   r   r   r   r   /  r   zInternVLProcessor.decodec                 C   s    | j j}| jj}t|t| S N)r)   model_input_namesr(   rR   )r=   tokenizer_input_namesimage_processor_input_namesr   r   r   r   6  s   z#InternVLProcessor.model_input_names)NNNr.   N)NNNNr   )r   r   r   __doc__
attributesimage_processor_classvideo_processor_classtokenizer_classr   r<   rR   strr}   ndarrayrf   r   r   r   r   r   r   r   r   r   r   r   r   r   propertyr   __classcell__r   r   r?   r   r'   /   s^    	
B

pr'   )typingr   r   numpyr}   image_processing_utilsr   image_utilsr   r   r   processing_utilsr	   r
   r   r   r   tokenization_utils_baser   r   video_utilsr   r   r   r   r'   __all__r   r   r   r   <module>   s     
