o
    it>                     @   s   d dl mZmZ d dlZddlmZ ddlmZm	Z	m
Z
 ddlmZmZmZmZmZ ddlmZmZ ddlmZ G d	d
 d
eddZG dd deddZG dd deZdgZdS )    )OptionalUnionN   )BatchFeature)
ImageInputconcatenate_listmake_flat_list_of_images)ImagesKwargsMultiModalDataProcessingKwargsProcessorMixinUnpack)PreTokenizedInput	TextInput)
VideoInputc                   @   s2   e Zd ZU ee ed< ee ed< ee ed< dS )InternVLImagesKwargscrop_to_patchesmin_patchesmax_patchesN)__name__
__module____qualname__r   bool__annotations__int r   r   d/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/internvl/processing_internvl.pyr      s   
 r   F)totalc                   @   s0   e Zd ZU eed< dddddiddid	Zd
S )InternVLProcessorKwargsimages_kwargsleftF)padding_sidereturn_mm_token_type_idsr   Treturn_tensorspt)text_kwargsr   videos_kwargsN)r   r   r   r   r   	_defaultsr   r   r   r   r   !   s   
 
r   c                       s   e Zd ZdZg dZdZdZdZ					ddef fd	d
Z	de
e de
e de
e dejdejdejfddZ				ddee deeeee
e e
e f  dee dee def
ddZdddZedd Z  ZS ) InternVLProcessoraM  
    Constructs a InternVL processor which wraps a [`AutoImageProcessor`] and
    [`PretrainedTokenizerFast`] tokenizer into a single processor that inherits both the image processor and
    tokenizer functionalities. See the [`~InternVLProcessor.__call__`] and [`~InternVLProcessor.decode`] for more information.
    Args:
        image_processor ([`AutoImageProcessor`], *optional*):
            The image processor is a required input.
        tokenizer ([`PreTrainedTokenizer`, `PreTrainedTokenizerFast`], *optional*):
            The tokenizer is a required input.
        video_processor ([`AutoVideoProcessor`], *optional*):
            The video processor is a required input.
        image_seq_length (`int`, *optional*, defaults to 256):
            The number of image token to use per image patch. it should be set so that:
            image_seq_length = (config.image_size // config.patch_size) ** 2 * (config.scale_factor**2)
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    )image_processor	tokenizervideo_processorAutoImageProcessorAutoVideoProcessorAutoTokenizerN   image_seq_lengthc                    sp   || _ |j| _|j| _|j| _|j| _|j| _|j| _|j| _	| j	| j| jg| _
t j|||fd|i| d S )Nchat_template)r0   start_image_tokenend_image_tokenstart_image_token_idend_image_token_idcontext_image_tokenimage_tokenvideo_tokencontext_image_token_idimage_token_id	image_idssuper__init__)selfr)   r*   r+   r0   r1   kwargs	__class__r   r   r=   I   s   	 zInternVLProcessor.__init__textimage_num_patchesvideo_num_patchesimage_num_patches_indicesvideo_num_patches_indicesvideo_patch_indicesc	                    s  d}	d}
g }g }g }|D ]}|}j |v sj|v rj |v rkj|vs0|j |jk rk|	dkr:||	d  nd}||	 }||||  |j dd}|j j j ||	   j  |	d7 }	nE||
 }||
d  }|| }|| }||||  t|||  d	 fddt
t D }|| |jdd}|
d7 }
j |v sj|v sd|v r|d}|d|d}d|v s|| q|||	|
fS )z
        Processes interleaved text with <image> and <video> placeholders, replacing them with appropriate
        image and video tokens while keeping track of the patches used.
        r      z<placeholder>
c                 3   s@    | ]}d |d  dj  jj  |   j V  qdS )FramerH   z: N)r2   r7   r0   r3   ).0inum_patchesr>   r   r   	<genexpr>   s
    .
z?InternVLProcessor._insert_media_placeholders.<locals>.<genexpr>)r7   r8   indexappendreplacer2   r0   r3   listjoinrangelenpop)r>   rB   image_pixel_valuesvideo_pixel_valuesrC   rD   rE   rF   rG   image_indexvideo_indexprocessed_textimage_video_patchesreplace_stringsprompt
new_promptstart_index	end_indexcurrent_patch_indexend_patch_indexvideo_promptreplace_strr   rM   r   _insert_media_placeholders^   sL   

"


!
z,InternVLProcessor._insert_media_placeholdersimagesvideosr?   returnc              
   K   s  |du rt d| jtfd| jji|}t|ttfs|g}g }d}t	dg}	|durR| j
|}t|}| j
dd|i|d }
|
d}|
d}t|}	g }d}t	dg}t	dg}|dur|d	 }| jdd
|i|}|d}|j^}}}t||}t|}t|d t}d|d< t||dd< dg| }t|d t}d|d< t||dd< |dd}i }|dus|dur| ||||||	||\}}}}|dur|t|krt d|dur|t|krt ddt|i}|d dd}|d dd}| j|fi |d }| j||dgd |r>t	|d }t|d }d|t|| j< | |d< ti |||dS )a	  
        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
        and `kwargs` arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizerFast.__call__`] to encode the text if `text`
        is not `None`, otherwise encode default OCR queries which depends on the `format`, `box`, `color`, `multi_page` and
        `crop_to_patches` arguments. To prepare the vision inputs, this method forwards the `images` and `kwargs` arguments to
        GotOcr2ImageProcessor's [`~GotOcr2ImageProcessor.__call__`] if `images` is not `None`.

        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. Both channels-first and channels-last formats are supported.
            text (`str`, `list[str]`, `list[list[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
                The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors of a particular framework. Acceptable values are:
                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return NumPy `np.ndarray` objects.
                - `'jax'`: Return JAX `jnp.ndarray` objects.

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
        NzYou have to specify text.tokenizer_init_kwargsr   rh   r   rN   pixel_valuesr&   ri   pixel_values_videosrH   zONumber of image placeholders in the prompt does not match the number of images.zONumber of video placeholders in the prompt does not match the number of videos.r%   r#   r"   image)
modalities	input_idsmm_token_type_ids)datatensor_typer   )
ValueError_merge_kwargsr   r*   init_kwargs
isinstancerS   tuplenparrayr)   fetch_imagesr   rW   cumsumr+   shapefullsumemptyr   flattenrg   rV   r   _check_special_mm_tokens
zeros_likeisinr;   tolistr   )r>   rh   rB   audiori   r?   output_kwargsrC   rX   rE   image_inputsrD   rY   rG   rF   video_kwargsvideo_inputs
batch_size
num_frames_num_frames_per_videoimage_videos_inputsr]   rZ   r[   r#   r"   text_inputs	array_idsrq   r   r   r   __call__   s   )





zInternVLProcessor.__call__c                    sh   i }|dur-t jdi   |  fdd|D }fdd|D }|||d tdi |S )a  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr   c                    s"   g | ]}j jg | R  qS r   )r)   get_number_of_image_patches)rK   
image_sizer   r>   r   r   
<listcomp>'  s    z@InternVLProcessor._get_num_multimodal_tokens.<locals>.<listcomp>c                    s   g | ]	}d  j |  qS )   )r0   )rK   rN   )r>   r   r   r   ,  s    )num_image_tokensnum_image_patchesr   )r   r'   getupdater
   )r>   image_sizesr?   vision_datar   r   r   r   r   _get_num_multimodal_tokens  s   
z,InternVLProcessor._get_num_multimodal_tokensc                 C   s   | j j}| jj}|| S N)r*   model_input_namesr)   )r>   tokenizer_input_namesimage_processor_input_namesr   r   r   r   1  s   z#InternVLProcessor.model_input_names)NNNr/   N)NNNNr   )r   r   r   __doc__
attributesimage_processor_classvideo_processor_classtokenizer_classr   r=   rS   strry   ndarrayrg   r   r   r   r   r   r   r   r   r   r   r   propertyr   __classcell__r   r   r@   r   r(   1   sZ    	
B

wr(   )typingr   r   numpyry   image_processing_utilsr   image_utilsr   r   r   processing_utilsr	   r
   r   r   r   tokenization_utils_baser   r   video_utilsr   r   r   r(   __all__r   r   r   r   <module>   s     

