o
    ei1                     @   s   d dl ZddlmZ ddlmZmZmZ ddlm	Z	m
Z
mZmZ ddlmZmZ ddlmZ ddlmZ G d	d
 d
e
ddZeG dd deZdgZdS )    N   )BatchFeature)
ImageInputconcatenate_listmake_flat_list_of_images)MultiModalDataProcessingKwargsProcessorMixinUnpack)PreTokenizedInput	TextInput)auto_docstring)
VideoInputc                   @   s&   e Zd ZdddddiddidZd	S )
InternVLProcessorKwargsleftF)padding_sidereturn_mm_token_type_idscrop_to_patchesTreturn_tensorspt)text_kwargsimages_kwargsvideos_kwargsN)__name__
__module____qualname__	_defaults r   r   n/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/internvl/processing_internvl.pyr      s    
r   F)totalc                       s   e Zd Z					ddef fddZdee dee dee d	ejd
ejdejfddZ	e
			ddedB deeB ee B ee B dB dedB dee def
ddZdddZedd Z  ZS )InternVLProcessorN   image_seq_lengthc                    sp   || _ |j| _|j| _|j| _|j| _|j| _|j| _|j| _	| j	| j| jg| _
t j|||fd|i| dS )a  
        image_seq_length (`int`, *optional*, defaults to 256):
            The number of image token to use per image patch. it should be set so that:
            image_seq_length = (config.image_size // config.patch_size) ** 2 * (config.scale_factor**2)
        chat_templateN)r"   start_image_tokenend_image_tokenstart_image_token_idend_image_token_idcontext_image_tokenimage_tokenvideo_tokencontext_image_token_idimage_token_id	image_idssuper__init__)selfimage_processor	tokenizervideo_processorr"   r#   kwargs	__class__r   r   r/   +   s    zInternVLProcessor.__init__textimage_num_patchesvideo_num_patchesimage_num_patches_indicesvideo_num_patches_indicesvideo_patch_indicesc	                    s  d}	d}
g }g }g }|D ]}|}j |v sj|v rj |v rkj|vs0|j |jk rk|	dkr:||	d  nd}||	 }||||  |j dd}|j j j ||	   j  |	d7 }	nE||
 }||
d  }|| }|| }||||  t|||  d	 fddt
t D }|| |jdd}|
d7 }
j |v sj|v sd|v r|d}|d|d}d|v s|| q|||	|
fS )z
        Processes interleaved text with <image> and <video> placeholders, replacing them with appropriate
        image and video tokens while keeping track of the patches used.
        r      z<placeholder>
c                 3   s@    | ]}d |d  dj  jj  |   j V  qdS )Framer=   z: N)r$   r)   r"   r%   ).0inum_patchesr0   r   r   	<genexpr>w   s
    .
z?InternVLProcessor._insert_media_placeholders.<locals>.<genexpr>)r)   r*   indexappendreplacer$   r"   r%   listjoinrangelenpop)r0   r7   image_pixel_valuesvideo_pixel_valuesr8   r9   r:   r;   r<   image_indexvideo_indexprocessed_textimage_video_patchesreplace_stringsprompt
new_promptstart_index	end_indexcurrent_patch_indexend_patch_indexvideo_promptreplace_strr   rB   r   _insert_media_placeholdersE   sL   

"


!
z,InternVLProcessor._insert_media_placeholdersimagesvideosr4   returnc              
   K   s  |du rt d| jtfd| jji|}t|ttfs|g}g }d}t	dg}|durR| j
|}t|}| j
dd|i|d }	|	d}|	d}t|}g }
d}t	dg}t	dg}|dur|d	 }| jdd
|i|}|d}|j^}}}t||}t|}t|d t}d|d< t||dd< dg| }
t|d t}d|d< t|
|dd< |dd}i }|dus|dur| |||||
|||\}}}}|dur|t|krt d|dur|t|krt ddt|i}|d dd}|d dd}| j|fi |d }| j||dgd |r>t	|d }t|d }d|t|| j< | |d< ti |||dS )aA  
        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
        NzYou have to specify text.tokenizer_init_kwargsr   r]   r   rC   pixel_valuesr   r^   pixel_values_videosr=   zONumber of image placeholders in the prompt does not match the number of images.zONumber of video placeholders in the prompt does not match the number of videos.r   r   r   image)
modalities	input_idsmm_token_type_ids)datatensor_typer   )
ValueError_merge_kwargsr   r2   init_kwargs
isinstancerH   tuplenparrayr1   fetch_imagesr   rL   cumsumr3   shapefullsumemptyintflattenr\   rK   r   _check_special_mm_tokens
zeros_likeisinr-   tolistr   )r0   r]   r7   r^   r4   output_kwargsr8   rM   r:   image_inputsr9   rN   r<   r;   video_kwargsvideo_inputs
batch_size
num_frames_num_frames_per_videoimage_videos_inputsrR   rO   rP   r   r   text_inputs	array_idsrf   r   r   r   __call__   s   





zInternVLProcessor.__call__c                    sh   i }|dur-t jdi   |  fdd|D }fdd|D }|||d tdi |S )a  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr   c                    s"   g | ]}j jg | R  qS r   )r1   get_number_of_image_patches)r@   
image_sizer   r0   r   r   
<listcomp>   s    z@InternVLProcessor._get_num_multimodal_tokens.<locals>.<listcomp>c                    s   g | ]	}d  j |  qS )   )r"   )r@   rC   )r0   r   r   r      s    )num_image_tokensnum_image_patchesr   )r   r   getupdater   )r0   image_sizesr4   vision_datar   r   r   r   r   _get_num_multimodal_tokens   s   
z,InternVLProcessor._get_num_multimodal_tokensc                 C   s   | j j}| jj}|| S N)r2   model_input_namesr1   )r0   tokenizer_input_namesimage_processor_input_namesr   r   r   r     s   z#InternVLProcessor.model_input_names)NNNr!   N)NNNr   )r   r   r   rv   r/   rH   strrn   ndarrayr\   r   r   r   r   r   r
   r   r   r   r   propertyr   __classcell__r   r   r5   r   r    )   sP    	
@
_r    )numpyrn   image_processing_utilsr   image_utilsr   r   r   processing_utilsr   r   r	   r
   tokenization_utils_baser   r   utilsr   video_utilsr   r   r    __all__r   r   r   r   <module>   s    
`