o
    	۷iP                     @   s<  d dl mZ d dlmZmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ eeddG dd deZeeddG dd deZG dd dejZeG dd deZeddG dd deZeddG dd  d eeZg d!Z dS )"    )	dataclass)OptionalUnionN)nn   )ACT2FN)Cache)GenerationMixin)BaseModelOutputWithPastModelOutput)PreTrainedModel)auto_docstringcan_return_tuple   )	AutoModel   )VipLlavaConfigzM
    Base class for VipLlava outputs, with hidden states and attentions.
    )custom_introc                   @   s$   e Zd ZU dZdZeej ed< dS )VipLlavaModelOutputWithPasta  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nimage_hidden_states)	__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__ r   r   d/home/ubuntu/vllm_env/lib/python3.10/site-packages/transformers/models/vipllava/modeling_vipllava.pyr   &   s   
 r   zT
    Base class for VipLlava causal language model (or autoregressive) outputs.
    c                   @   s   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
ee ed< dZeeej  ed< dZeeej  ed< dZeej ed< dS )	VipLlavaCausalLMOutputWithPasta4  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nlosslogitspast_key_valueshidden_states
attentionsr   )r   r   r   r   r    r   r   r   r   r!   r"   r   r#   tupler$   r   r   r   r   r   r   ;   s   
 r   c                       s*   e Zd Zdef fddZdd Z  ZS )VipLlavaMultiModalProjectorconfigc                    s   t    t|jtrdnt|j}tj||jj	 |j
d| _tj||jj	 |jj	dd| _t|j | _tj|jj	|jj	dd| _d S )Nr   )epsTbias)super__init__
isinstancevision_feature_layersintlenr   	LayerNormvision_confighidden_sizeprojector_layernorm_epsprojector_layernormLineartext_configlinear_1r   projector_hidden_actactlinear_2)selfr'   num_feature_layers	__class__r   r   r,   Z   s   

z$VipLlavaMultiModalProjector.__init__c                 C   s,   |  |}| |}| |}| |}|S N)r5   r8   r:   r;   )r<   r#   r   r   r   forwardi   s
   



z#VipLlavaMultiModalProjector.forward)r   r   r   r   r,   rA   __classcell__r   r   r>   r   r&   Y   s    r&   c                   @   s6   e Zd ZU eed< dZdZdZdZdZ	dZ
dZdZdS )VipLlavaPreTrainedModelr'    Tr"   N)r   r   r   r   r   base_model_prefixsupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph_supports_flex_attn_supports_attention_backendr   r   r   r   rC   q   s   
 rC   zx
    The VipLlava model which consists of a vision backbone and a language model, without a language modeling head.
    c                       s:  e Zd ZddiZdef fddZdd Zdd	 Zd
d Zdd Z		d#de
jdeeeee f  fddZde
jde
jde
jfddZe												d$dee
j dee
j dee
j dee
j dee dee
j deeeee f  dee dee dee dee dee
j d eeef fd!d"Z  ZS )%VipLlavaModelzlanguage_model.modellanguage_modelr'   c                    s>   t  | t|j| _t|| _t|j| _	| 
  d S r@   )r+   r,   r   from_configr2   vision_towerr&   multi_modal_projectorr7   rN   	post_initr<   r'   r>   r   r   r,      s
   
zVipLlavaModel.__init__c                 C   
   | j  S r@   )rN   get_input_embeddingsr<   r   r   r   rU         
z"VipLlavaModel.get_input_embeddingsc                 C      | j | d S r@   )rN   set_input_embeddingsr<   valuer   r   r   rY         z"VipLlavaModel.set_input_embeddingsc                 C   s
   || _ d S r@   rN   r<   decoderr   r   r   set_decoder   rW   zVipLlavaModel.set_decoderc                 C      | j S r@   r]   rV   r   r   r   get_decoder      zVipLlavaModel.get_decoderNpixel_valuesr.   c                    sv   |dur|n| j j}| j|dd t|tr$ j| ddddf }n fdd|D }tj|dd}| |}|S )	aW  
        Obtains image last hidden states from the vision tower and apply multimodal projection.

        Args:
            pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
               The tensors corresponding to the input images.
            vision_feature_layers (`Union[int, list[int]]`):
                The vision feature layer, or the list of indexes of the layers to select
                the vision feature.
        Returns:
            image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
        NT)output_hidden_statesr   c                    s&   g | ]} j | d d dd f qS )Nr   )r#   ).0indeximage_outputsr   r   
<listcomp>   s   & z4VipLlavaModel.get_image_features.<locals>.<listcomp>)dim)	r'   r.   rP   r-   r/   r#   r   catrQ   )r<   rd   r.   image_featuresr   rh   r   get_image_features   s   

z VipLlavaModel.get_image_features	input_idsinputs_embedsrn   c                 C   s   |du r||   tj| jjtj|jdk}|d}n|| jjk}| }|	d
||j}|jd |jd  }||  | krPtd| d| |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        N)dtypedevicerk   r   r   z6Image features and image tokens do not match: tokens: z, features )rU   r   tensorr'   image_token_idlongrs   allsum	unsqueeze	expand_astoshapenumel
ValueError)r<   rp   rq   rn   special_image_maskn_image_tokensn_image_featuresr   r   r   get_placeholder_mask   s   z"VipLlavaModel.get_placeholder_maskattention_maskposition_idsr"   	use_cacheoutput_attentionsre   return_dictcache_positionreturnc                 K   s  |	dur|	n| j j}	|
dur|
n| j j}
|dur|n| j j}|dur$|n| j j}|du |duA r4td|du r>|  |}|dur_| j||d}||j	|j
}| j|||d}|||}| jd||||||	|
d|d	|}t|j|j|j|j|dur|ndd}|r|S | S )	z
        vision_feature_layers (`Union[int, list[int]]`, *optional*):
            The vision feature layer, or the list of indexes of the layers to select
            the vision feature.
        Nz:You must specify exactly one of input_ids or inputs_embedsrd   r.   )rq   rn   T)	r   r   r"   rq   r   r   re   r   r   )last_hidden_stater"   r#   r$   r   r   )r'   r   re   use_return_dictr.   r~   rU   ro   r{   rs   rr   r   masked_scatterrN   r   r   r"   r#   r$   to_tuple)r<   rp   rd   r   r   r"   rq   r.   r   r   re   r   r   	lm_kwargsrn   r   outputsoutputr   r   r   rA      sP   
zVipLlavaModel.forwardr@   )NNNNNNNNNNNN)r   r   r   _checkpoint_conversion_mappingr   r,   rU   rY   r`   rb   r   r   r   r   r/   listro   
LongTensorr   r   Tensorr   boolr%   r   rA   rB   r   r   r>   r   rM      sx    

	

rM   zV
    The VIPLLAVA model which consists of a vision backbone and a language model.
    c                #       s  e Zd ZdddddZdgZdef fdd	Zd
d Zdd Zde	j
fddZdd Zdd Z	d1dejdeeeee f  fddZedd Zedd Zedd Zee														 d2d!eej deej d"eej d#eej d$ee d%eej deeeee f  d&eej d'ee d(ee d)ee d*ee d+eej d,eeejf deee f fd-d.Z!						d3 fd/d0	Z"  Z#S )4 VipLlavaForConditionalGenerationzmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)z^language_model.modelz^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightr'   c                    s<   t  | t|| _tj|jj|jjdd| _	| 
  d S )NFr)   )r+   r,   rM   modelr   r6   r7   r3   
vocab_sizer   rR   rS   r>   r   r   r,   '  s   
z)VipLlavaForConditionalGeneration.__init__c                 C   rT   r@   )r   rU   rV   r   r   r   rU   -  rW   z5VipLlavaForConditionalGeneration.get_input_embeddingsc                 C   rX   r@   )r   rY   rZ   r   r   r   rY   0  r\   z5VipLlavaForConditionalGeneration.set_input_embeddingsr   c                 C   ra   r@   )r   rV   r   r   r   get_output_embeddings3  rc   z6VipLlavaForConditionalGeneration.get_output_embeddingsc                 C   rX   r@   )r   r`   r^   r   r   r   r`   6  r\   z,VipLlavaForConditionalGeneration.set_decoderc                 C   rT   r@   )r   rb   rV   r   r   r   rb   9  rW   z,VipLlavaForConditionalGeneration.get_decoderNrd   r.   c                 C   s   | j j||dS )Nr   )r   ro   )r<   rd   r.   r   r   r   ro   <  s   z3VipLlavaForConditionalGeneration.get_image_featuresc                 C      | j jS r@   )r   rN   rV   r   r   r   rN   B     z/VipLlavaForConditionalGeneration.language_modelc                 C   r   r@   )r   rP   rV   r   r   r   rP   F  r   z-VipLlavaForConditionalGeneration.vision_towerc                 C   r   r@   )r   rQ   rV   r   r   r   rQ   J  r   z6VipLlavaForConditionalGeneration.multi_modal_projectorr   rp   r   r   r"   rq   labelsr   r   re   r   r   logits_to_keepc                 K   s   |
dur|
n| j j}
|dur|n| j j}|dur|n| j j}|dur$|n| j j}| jd|||||||	||
|d|d|}|d }t|trLt| dn|}| 	|dd|ddf }d}|durm| j
||| j jjd}t|||j|j|j|jdS )a  
        vision_feature_layers (`Union[int, list[int]]`, *optional*):
            The vision feature layer, or the list of indexes of the layers to select
            the vision feature.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> import torch
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, VipLlavaForConditionalGeneration

        >>> model = VipLlavaForConditionalGeneration.from_pretrained("llava-hf/vip-llava-7b-hf", device_map="auto", dtype=torch.float16)
        >>> processor = AutoProcessor.from_pretrained("llava-hf/vip-llava-7b-hf")

        >>> prompt = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.###Human: <image>\n{}###Assistant:"
        >>> question = "Can you please describe this image?"
        >>> prompt = prompt.format(question)
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/compel-neg.png"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(text=text, images=image, return_tensors="pt").to(0, torch.float16)

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=20)
        >>> processor.decode(generate_ids[0][len(inputs["input_ids"][0]):], skip_special_tokens=True)
        The image features a brown and white cat sitting on a green surface, with a red ball in its
        ```NT)rp   rd   r   r   r"   rq   r   r.   r   re   r   r   r   )r!   r   r   )r    r!   r"   r#   r$   r   r   )r'   r   re   r   r.   r   r-   r/   slicer   loss_functionr7   r   r   r"   r#   r$   r   )r<   rp   rd   r   r   r"   rq   r.   r   r   r   re   r   r   r   r   r   r#   slice_indicesr!   r    r   r   r   rA   N  sH   6z(VipLlavaForConditionalGeneration.forwardc           
         s8   t  j|f|||||d|}	|d dkr||	d< |	S )N)r"   rq   r   r   r   r   rd   )r+   prepare_inputs_for_generation)
r<   rp   r"   rq   rd   r   r   r   kwargsmodel_inputsr>   r   r   r     s   
z>VipLlavaForConditionalGeneration.prepare_inputs_for_generationr@   )NNNNNNNNNNNNNr   )NNNNNN)$r   r   r   r   _tied_weights_keysr   r,   rU   rY   r   Moduler   r`   rb   r   r   r   r   r/   r   ro   propertyrN   rP   rQ   r   r   r   r   r   r   r%   r   rA   r   rB   r   r   r>   r   r     s    



	

br   )rM   r   rC   )!dataclassesr   typingr   r   r   r   activationsr   cache_utilsr   
generationr	   modeling_outputsr
   r   modeling_utilsr   utilsr   r   autor   configuration_vipllavar   r   r   r   r&   rC   rM   r   __all__r   r   r   r   <module>   sH     1