o
    in                     @   s"  d Z ddlZddlmZ ddlZddlm  mZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z(m)Z)m*Z* G dd de&Z+G dd de%Z,G dd de$Z-G dd de*Z.G dd de"Z/G dd de!Z0G dd  d ej1Z2G d!d" d"eZ3G d#d$ d$e(Z4G d%d& d&eZ5G d'd( d(e)Z6G d)d* d*ej1Z7eG d+d, d,eZ8ed-d.G d/d0 d0e8Z9ed1d.G d2d3 d3e8Z:eG d4d5 d5eZ;g d6Z<dS )7z%Pytorch implementation of AIMv2 Model    N)Optional)nn   )create_causal_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPooling)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)check_model_inputs   )	CLIPModelCLIPTextEmbeddings_get_vector_norm)LlamaMLPLlamaRMSNorm)SiglipConfigSiglipTextConfigSiglipVisionConfig)SiglipAttentionSiglipEncoderSiglipOutputc                       s~   e Zd ZdZ										
						d dededededededededededededededef fddZ  Z	S )!Aimv2VisionConfiga  
    This is the configuration class to store the configuration of a [`Aimv2VisionModel`]. It is used to instantiate a
    AIMv2 vision encoder according to the specified arguments, defining the model architecture. Instantiating a
    configuration with the defaults will yield a similar configuration to that of the vision encoder of the AIMv2
    [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the encoder layers and the pooler layer.
        intermediate_size (`int`, *optional*, defaults to 2816):
            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
        num_hidden_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer encoder.
        num_channels (`int`, *optional*, defaults to 3):
            Number of channels in the input images.
        image_size (`int`, *optional*, defaults to 224):
            The size (resolution) of each image.
        patch_size (`int`, *optional*, defaults to 14):
            The size (resolution) of each patch.
        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the rms normalization layers.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        qkv_bias (`bool`, *optional*, defaults to `False`):
            Whether to add a bias to the queries, keys and values.
        mlp_bias (`bool`, *optional*, defaults to `False`):
            Whether to add a bias to the Linear layers or Not.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the for initializing all weight matrices.
        use_head (`str`, *optional*, defaults to `True`):
            Whether to use Attention Pooling Head or Not.
        is_native (`str`, *optional*, defaults to `False`):
            Whether to use ckpt trained for image native resolution or not.
    Example:

    ```python
    >>> from transformers import SiglipVisionConfig, SiglipVisionModel

    >>> # Initializing a Aimv2VisionConfig with apple/aimv2-large-patch14-224 style configuration
    >>> configuration = Aimv2VisionConfig()

    >>> # Initializing a Aimv2VisionModel (with random weights) from the apple/aimv2-large-patch14-224 style configuration
    >>> model = Aimv2VisionModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```            r         h㈵>        Fsilu{Gz?Thidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_channels
image_size
patch_sizerms_norm_epsattention_dropoutqkv_biasmlp_bias
hidden_actinitializer_rangeuse_head	is_nativec                    sX   t  jd|||||||||
d	| || _|| _|	| _|| _|
| _|| _|| _| `	d S )N)	r'   r(   r)   r*   r2   r+   r,   r-   r0    )
super__init__r4   r3   r/   r1   r0   r.   r5   layer_norm_eps)selfr'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   kwargs	__class__r6   d/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/aimv2/modular_aimv2.pyr8   d   s*   
zAimv2VisionConfig.__init__)r   r   r   r    r   r!   r"   r#   r$   FFr%   r&   TF)
__name__
__module____qualname____doc__intfloatboolstrr8   __classcell__r6   r6   r<   r>   r   +   s`    :	
r   c                       s   e Zd ZdZ												
					d dededededededededededee dee dededef fddZ	  Z
S )!Aimv2TextConfiga  
    This is the configuration class to store the configuration of a [`Aimv2TextModel`]. It is used to instantiate a
    AIMv2 text encoder according to the specified arguments, defining the model architecture. Instantiating a
    configuration with the defaults will yield a similar configuration to that of the text encoder of the AIMv2
    [apple/aimv2-large-patch14-224-lit](https://huggingface.co/apple/aimv2-large-patch14-224-lit) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 49408):
            Vocabulary size of the AIMv2 text model. Defines the number of different tokens that can be represented by
            the `inputs_ids` passed when calling [`Aimv2Model`].
        hidden_size (`int`, *optional*, defaults to 768):
            Dimensionality of the encoder layers and the pooler layer.
        intermediate_size (`int`, *optional*, defaults to 2048):
            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
        num_hidden_layers (`int`, *optional*, defaults to 12):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 6):
            Number of attention heads for each attention layer in the Transformer encoder.
        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the rms normalization layers.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        qkv_bias (`bool`, *optional*, defaults to `False`):
            Whether to add a bias to the queries, keys and values.
        mlp_bias (`bool`, *optional*, defaults to `False`):
            Whether to add a bias to the Linear layers or Not.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
        pad_token_id (`int`, *optional*, defaults to 1):
            The id of the padding token in the vocabulary.
        bos_token_id (`int`, *optional*, defaults to 49406):
            The id of the beginning-of-sequence token in the vocabulary.
        eos_token_id (`int`, *optional*, defaults to 49407):
            The id of the end-of-sequence token in the vocabulary.
        max_position_embeddings (`int`, *optional*, defaults to 77):
            The maximum sequence length that this model might ever be used with. Typically set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the for initializing all weight matrices.
                   r#   r$   Fr%   N  M   r&   
vocab_sizer'   r(   r)   r*   r.   r/   r0   r1   r2   pad_token_idbos_token_ideos_token_idmax_position_embeddingsr3   c                    sZ   t  jd||||||
||||d
| || _|| _|	| _|| _|| _| `| `| `	| `
d S )N)
rP   r'   r(   r)   r*   r2   rT   rQ   rR   rS   r6   )r7   r8   r3   r/   r1   r0   r.   rR   rQ   projection_sizer9   )r:   rP   r'   r(   r)   r*   r.   r/   r0   r1   r2   rQ   rR   rS   rT   r3   r;   r<   r6   r>   r8      s.   zAimv2TextConfig.__init__)rI   rJ   rK   rL   rM   r#   r$   FFr%   NNrN   rO   r&   )r?   r@   rA   rB   rC   rD   rE   rF   r   r8   rG   r6   r6   r<   r>   rH      s`    /	
rH   c                       s$   e Zd ZdZ	d fdd	Z  ZS )Aimv2Configa@  
    [`Aimv2Config`] is the configuration class to store the configuration of a [`Aimv2Model`]. It is used to
    instantiate a AIMv2 model according to the specified arguments, defining the text model and vision model configs.
    Instantiating a configuration with the defaults will yield a similar configuration to that of the AIMv2
    [apple/aimv2-large-patch14-224-lit](https://huggingface.co/apple/aimv2-large-patch14-224-lit) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        text_config (`dict`, *optional*):
            Dictionary of configuration options used to initialize [`Aimv2TextConfig`].
        vision_config (`dict`, *optional*):
            Dictionary of configuration options used to initialize [`Aimv2VisionConfig`].
        projection_dim (`int`, *optional*, defaults to 512):
            Dimensionality of text and vision projection layers.
        logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
            The initial value of the *logit_scale* parameter.
        kwargs (*optional*):
            Dictionary of keyword arguments.

    Example:

    ```python
    >>> from transformers import Aimv2Config, Aimv2Model

    >>> # Initializing a Aimv2Config with apple/aimv2-large-patch14-224-lit style configuration
    >>> configuration = Aimv2Config()

    >>> # Initializing a Aimv2Model (with random weights) from the apple/aimv2-large-patch14-224-lit style configuration
    >>> model = Aimv2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config

    >>> # We can also initialize a Aimv2Config from a Aimv2TextConfig and a Aimv2VisionConfig
    >>> from transformers import Aimv2TextConfig, Aimv2VisionConfig

    >>> # Initializing a AIMv2Text and AIMv2Vision configuration
    >>> config_text = Aimv2TextConfig()
    >>> config_vision = Aimv2VisionConfig()

    >>> config = Aimv2Config(text_config=config_text, vision_config=config_vision)
    ```N   /L
F@c                    s0   t  j||fi | || _|| _d| _| `d S )Ng      Y@)r7   r8   projection_dimlogit_scale_init_valuemax_logit_scaleinitializer_factor)r:   text_configvision_configrY   rZ   r;   r<   r6   r>   r8     s
   zAimv2Config.__init__)NNrW   rX   )r?   r@   rA   rB   r8   rG   r6   r6   r<   r>   rV      s    .rV   c                   @      e Zd ZdS )Aimv2OutputNr?   r@   rA   r6   r6   r6   r>   r`   #      r`   c                   @   r_   )Aimv2RMSNormNra   r6   r6   r6   r>   rc   '  rb   rc   c                   @   r_   )Aimv2MLPNra   r6   r6   r6   r>   rd   +  rb   rd   c                       sX   e Zd Zdef fddZedddejfdejfdd	Z	d
ejdejfddZ
  ZS )Aimv2VisionEmbeddingsconfigc                    s   t    || _|j| _tj|j|j|j|jd| _t	|j|j
| _|j|j d }| jjs6t||j| _| jdt|ddd d S )N)kernel_sizestrider   position_ids)   F)
persistent)r7   r8   rf   r-   r   Conv2dr+   r'   patch_embedrc   r.   rms_normr,   r5   	Embeddingposition_embeddingregister_buffertorcharangeexpand)r:   rf   num_patchesr<   r6   r>   r8   0  s   
 zAimv2VisionEmbeddings.__init__   g     @cpureturnc                 C   s   t jt|||d}t jt| ||d}t j||dd\}}|d }t j|||d| }	d||	  }	| d |	d d d f  }
| d |	d d d f  }t j|
 |
 | | gddd d d d d f S )	Ndtypedevicexy)indexing   g      ?).Nrj   dim)rs   rt   rC   meshgridflattenconcatsincos)heightwidth	embed_dimtemperaturer|   r{   grid_wgrid_hpos_dimomegaout_hout_wr6   r6   r>   "build_2d_sincos_position_embedding>  s   8z8Aimv2VisionEmbeddings.build_2d_sincos_position_embeddingpixel_valuesc                 C   s|   |  \}}}}| |ddd}| |}| jjr2| j|| j || j | jj	|j
|jd}n| | j}|| }|S )Nr   rj   )r   r|   r{   )sizern   r   	transposero   rf   r5   r   r-   r'   r|   r{   rq   ri   )r:   r   _r   r   hidden_states	pos_embedr6   r6   r>   forwardO  s   
zAimv2VisionEmbeddings.forward)r?   r@   rA   r   r8   staticmethodrs   float32Tensorr   r   rG   r6   r6   r<   r>   re   /  s    
re   c                   @   r_   )Aimv2TextEmbeddingsNra   r6   r6   r6   r>   r   c  rb   r   c                       s   e Zd Z fddZ  ZS )Aimv2Attentionc                    sp   t  | tj| j| j|jd| _tj| j| j|jd| _tj| j| j|jd| _tj| j| j|jd| _	d S )Nbias)
r7   r8   r   Linearr   r0   k_projv_projq_projout_projr:   rf   r<   r6   r>   r8   h  s
   zAimv2Attention.__init__)r?   r@   rA   r8   rG   r6   r6   r<   r>   r   g  s    r   c                	       sN   e Zd Zdef fddZ	ddejdeej dee	 dejfd	d
Z
  ZS )Aimv2EncoderLayerrf   c                    sB   t    t|| _t|| _t|j|j| _	t|j|j| _
d S N)r7   r8   r   	attentionrd   ffnrc   r'   r.   	rms_norm1	rms_norm2r   r<   r6   r>   r8   q  s
   


zAimv2EncoderLayer.__init__Nr   attention_maskr;   ry   c                 K   sL   |  |}| jd||d|\}}|| }| |}| |}|| }|S )N)r   r   r6   )r   r   r   r   )r:   r   r   r;   norm_hidden_statesattn_outputr   
mlp_outputr6   r6   r>   r   x  s   


zAimv2EncoderLayer.forwardr   )r?   r@   rA   r   r8   rs   r   r   r
   r   r   rG   r6   r6   r<   r>   r   p  s    
r   c                   @   r_   )Aimv2EncoderNra   r6   r6   r6   r>   r     rb   r   c                       s8   e Zd Zdef fddZdejdejfddZ  ZS )Aimv2AttentionPoolingHeadrf   c                    s|   t    |j| _|j| _tj| j| j|jd| _tj| j| j|jd| _	t
tdd| j| _tj| j| jdd| _d S )Nr   rj   T)r7   r8   r'   r*   	num_headsr   r   r0   r   r   	Parameterrs   zeros	cls_tokenoutput_projr   r<   r6   r>   r8     s   
z"Aimv2AttentionPoolingHead.__init__r   ry   c                 C   s   |j \}}}| j|dd}| |||| j|| j }| |||| j|| j }||d| j|| j }|dddd}|dddd}|dddd}t	|||}	|	
dd|d|}	|	jdd}	| |	}
|
S )Nrk   rj   r   r   r   r   )shaper   ru   r   reshaper   r   permuteFscaled_dot_product_attentionr   meanr   )r:   r   
batch_sizeseq_len
hidden_dimr   keyvaluequeryr   outputr6   r6   r>   r     s   
z!Aimv2AttentionPoolingHead.forward)	r?   r@   rA   r   r8   rs   r   r   rG   r6   r6   r<   r>   r     s    r   c                       sF   e Zd ZU dZeed< dZdZg dZdZ	dZ
dZ fddZ  ZS )Aimv2PreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models. The model is only intended for inference and doesn't support finetuning.
    rf   aimv2T)r   r   re   r   c                    sh   t  | t|dr t|jtjr|jjt	
d d S d S t|tr2|jjjd| jjd d S d S )Nlogit_scaleg$I$I,@r$   )r   std)r7   _init_weightshasattr
isinstancer   r   r   datafill_mathlogr   r   normal_rf   r3   )r:   moduler<   r6   r>   r     s   

z"Aimv2PreTrainedModel._init_weights)r?   r@   rA   rB   rV   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attnr   rG   r6   r6   r<   r>   r     s   
 r   zL
    The Vision model from AIMv2 without any head or projection on top.
    )custom_introc                
       s   e Zd ZU eed< dZeedZdef fddZ	de
jfddZed	d
dedde	dd	eej dee defddZ  ZS )Aimv2VisionModelrf   r   r   
attentionsc                    sZ   t  | || _t|| _t|| _t|j|j	| _
|j| _| jr't|| _|   d S r   )r7   r8   rf   re   
embeddingsr   encoderrc   r'   r.   ro   r4   r   head	post_initr   r<   r6   r>   r8     s   


zAimv2VisionModel.__init__ry   c                 C      | j jS r   )r   rn   r:   r6   r6   r>   get_input_embeddings     z%Aimv2VisionModel.get_input_embeddingsr   zv4.58.0)versionFtie_last_hidden_statesNr;   c                 K   sN   |  |}| jdd|i|}|j}| |}| jr| |nd}t||dS )a  
        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, Siglip2VisionModel

        >>> model = Aimv2VisionModel.from_pretrained("apple/aimv2-large-patch14-native")
        >>> processor = AutoProcessor.from_pretrained("apple/aimv2-large-patch14-native")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled features
        ```inputs_embedsNlast_hidden_statepooler_outputr6   )r   r   r   ro   r4   r   r   )r:   r   r   r;   r   encoder_outputsr   r   r6   r6   r>   r     s   

zAimv2VisionModel.forwardr   )r?   r@   rA   r   r   main_input_namer   r   _can_record_outputsr8   r   Moduler   r   r   r   r   rs   r   r
   r   r   r   rG   r6   r6   r<   r>   r     s&   
 
r   zJ
    The text model from AIMv2 without any head or projection on top.
    c                	       sz   e Zd ZdZeedZdef fddZde	j
fddZd	d
 Zedde	ddeej dee defddZ  ZS )Aimv2TextModel	input_idsr   rf   c                    sJ   t  | || _t|| _t|| _t|j|j	| _
|j| _|   d S r   )r7   r8   rf   r   r   r   r   rc   r'   r.   ro   rS   r   r   r<   r6   r>   r8   &  s   

zAimv2TextModel.__init__ry   c                 C   r   r   r   token_embeddingr   r6   r6   r>   r   1  r   z#Aimv2TextModel.get_input_embeddingsc                 C   s   || j _d S r   r   )r:   r   r6   r6   r>   set_input_embeddings4  s   z#Aimv2TextModel.set_input_embeddingsFr   Nr   r;   c                 K   s   |  |}|j\}}}tj|tj|jd}|d|d}	|d ur-t| j	||	||d d}| j
d	||d|}
|
j}| |}|tj|jd |jd|jtj|jd| jk jddf }t||dS )
Nrz   r   rk   )rf   input_embedsri   r   cache_positionpast_key_values)r   r   )r|   r   r   r6   )r   r   rs   rt   longr|   	unsqueezeru   r   rf   r   r   ro   torC   rS   argmaxr   )r:   r   r   r;   r   r   r   r   r   ri   r   r   pooled_outputr6   r6   r>   r   7  s<   
	
"zAimv2TextModel.forwardr   )r?   r@   rA   r   r   r   r   rH   r8   r   r   r   r   r   r   r   rs   r   r
   r   r   r   rG   r6   r6   r<   r>   r     s$    r   c                   @   sb   e Zd ZdZdefddZee			ddee	j
 dee	j dee	j d	ee d
ef
ddZdS )
Aimv2ModelTrf   c                 C   s   t | | |j| _|jj| _|jj| _t	|j| _
t	|j| _tj| j| jdd| _tj| j| jdd| _tt| jj| _t|j| _|   d S )NFr   )r	   r8   rY   r^   r'   vision_embed_dimr]   text_embed_dimr   _from_configvision_modelr   
text_modelr   r   visual_projectiontext_projectionr   rs   tensorrf   rZ   r   r   r   r[   max_log_logit_scaler   r   r6   r6   r>   r8   g  s   

zAimv2Model.__init__Nr   r   r   r;   ry   c                 K   s   | j dd|i|}| jd||d|}|j}| |}|j}| |}|t| }|t| }| jd| j	 
|j}	|	| |  }
|
 }t||
||||dS )a  
        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, Aimv2Model

        >>> model = Aimv2Model.from_pretrained("apple/aimv2-large-patch14-224-lit")
        >>> processor = AutoProcessor.from_pretrained("apple/aimv2-large-patch14-224-lit")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(
        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
        ... )

        >>> outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```r   )r   r   r$   )logits_per_imagelogits_per_texttext_embedsimage_embedstext_model_outputvision_model_outputNr6   )r   r  r   r  r  r   r   clampr  expr   r|   tr`   )r:   r   r   r   r;   vision_outputstext_outputsr	  r  r   r  r  r6   r6   r>   r   y  s8   

zAimv2Model.forward)NNN)r?   r@   rA   r   rV   r8   r   r   r   rs   
LongTensorFloatTensorr   r
   r   r`   r   r6   r6   r6   r>   r   c  s&    r   )rV   r   rH   r   r   r   r   )=rB   r   typingr   rs   torch.nn.functionalr   
functionalr   masking_utilsr   modeling_layersr   modeling_outputsr   r   modeling_utilsr	   processing_utilsr
   utilsr   r   r   utils.deprecationr   utils.genericr   clip.modeling_clipr   r   r   llama.modeling_llamar   r   siglip.configuration_siglipr   r   r   siglip.modeling_siglipr   r   r   r   rH   rV   r`   rc   rd   r   re   r   r   r   r   r   r   r   r   r   __all__r6   r6   r6   r>   <module>   sV   d[94	"HEW