o wij[@svddlmZmZmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZmZddlmZdd lmZmZd d lmZmZmZmZmZmZmZd d lmZd d l m!Z!m"Z"e#e$Z%GdddeZ&GdddeZ'Gddde!Z(GdddeZ)GdddeZ*GdddeZ+GdddeZ,GdddeZ-eGdddeZ.Gd d!d!eZ/gd"Z0dS)#)CallableOptionalUnionN)PretrainedConfig)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringlogging)CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayerCLIPVisionEmbeddingsCLIPVisionModelCLIPVisionTransformer)eager_attention_forward)VisionRotaryEmbeddingapply_rotary_pos_emb_visioncsDeZdZdZdZdZ        dfdd ZZS)MLCDVisionConfiga This is the configuration class to store the configuration of a [`MLCDVisionModel`]. It is used to instantiate a MLCD vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the MLCD [DeepGlint-AI/mlcd-vit-bigG-patch14-336](https://huggingface.co/DeepGlint-AI/mlcd-vit-bigG-patch14-336) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: hidden_size (`int`, *optional*, defaults to 1664): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 8192): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. projection_dim (`int`, *optional*, defaults to 1024): Dimensionality of text and vision projection layers. num_hidden_layers (`int`, *optional*, defaults to 48): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 16): Number of attention heads for each attention layer in the Transformer encoder. num_channels (`int`, *optional*, defaults to 3): The number of input channels. image_size (`int`, *optional*, defaults to 336): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 14): The size (resolution) of each patch. hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. initializer_factor (`float`, *optional*, defaults to 1.0): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). Example: ```python >>> from transformers import MLCDVisionConfig, MLCDVisionModel >>> # Initializing a MLCDVisionConfig with DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> configuration = MLCDVisionConfig() >>> # Initializing a MLCDVisionModel (with random weights) from the DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> model = MLCDVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```mlcd_vision_model vision_config 0rPgeluh㈵>{Gz??c sdtjdi|||_||_||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_dS)N)super__init__ hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_groups num_channels patch_size image_sizeinitializer_rangeinitializer_factorattention_dropoutlayer_norm_eps hidden_act)selfr,r-r.r/r0r1r3r2r8r7r6r4r5kwargs __class__r)b/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/mlcd/modular_mlcd.pyr+ds zMLCDVisionConfig.__init__) rrrr r!rr"r#r$r%r&r'r()__name__ __module__ __qualname____doc__ model_typebase_config_keyr+ __classcell__r)r)r;r=r*s$6rc@s eZdZdS)MLCDMLPN)r>r?r@r)r)r)r=rEsrEc@s$eZdZdededejfddZdS)MLCDRotaryEmbeddingnum_patches_heightnum_patches_widthreturnc Cstj||jjddd|}tj||jjdd|d}tj||gdd}t||}tj||jj|jj d}t ||j}||d} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicer!rdim)rJdtype) torcharangeinv_freqrJ unsqueezeexpandstackflattenmaxrNouter) r9rGrHhpos_idswpos_idspos_ids max_grid_sizeseqrotary_pos_emb_fullrotary_pos_embr)r)r=forwards  zMLCDRotaryEmbedding.forwardN)r>r?r@intrOTensorr_r)r)r)r=rFsrFcs8eZdZdeffdd ZdejdejfddZZ S)MLCDVisionEmbeddingsconfigcst||`dSN)r*r+position_embeddingr9rcr;r)r=r+s zMLCDVisionEmbeddings.__init__ pixel_valuesrIcCs^|jd}|jjj}||j|d}|ddd}|j|dd}t j ||gdd}|S)Nr)rNrr!rKrL) shapepatch_embeddingweightrNtorU transposeclass_embeddingrSrOcat)r9rg batch_size target_dtype patch_embeds class_embeds embeddingsr)r)r=r_s  zMLCDVisionEmbeddings.forward) r>r?r@rr+rO FloatTensorrar_rDr)r)r;r=rbsrbcspeZdZdZdeffdd Z d dejdeejejfde ejd e e d eeje ejff d d Z Z S) MLCDAttentionzMulti-headed attention with RoPE. Refer to papers: - Attention is all you need: https://huggingface.co/papers/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://huggingface.co/papers/2104.09864 rccst||j|_d|_dSNF)r*r+r0 is_causalrfr;r)r=r+s  zMLCDAttention.__init__N hidden_statesposition_embeddingsattention_maskr:rIcKsn|jdd\}}|||||j|jf}|||||j|jf}|||||j|jf} |dd} |dd} t ||| | \}}| dddd }| dddd }| dddd } t } |j jdkrzt|j j} | |||| |f|jsdn|j|j|jd|\} }| dddd } | ||d} || } | ddd } | |fS) NrKrr!rreagerr&)dropoutscalingrw)rhq_projreshape num_headshead_dimk_projv_projrRfloatrpermute contiguousrrc_attn_implementationr trainingr|scalerwviewout_proj)r9rxryrzr:ro seq_length query_states key_states value_statescossinattention_interface attn_output attn_weightsr)r)r=r_s>     zMLCDAttention.forwardrd)r>r?r@rArr+rOratuplerr rr_rDr)r)r;r=rus ruc sdeZdZdeffdd Z  d dejdeejejfdeejd ee d eej f d d Z Z S)MLCDEncoderLayerrccst|t||_dSrd)r*r+ru self_attnrfr;r)r=r+s zMLCDEncoderLayer.__init__NFrxryrzoutput_attentionsrIcCsd|}||}|j||||d\}}||}|}||}||}||}|f}|r0||f7}|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. rxryrzr) layer_norm1r layer_norm2mlp)r9rxryrzrresidualroutputsr)r)r=r_s"     zMLCDEncoderLayer.forwardrv) r>r?r@rr+rOrarrboolrtr_rDr)r)r;r=rsrcs~eZdZdZdeffdd Z    ddejdeej ej fde ej d e e d e e d e e d e ee ffd dZZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rccst|dS)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.N)r*r+rfr;r)r=r+5szMLCDEncoder.__init__N inputs_embedsryrzroutput_hidden_states return_dictrIc Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|r"dnd}|r(dnd}|} t|jD] \} } |r<|| f}| | |||d} | d} |rQ|| df}q1|rY|| f}|sgtdd| ||fDSt| ||dS) aj Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr)rrr!css|] }|dur|VqdSrdr)).0vr)r)r= wsz&MLCDEncoder.forward..)last_hidden_staterx attentions)rcruse_return_dictr enumeratelayersrr) r9rryrzrrrencoder_statesall_attentionsrxidx encoder_layer layer_outputsr)r)r=r_9s:"    zMLCDEncoder.forwardNNNN)r>r?r@rArr+rOrtrrarrrrr_rDr)r)r;r=r,s, rcsheZdZdeffdd Ze    d deejdee dee dee d e e e ff d d Z ZS) MLCDVisionTransformerrccsFt|t|j|jd|_tt d|j|jd|_ dS)Nrr!) r*r+rFr,r/vision_rotary_embeddingnn ParameterrOrandn class_pos_embrfr;r)r=r+s $zMLCDVisionTransformer.__init__NrgrrrrIcCs<|dur|n|jj}|dur|n|jj}|dur|n|jj}|dur&td|jd|jj}|jd|jj}|||}||j j }t j |j |gdd}t j ||fdd}| |f} ||} || } |j| | |||d} | d} | dddddf} || } |s| | f| ddSt| | | j| jdS) Nz You have to specify pixel_valuesrKrrL)rryrrrr!)r pooler_outputrxr)rcrrr ValueErrorrhr2rrkrrJrOrnrrrs pre_layrnormencoderpost_layernormr rxr)r9rgrrrrGrHr^embryrxencoder_outputsr pooled_outputr)r)r=r_sB     zMLCDVisionTransformer.forwardr)r>r?r@rr+r rrOrtrrrr r_rDr)r)r;r=rs$ rc@s(eZdZeZdZdZdZdZddZ dS)MLCDPreTrainedModelmlcdTcCs|jj}t|tr,|jj}tjj|jd|jd|dtjj|j j |jj |ddSt|t ru|jj}|jdd|jj d|}|jd|}tjj|jj |dtjj|jj |dtjj|jj |dtjj|jj |ddSt|tr|jj}|jjdd|jj d|}d|jjd|}tjj|jj |dtjj|jj |ddSt|tr|jj}|jj|jjdd|}tjj|jd|ddSt|tjr|jj|j jddSt|tjr|jdur|jjdSdSdS)zInitialize the weightsr&g)meanstd)rrr(N)rcr5 isinstancerbrinitnormal_rm embed_dimrirjr4rur.r~rrrrEr,fc1fc2rr/r LayerNormbiasdatazero_fill_Linear)r9modulefactor in_proj_std out_proj_stdfc_std pos_emb_stdr)r)r= _init_weightss:        z!MLCDPreTrainedModel._init_weightsN) r>r?r@r config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_2_supports_sdparr)r)r)r=rs rc@sReZdZe    d deejdeedeedeedee e ff ddZ dS) MLCDVisionModelNrgrrrrIcCsN|dur|n|jj}|dur|n|jj}|dur|n|jj}|j||||dS)a Example: ```python >>> import requests >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```N)rgrrr)rcrrr vision_model)r9rgrrrr)r)r=r_szMLCDVisionModel.forwardr) r>r?r@r rrOrtrrrr r_r)r)r)r=rs" r)rrr)1typingrrrrOtorch.nnrconfiguration_utilsrmodeling_flash_attention_utilsrmodeling_outputsrr modeling_utilsr r processing_utilsr utilsr rclip.modeling_cliprrrrrrrllama.modeling_llamarqwen2_vl.modeling_qwen2_vlrr get_loggerr>loggerrrErFrbrurrrrr__all__r)r)r)r=s2    $  \"<2S9' -