o
    wi                  	   @   s,  d Z ddlmZmZ ddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ eeZd?dededee defddZe de dfde de de de fddZ!G dd dej"Z#G dd dej"Z$G dd  d ej"Z%G d!d" d"ej"Z&G d#d$ d$ej"Z'G d%d& d&ej"Z(G d'd( d(ej"Z)G d)d* d*eZ*G d+d, d,ej"Z+eG d-d. d.eZ,eG d/d0 d0e,Z-ed1d2G d3d4 d4e,Z.G d5d6 d6ej"Z/G d7d8 d8ej"Z0G d9d: d:ej"Z1ed;d2G d<d= d=e,Z2g d>Z3dS )@zPyTorch MobileViTV2 model.    )OptionalUnionN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)GradientCheckpointingLayer)BaseModelOutputWithNoAttention(BaseModelOutputWithPoolingAndNoAttention$ImageClassifierOutputWithNoAttentionSemanticSegmenterOutput)PreTrainedModel)auto_docstringlogging   )MobileViTV2Config   valuedivisor	min_valuereturnc                 C   sF   |du r|}t |t| |d  | | }|d|  k r||7 }t|S )a  
    Ensure that all layers have a channel count that is divisible by `divisor`. This function is taken from the
    original TensorFlow repo. It can be seen here:
    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py
    N   g?)maxint)r   r   r   	new_value r   q/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/mobilevitv2/modeling_mobilevitv2.pymake_divisible+   s   r   z-infinfmin_valmax_valc                 C   s   t |t|| S N)r   minr   r!   r"   r   r   r   clip:   s   r&   c                       sv   e Zd Z						ddededededed	ed
edededeeef ddf fddZde	j
de	j
fddZ  ZS )MobileViTV2ConvLayerr   FTconfigin_channelsout_channelskernel_sizestridegroupsbiasdilationuse_normalizationuse_activationr   Nc                    s   t    t|d d | }|| dkr td| d| d|| dkr1td| d| dtj||||||||dd		| _|	rNtj|d
dddd| _nd | _|
rst	|
t
r_t|
 | _d S t	|jt
rmt|j | _d S |j| _d S d | _d S )Nr   r   r   zInput channels (z) are not divisible by z groups.zOutput channels (zeros)	r)   r*   r+   r,   paddingr/   r-   r.   padding_modegh㈵>g?T)num_featuresepsmomentumaffinetrack_running_stats)super__init__r   
ValueErrorr   Conv2dconvolutionBatchNorm2dnormalization
isinstancestrr	   
activation
hidden_act)selfr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r3   	__class__r   r   r;   @   sB   



zMobileViTV2ConvLayer.__init__featuresc                 C   s6   |  |}| jd ur| |}| jd ur| |}|S r#   )r>   r@   rC   )rE   rH   r   r   r   forwardv   s   




zMobileViTV2ConvLayer.forward)r   r   Fr   TT)__name__
__module____qualname__r   r   boolr   rB   r;   torchTensorrI   __classcell__r   r   rF   r   r'   ?   s>    	

6r'   c                       sT   e Zd ZdZ	ddedededededd	f fd
dZdejdejfddZ	  Z
S )MobileViTV2InvertedResidualzY
    Inverted residual block (MobileNetv2): https://huggingface.co/papers/1801.04381
    r   r(   r)   r*   r,   r/   r   Nc              	      s   t    ttt||j d}|dvrtd| d|dko$||k| _t|||dd| _	t|||d|||d| _
t|||dd	d
| _d S )Nr   )r   r   zInvalid stride .r   )r)   r*   r+   r   )r)   r*   r+   r,   r-   r/   Fr)   r*   r+   r1   )r:   r;   r   r   roundexpand_ratior<   use_residualr'   
expand_1x1conv_3x3
reduce_1x1)rE   r(   r)   r*   r,   r/   expanded_channelsrF   r   r   r;      s0   

z$MobileViTV2InvertedResidual.__init__rH   c                 C   s4   |}|  |}| |}| |}| jr|| S |S r#   )rW   rX   rY   rV   )rE   rH   residualr   r   r   rI      s
   


z#MobileViTV2InvertedResidual.forward)r   rJ   rK   rL   __doc__r   r   r;   rN   rO   rI   rP   r   r   rF   r   rQ      s"    !rQ   c                       sP   e Zd Z	ddedededededdf fd	d
ZdejdejfddZ  Z	S )MobileViTV2MobileNetLayerr   r(   r)   r*   r,   
num_stagesr   Nc                    sR   t    t | _t|D ]}t||||dkr|ndd}| j| |}qd S )Nr   r   )r)   r*   r,   )r:   r;   r   
ModuleListlayerrangerQ   append)rE   r(   r)   r*   r,   r_   ira   rF   r   r   r;      s   

z"MobileViTV2MobileNetLayer.__init__rH   c                 C      | j D ]}||}q|S r#   ra   )rE   rH   layer_moduler   r   r   rI         

z!MobileViTV2MobileNetLayer.forward)r   r   
rJ   rK   rL   r   r   r;   rN   rO   rI   rP   r   r   rF   r   r^      s     r^   c                       sD   e Zd ZdZdededdf fddZdejdejfd	d
Z	  Z
S )MobileViTV2LinearSelfAttentionay  
    This layer applies a self-attention with linear complexity, as described in MobileViTV2 paper:
    https://huggingface.co/papers/2206.02680

    Args:
        config (`MobileVitv2Config`):
             Model configuration object
        embed_dim (`int`):
            `input_channels` from an expected input of size :math:`(batch_size, input_channels, height, width)`
    r(   	embed_dimr   Nc              	      s\   t    t||dd|  ddddd| _tj|jd| _t|||ddddd| _|| _d S )Nr   r   TF)r(   r)   r*   r.   r+   r0   r1   p)	r:   r;   r'   qkv_projr   Dropoutattn_dropoutout_projrk   )rE   r(   rk   rF   r   r   r;      s*   



	z'MobileViTV2LinearSelfAttention.__init__hidden_statesc           	      C   s   |  |}tj|d| j| jgdd\}}}tjjj|dd}| |}|| }tj|ddd}tjj	||
| }| |}|S )Nr   )split_size_or_sectionsdimrt   Trt   keepdim)rn   rN   splitrk   r   
functionalsoftmaxrp   sumrelu	expand_asrq   )	rE   rr   qkvquerykeyr   context_scorescontext_vectoroutr   r   r   rI      s   
 

z&MobileViTV2LinearSelfAttention.forwardr\   r   r   rF   r   rj      s    rj   c                       L   e Zd Z	ddededededdf
 fdd	Zd
ejdejfddZ	  Z
S )MobileViTV2FFN        r(   rk   ffn_latent_dimffn_dropoutr   Nc              
      sZ   t    t|||dddddd| _t|| _t|||dddddd| _t|| _d S )Nr   TF)r(   r)   r*   r+   r,   r.   r0   r1   )	r:   r;   r'   conv1r   ro   dropout1conv2dropout2)rE   r(   rk   r   r   rF   r   r   r;     s.   


zMobileViTV2FFN.__init__rr   c                 C   s,   |  |}| |}| |}| |}|S r#   )r   r   r   r   )rE   rr   r   r   r   rI   (  s
   



zMobileViTV2FFN.forwardr   rJ   rK   rL   r   r   floatr;   rN   rO   rI   rP   r   r   rF   r   r     s     r   c                       r   )MobileViTV2TransformerLayerr   r(   rk   r   dropoutr   Nc                    sb   t    tjd||jd| _t||| _tj|d| _	tjd||jd| _
t||||j| _d S )Nr   
num_groupsnum_channelsr6   rl   )r:   r;   r   	GroupNormlayer_norm_epslayernorm_beforerj   	attentionro   r   layernorm_afterr   r   ffn)rE   r(   rk   r   r   rF   r   r   r;   1  s   
z$MobileViTV2TransformerLayer.__init__rr   c                 C   s<   |  |}| |}|| }| |}| |}|| }|S r#   )r   r   r   r   )rE   rr   layernorm_1_outattention_outputlayer_outputr   r   r   rI   ?  s   



z#MobileViTV2TransformerLayer.forwardr   r   r   r   rF   r   r   0  s    r   c                       D   e Zd Zdedededdf fddZdejdejfd	d
Z  Z	S )MobileViTV2Transformerr(   n_layersd_modelr   Nc                    sf   t    |j}|| g| }dd |D }t | _t|D ]}t|||| d}| j| qd S )Nc                 S   s   g | ]
}t |d  d  qS )   )r   ).0dr   r   r   
<listcomp>T  s    z3MobileViTV2Transformer.__init__.<locals>.<listcomp>)rk   r   )	r:   r;   ffn_multiplierr   r`   ra   rb   r   rc   )rE   r(   r   r   r   ffn_dims	block_idxtransformer_layerrF   r   r   r;   L  s   


zMobileViTV2Transformer.__init__rr   c                 C   re   r#   rf   )rE   rr   rg   r   r   r   rI   ]  rh   zMobileViTV2Transformer.forwardri   r   r   rF   r   r   K  s    r   c                       s   e Zd ZdZ			ddededededed	ed
eddf fddZdejde	eje	eef f fddZ
dejde	eef dejfddZdejdejfddZ  ZS )MobileViTV2LayerzE
    MobileViTV2 layer: https://huggingface.co/papers/2206.02680
    r   r   r(   r)   r*   attn_unit_dimn_attn_blocksr/   r,   r   Nc           	         s   t    |j| _|j| _|}|dkr.t||||dkr|nd|dkr&|d ndd| _|}nd | _t||||j|d| _	t|||dddd| _
t|||d| _tjd||jd| _t|||dd	dd| _d S )
Nr   r   )r)   r*   r,   r/   )r)   r*   r+   r-   F)r)   r*   r+   r0   r1   )r   r   r   T)r:   r;   
patch_sizepatch_widthpatch_heightrQ   downsampling_layerr'   conv_kernel_sizeconv_kxkconv_1x1r   transformerr   r   r   	layernormconv_projection)	rE   r(   r)   r*   r   r   r/   r,   cnn_out_dimrF   r   r   r;   h  sN   


zMobileViTV2Layer.__init__feature_mapc                 C   sT   |j \}}}}tjj|| j| jf| j| jfd}|||| j| j d}|||ffS )N)r+   r,   ru   )shaper   rz   unfoldr   r   reshape)rE   r   
batch_sizer)   
img_height	img_widthpatchesr   r   r   	unfolding  s   

zMobileViTV2Layer.unfoldingr   output_sizec                 C   sH   |j \}}}}|||| |}tjj||| j| jf| j| jfd}|S )N)r   r+   r,   )r   r   r   rz   foldr   r   )rE   r   r   r   in_dimr   	n_patchesr   r   r   r   folding  s   

zMobileViTV2Layer.foldingrH   c                 C   s`   | j r|  |}| |}| |}| |\}}| |}| |}| ||}| |}|S r#   )r   r   r   r   r   r   r   r   )rE   rH   r   r   r   r   r   rI     s   





zMobileViTV2Layer.forward)r   r   r   )rJ   rK   rL   r]   r   r   r;   rN   rO   tupler   r   rI   rP   r   r   rF   r   r   c  s2    
	&="r   c                       sP   e Zd Zdeddf fddZ		ddejd	ed
edee	e
f fddZ  ZS )MobileViTV2Encoderr(   r   Nc                    s  t    || _t | _d| _d }}|jdkrd}d}n|jdkr%d}d}tt	d|j
 dddddd	}td|j
 dd
}td|j
 dd
}td|j
 dd
}td|j
 dd
}	td|j
 dd
}
t|||ddd}| j| t|||ddd}| j| t|||t|jd |j
 dd
|jd d}| j| |r|d9 }t|||	t|jd |j
 dd
|jd |d}| j| |r|d9 }t||	|
t|jd |j
 dd
|jd |d}| j| d S )NFr   Tr   r       @   r%   r   r   r         i     )r)   r*   r,   r_   r   r   )r)   r*   r   r   )r)   r*   r   r   r/   )r:   r;   r(   r   r`   ra   gradient_checkpointingoutput_strider   r&   width_multiplierr^   rc   r   base_attn_unit_dimsr   )rE   r(   dilate_layer_4dilate_layer_5r/   layer_0_dimlayer_1_dimlayer_2_dimlayer_3_dimlayer_4_dimlayer_5_dimlayer_1layer_2layer_3layer_4layer_5rF   r   r   r;     s   



zMobileViTV2Encoder.__init__FTrr   output_hidden_statesreturn_dictc                 C   s\   |rdnd }t | jD ]\}}||}|r||f }q|s(tdd ||fD S t||dS )Nr   c                 s   s    | ]	}|d ur|V  qd S r#   r   )r   vr   r   r   	<genexpr>6  s    z-MobileViTV2Encoder.forward.<locals>.<genexpr>)last_hidden_staterr   )	enumeratera   r   r   )rE   rr   r   r   all_hidden_statesrd   rg   r   r   r   rI   '  s   
zMobileViTV2Encoder.forward)FT)rJ   rK   rL   r   r;   rN   rO   rM   r   r   r   rI   rP   r   r   rF   r   r     s    T
r   c                   @   sD   e Zd ZeZdZdZdZdgZde	e
je
je
jf ddfdd	ZdS )
MobileViTV2PreTrainedModelmobilevitv2pixel_valuesTr   moduler   Nc                 C   st   t |tjtjfr#|jjjd| jjd |j	dur!|j	j
  dS dS t |tjr8|j	j
  |jjd dS dS )zInitialize the weightsr   )meanstdNg      ?)rA   r   Linearr=   weightdatanormal_r(   initializer_ranger.   zero_	LayerNormfill_)rE   r   r   r   r   _init_weightsD  s   
z(MobileViTV2PreTrainedModel._init_weights)rJ   rK   rL   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modulesr   r   r   r=   r   r   r   r   r   r   r   ;  s    &r   c                       sl   e Zd Zddedef fddZdd Ze			dd	ee	j
 d
ee dee deeef fddZ  ZS )MobileViTV2ModelTr(   expand_outputc              	      sf   t  | || _|| _ttd|j dddddd}t||j|ddd	d	d
| _	t
|| _|   dS )a  
        expand_output (`bool`, *optional*, defaults to `True`):
            Whether to expand the output of the model. If `True`, the model will output pooled features in addition to
            hidden states. If `False`, only the hidden states will be returned.
        r   r   r   r%   r   r   r   r   Tr)   r*   r+   r,   r0   r1   N)r:   r;   r(   r   r   r&   r   r'   r   	conv_stemr   encoder	post_init)rE   r(   r   r   rF   r   r   r;   S  s"   
	zMobileViTV2Model.__init__c                 C   sF   |  D ]\}}| jj| }t|tr |jjD ]}|j| qqdS )zPrunes heads of the model.
        heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel
        N)itemsr   ra   rA   r   r   r   prune_heads)rE   heads_to_prunelayer_indexheadsmobilevitv2_layerr   r   r   r   _prune_headso  s   
zMobileViTV2Model._prune_headsNr   r   r   r   c           	      C   s   |d ur|n| j j}|d ur|n| j j}|d u rtd| |}| j|||d}| jr;|d }tj|ddgdd}n|d }d }|sV|d urK||fn|f}||dd   S t	|||j
d	S )
Nz You have to specify pixel_valuesr   r   r   ru   Frw   r   )r   pooler_outputrr   )r(   r   use_return_dictr<   r   r   r   rN   r   r   rr   )	rE   r   r   r   embedding_outputencoder_outputsr   pooled_outputoutputr   r   r   rI   y  s0   
zMobileViTV2Model.forward)T)NNN)rJ   rK   rL   r   rM   r;   r  r   r   rN   rO   r   r   r   rI   rP   r   r   rF   r   r   Q  s     

r   z
    MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    )custom_introc                       sn   e Zd Zdeddf fddZe				ddeej dee	 deej d	ee	 de
eef f
d
dZ  ZS )!MobileViTV2ForImageClassificationr(   r   Nc                    s`   t  | |j| _t|| _td|j dd}|jdkr%tj||jdnt	 | _
|   d S )Nr   r   r   r   )in_featuresout_features)r:   r;   
num_labelsr   r   r   r   r   r   Identity
classifierr  )rE   r(   r*   rF   r   r   r;     s   

z*MobileViTV2ForImageClassification.__init__r   r   labelsr   c                 C   sb  |dur|n| j j}| j|||d}|r|jn|d }| |}d}|dur| j jdu rP| jdkr6d| j _n| jdkrL|jtj	ksG|jtj
krLd| j _nd| j _| j jdkrnt }	| jdkrh|	| | }n+|	||}n%| j jdkrt }	|	|d| j|d}n| j jdkrt }	|	||}|s|f|dd  }
|dur|f|
 S |
S t|||jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr	  r   
regressionsingle_label_classificationmulti_label_classificationru   r   )losslogitsrr   )r(   r  r   r  r  problem_typer  dtyperN   longr   r   squeezer   viewr   r   rr   )rE   r   r   r  r   outputsr  r  r  loss_fctr  r   r   r   rI     s>   


"


z)MobileViTV2ForImageClassification.forwardNNNN)rJ   rK   rL   r   r;   r   r   rN   rO   rM   r   r   r   rI   rP   r   r   rF   r   r    s$    
r  c                       r   )MobileViTV2ASPPPoolingr(   r)   r*   r   Nc              	      s4   t    tjdd| _t|||ddddd| _d S )Nr   )r   Tr}   r   )r:   r;   r   AdaptiveAvgPool2dglobal_poolr'   r   )rE   r(   r)   r*   rF   r   r   r;     s   
zMobileViTV2ASPPPooling.__init__rH   c                 C   s:   |j dd  }| |}| |}tjj||ddd}|S )Nr
  bilinearFsizemodealign_corners)r   r(  r   r   rz   interpolate)rE   rH   spatial_sizer   r   r   rI     s
   

zMobileViTV2ASPPPooling.forwardri   r   r   rF   r   r&    s    r&  c                       @   e Zd ZdZdeddf fddZdejdejfdd	Z  Z	S )
MobileViTV2ASPPz
    ASPP module defined in DeepLab papers: https://huggingface.co/papers/1606.00915, https://huggingface.co/papers/1706.05587
    r(   r   Nc                    s   t    td j dd}| jt jdkrtdt	 | _
t ddd}| j
| | j
 fd	d
 jD  t }| j
| t d ddd| _tj jd| _d S )Nr   r   r   r   z"Expected 3 values for atrous_ratesr   r}   rS   c              
      s    g | ]}t  d |ddqS )r   r}   )r)   r*   r+   r/   r1   )r'   )r   rater(   r)   r*   r   r   r   (  s    	z,MobileViTV2ASPP.__init__.<locals>.<listcomp>   rl   )r:   r;   r   r   aspp_out_channelslenatrous_ratesr<   r   r`   convsr'   rc   extendr&  projectro   aspp_dropout_probr   )rE   r(   encoder_out_channelsin_projection
pool_layerrF   r3  r   r;     s4   

	zMobileViTV2ASPP.__init__rH   c                 C   sD   g }| j D ]	}||| qtj|dd}| |}| |}|S )Nr   rv   )r8  rc   rN   catr:  r   )rE   rH   pyramidconvpooled_featuresr   r   r   rI   >  s   


zMobileViTV2ASPP.forward
rJ   rK   rL   r]   r   r;   rN   rO   rI   rP   r   r   rF   r   r1    s    ,r1  c                       r0  )
MobileViTV2DeepLabV3zJ
    DeepLabv3 architecture: https://huggingface.co/papers/1706.05587
    r(   r   Nc              	      sB   t    t|| _t|j| _t||j	|j
ddddd| _d S )Nr   FT)r)   r*   r+   r0   r1   r.   )r:   r;   r1  asppr   	Dropout2dclassifier_dropout_probr   r'   r5  r  r  rE   r(   rF   r   r   r;   O  s   

zMobileViTV2DeepLabV3.__init__rr   c                 C   s&   |  |d }| |}| |}|S )Nru   )rE  r   r  )rE   rr   rH   r   r   r   rI   _  s   

zMobileViTV2DeepLabV3.forwardrC  r   r   rF   r   rD  J  s    rD  zZ
    MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC.
    c                       sn   e Zd Zdeddf fddZe				ddeej deej dee	 d	ee	 de
eef f
d
dZ  ZS )"MobileViTV2ForSemanticSegmentationr(   r   Nc                    s8   t  | |j| _t|dd| _t|| _|   d S )NF)r   )r:   r;   r  r   r   rD  segmentation_headr  rH  rF   r   r   r;   l  s
   
z+MobileViTV2ForSemanticSegmentation.__init__r   r  r   r   c                 C   s  |dur|n| j j}|dur|n| j j}|dur"| j jdkr"td| j|d|d}|r/|jn|d }| |}d}|durYtj	j
||jdd ddd	}	t| j jd
}
|
|	|}|s{|rg|f|dd  }n	|f|dd  }|dury|f| S |S t|||r|jddS dddS )a  
        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> import requests
        >>> import torch
        >>> from PIL import Image
        >>> from transformers import AutoImageProcessor, MobileViTV2ForSemanticSegmentation

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256")
        >>> model = MobileViTV2ForSemanticSegmentation.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256")

        >>> inputs = image_processor(images=image, return_tensors="pt")

        >>> with torch.no_grad():
        ...     outputs = model(**inputs)

        >>> # logits are of shape (batch_size, num_labels, height, width)
        >>> logits = outputs.logits
        ```Nr   z/The number of labels should be greater than oneTr	  r
  r)  Fr*  )ignore_indexr   )r  r  rr   
attentions)r(   r   r  r  r<   r   rr   rJ  r   rz   r.  r   r   semantic_loss_ignore_indexr   )rE   r   r  r   r   r#  encoder_hidden_statesr  r  upsampled_logitsr$  r  r   r   r   rI   v  sB   $

z*MobileViTV2ForSemanticSegmentation.forwardr%  )rJ   rK   rL   r   r;   r   r   rN   rO   rM   r   r   r   rI   rP   r   r   rF   r   rI  f  s$    

rI  )r  rI  r   r   )r   N)4r]   typingr   r   rN   torch.utils.checkpointr   torch.nnr   r   r   activationsr	   modeling_layersr
   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   configuration_mobilevitv2r   
get_loggerrJ   loggerr   r   r   r&   Moduler'   rQ   r^   rj   r   r   r   r   r   r   r   r  r&  r1  rD  rI  __all__r   r   r   r   <module>   sP   
 (A1?)rfRK=X