o
    i                  	   @   s  d Z ddlZddlZddlZddlmZ ddlmZm	Z	 ddl
Z
ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZmZ ddlmZmZm Z  ddl!m"Z" e#e$Z%eeddG dd deZ&dKde
j'de(de)de
j'fddZ*G dd dej+Z,G dd dej+Z-G d d! d!ej+Z.G d"d# d#ej+Z/G d$d% d%e/Z0G d&d' d'ej+Z1e/e0d(Z2G d)d* d*ej+Z3G d+d, d,ej+Z4G d-d. d.ej+Z5G d/d0 d0eZ6G d1d2 d2ej+Z7G d3d4 d4ej+Z8eG d5d6 d6eZ9eG d7d8 d8e9Z:G d9d: d:ej+Z;ed;dG d<d= d=e9Z<G d>d? d?ej+Z=G d@dA dAej+Z>G dBdC dCej+Z?G dDdE dEej+Z@G dFdG dGej+ZAeG dHdI dIe9ZBg dJZCdS )LzPyTorch Data2VecVision model.    N)	dataclass)OptionalUnion)nn)CrossEntropyLoss   )ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputSemanticSegmenterOutput)PreTrainedModel)#compile_compatible_method_lru_cache find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging	torch_int   )Data2VecVisionConfigz7
    Class for outputs of [`Data2VecVisionModel`].
    )custom_introc                   @   s   e Zd ZdZdS )$Data2VecVisionModelOutputWithPoolingaF  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
        will be returned.
    N)__name__
__module____qualname____doc__ r   r   i/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/data2vec/modeling_data2vec_vision.pyr   ,   s    r           Finput	drop_probtrainingreturnc                 C   sd   |dks|s| S d| }| j d fd| jd   }|tj|| j| jd }|  | || }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    r   r   r   r   )dtypedevice)shapendimtorchrandr%   r&   floor_div)r    r!   r"   	keep_probr'   random_tensoroutputr   r   r   	drop_path=   s   
r0   c                       sT   e Zd ZdZddee ddf fddZdejdejfdd	Z	de
fd
dZ  ZS )Data2VecVisionDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr!   r#   c                    s   t    || _d S N)super__init__r!   )selfr!   	__class__r   r   r4   U   s   

zData2VecVisionDropPath.__init__hidden_statesc                 C   s   t || j| jS r2   )r0   r!   r"   r5   r8   r   r   r   forwardY   s   zData2VecVisionDropPath.forwardc                 C   s   d| j  S )Nzp=)r!   r5   r   r   r   
extra_repr\   s   z!Data2VecVisionDropPath.extra_reprr2   )r   r   r   r   r   floatr4   r)   Tensorr:   strr<   __classcell__r   r   r6   r   r1   R   s
    r1   c                	       sv   e Zd ZdZdeddf fddZdejded	edejfd
dZ			ddejde
ej de
e dejfddZ  ZS )Data2VecVisionEmbeddingszc
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.

    configr#   Nc                    s   t    ttdd|j| _|jr!ttdd|j| _	nd | _	t
|| _|j| _t|jtjjr8|jn|j|jf| _| jj}|jrUttd|d |j| _nd | _t|j| _d S )Nr   )r3   r4   r   	Parameterr)   zeroshidden_size	cls_tokenuse_mask_token
mask_tokenData2VecVisionPatchEmbeddingspatch_embeddings
patch_size
isinstance
image_sizecollectionsabcIterablenum_patches use_absolute_position_embeddingsposition_embeddingsDropouthidden_dropout_probdropout)r5   rB   rQ   r6   r   r   r4   g   s    


z!Data2VecVisionEmbeddings.__init__
embeddingsheightwidthc                 C   s   |j d d }| jj d d }tj s||kr||kr| jS | jddddf }| jddddf }|j d }|| j }	|| j }
t|d }|d|||}|dddd}t	j
j||	|
fdd	d
}|dddddd|}tj||fddS )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Ng      ?r   r      bicubicFsizemodealign_cornersdim)r'   rS   r)   jit
is_tracingrK   r   reshapepermuter   
functionalinterpolateviewcat)r5   rW   rX   rY   rQ   num_positionsclass_pos_embedpatch_pos_embedrb   
new_height	new_widthsqrt_num_positionsr   r   r   interpolate_pos_encoding~   s(   



z1Data2VecVisionEmbeddings.interpolate_pos_encodingpixel_valuesbool_masked_posrq   c                 C   s   | j d ur|d urtd |j\}}}}| |\}\}}	| \}
}}|d urC| j|
|d}|d	|}|d|  ||  }| j
|
dd}tj||fdd}| j d urb|| ||| }| |}|||	ffS )Nz`interpolate_pos_encoding` argument has no effect for BEiTEmbeddings, embeddings are always interpolated to the input image size. The argument will be removed in transformers v4.51.0.rZ   r   ra   )rS   warningswarnr'   rJ   r^   rH   expand	unsqueezetype_asrF   r)   rj   rq   rV   )r5   rr   rs   rq   _rX   rY   rW   patch_heightpatch_width
batch_sizeseq_lenmask_tokensw
cls_tokensr   r   r   r:      s"   

z Data2VecVisionEmbeddings.forwardNN)r   r   r   r   r   r4   r)   r>   intrq   r   
BoolTensorboolr:   r@   r   r   r6   r   rA   a   s    +rA   c                       s6   e Zd ZdZ fddZdejdejfddZ  ZS )rI   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    s   t    |j|j}}|j|j}}t|tjj	r|n||f}t|tjj	r)|n||f}|d |d  |d |d   }|d |d  |d |d  f}|| _|| _|| _|| _
|| _tj||||d| _d S )Nr   r   kernel_sizestride)r3   r4   rM   rK   num_channelsrE   rL   rN   rO   rP   rQ   patch_shaper   Conv2d
projection)r5   rB   rM   rK   r   rE   rQ   r   r6   r   r   r4      s   
  z&Data2VecVisionPatchEmbeddings.__init__rr   r#   c           	      C   s^   |j \}}}}|| jkrtd| |}|j d |j d }}|ddd}|||ffS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r[   r   r   )r'   r   
ValueErrorr   flatten	transpose)	r5   rr   r|   r   rX   rY   rW   rz   r{   r   r   r   r:      s   

z%Data2VecVisionPatchEmbeddings.forward)	r   r   r   r   r4   r)   r>   r:   r@   r   r   r6   r   rI      s    rI   c                       s   e Zd Zddedee ddf fddZ					ddejd	eej d
e	deej de	deee
  deeej eejejf f fddZ  ZS )Data2VecVisionSelfAttentionNrB   window_sizer#   c                    s   t    || _|j|j dkr"t|ds"td|j d|j d|j| _t|j|j | _| j| j | _	t
|j| j	| _t
j|j| j	dd| _t
|j| j	| _t
|j| _t|| _| jrkt||d| _d S d S )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .F)biasr   )r3   r4   rB   rE   num_attention_headshasattrr   r   attention_head_sizeall_head_sizer   LinearquerykeyvaluerT   attention_probs_dropout_probrV   r   has_relative_position_bias"Data2VecVisionRelativePositionBiasrelative_position_biasr5   rB   r   r6   r   r   r4      s&   


z$Data2VecVisionSelfAttention.__init__Fr8   	head_maskoutput_attentionsr   rq   
resolutionc                 C   sl  |j \}}}	| ||d| j| jdd}
| ||d| j| jdd}| ||d| j| jdd}t	|
|dd}|t
| j }| jrk|\}}|| jj || jj f}|| j|||j d d }|d urs|| }tjj|dd}| |}|d ur|| }t	||}|dddd }| d d | jf }|j| }|r||f}|S |f}|S )	NrZ   r   r[   dim_sizera   r   r   )r'   r   ri   r   r   r   r   r   r)   matmulmathsqrtr   rB   rK   r   r   rg   softmaxrV   rf   
contiguousr^   r   )r5   r8   r   r   r   rq   r   r|   
seq_lengthry   query_layer	key_layervalue_layerattention_scoresrX   rY   r   attention_probscontext_layernew_context_layer_shapeoutputsr   r   r   r:     sF   	

z#Data2VecVisionSelfAttention.forwardr2   NFNFN)r   r   r   r   r   tupler4   r)   r>   r   r   r   r:   r@   r   r   r6   r   r      s,     
r   c                       sv   e Zd Z					ddejdeej dedeej dedeee  d	e	eej eejejf f f fd
dZ
  ZS )Data2VecVisionSdpaSelfAttentionNFr8   r   r   r   rq   r   r#   c              	      sv  |s|d urt d t j||||||dS |j\}}}	| ||d| j| j	dd}
| 
||d| j| j	dd}| ||d| j| j	dd}d }| jrp|\}}|| jj || jj f}| j|||jd d}|d ur|d u r{|}n||7 }dt| j }tjjj|
|||| jr| jjndd|d	}|d
ddd }| d d | jf }|j| }|d fS )Na  `Data2VecVisionSdpaSelfAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)r8   r   r   r   rq   r   rZ   r   r[   r   r   F)	attn_mask	dropout_p	is_causalscaler   r   r   )loggerwarning_oncer3   r:   r'   r   ri   r   r   r   r   r   r   rB   rK   r   r   r   r)   r   rg   scaled_dot_product_attentionr"   r   rf   r   r^   r   )r5   r8   r   r   r   rq   r   r|   r   ry   r   r   r   	attn_biasrX   rY   r   scalingr   r   r6   r   r   r:   J  sd   			
z'Data2VecVisionSdpaSelfAttention.forwardr   )r   r   r   r)   r>   r   r   r   r   r   r:   r@   r   r   r6   r   r   I  s*    
r   c                       sH   e Zd ZdZdeddf fddZddejdejdejfd	d
Z  Z	S )Data2VecVisionSelfOutputz
    The residual connection is defined in Data2VecVisionLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    rB   r#   Nc                    s.   t    t|j|j| _t|j| _d S r2   )	r3   r4   r   r   rE   denserT   rU   rV   r5   rB   r6   r   r   r4        
z!Data2VecVisionSelfOutput.__init__r8   input_tensorc                 C      |  |}| |}|S r2   r   rV   )r5   r8   r   gammar   r   r   r:        

z Data2VecVisionSelfOutput.forwardr2   )
r   r   r   r   r   r4   r)   r>   r:   r@   r   r   r6   r   r     s    &r   )eagersdpac                       s   e Zd Zddedee ddf fddZdd Z							dd
ej	deej	 de
ded de
deee  deeej	 eej	ej	f f fddZ  ZS )Data2VecVisionAttentionNrB   r   r#   c                    s4   t    t|j ||d| _t|| _t | _d S )Nr   )	r3   r4   &DATA2VEC_VISION_SELF_ATTENTION_CLASSES_attn_implementation	attentionr   r/   setpruned_headsr   r6   r   r   r4     s   

z Data2VecVisionAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   ra   )lenr   r   r   r   r   r   r   r   r   r/   r   r   union)r5   headsindexr   r   r   prune_heads  s   z#Data2VecVisionAttention.prune_headsFr8   r   r   r   r   rq   r   c           
      C   s:   |  ||||||}| |d |}|f|dd   }	|	S )Nr   r   )r   r/   )
r5   r8   r   r   r   rq   r   self_outputsattention_outputr   r   r   r   r:     s   	zData2VecVisionAttention.forwardr2   r   )r   r   r   r   r   r   r4   r   r)   r>   r   r   r   r:   r@   r   r   r6   r   r     s.     
r   c                       <   e Zd Zdeddf fddZdejdejfddZ  ZS )	Data2VecVisionIntermediaterB   r#   Nc                    sD   t    t|j|j| _t|jt	rt
|j | _d S |j| _d S r2   )r3   r4   r   r   rE   intermediate_sizer   rL   
hidden_actr?   r   intermediate_act_fnr   r6   r   r   r4     s
   
z#Data2VecVisionIntermediate.__init__r8   c                 C   r   r2   )r   r   r9   r   r   r   r:     r   z"Data2VecVisionIntermediate.forward	r   r   r   r   r4   r)   r>   r:   r@   r   r   r6   r   r     s    r   c                       r   )	Data2VecVisionOutputrB   r#   Nc                    s.   t    t|j|j| _t|j| _	d S r2   )
r3   r4   r   r   r   rE   r   rT   rU   rV   r   r6   r   r   r4     r   zData2VecVisionOutput.__init__r8   c                 C   r   r2   r   r9   r   r   r   r:     r   zData2VecVisionOutput.forwardr   r   r   r6   r   r     s    r   c                       s   e Zd ZdZ	ddedee deddf fdd	Z		
		
	dde	j
dee	j
 dedee	j
 dedeeeef  deee	j
 ee	j
e	j
f f fddZ  ZS )Data2VecVisionLayerz?This corresponds to the Block class in the timm implementation.Nr   rB   r   drop_path_rater#   c                    s   t    |j| _d| _t||d| _t|| _t|| _	t
j|j|jd| _|dkr/t|nt
 | _t
j|j|jd| _|j}|dkrct
j|t|j dd| _t
j|t|j dd| _d S d\| _| _d S )	Nr   r   epsr   r   T)requires_gradr   )r3   r4   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r/   r   	LayerNormrE   layer_norm_epslayernorm_beforer1   Identityr0   layernorm_afterlayer_scale_init_valuerC   r)   oneslambda_1lambda_2)r5   rB   r   r   init_valuesr6   r   r   r4     s   


 zData2VecVisionLayer.__init__Fr8   r   r   r   rq   r   c                 C   s   | j | ||||||d}|d }|dd  }	| jd ur"| j| }| || }| |}
| |
}
| |
}
| jd urB| j|
 }
| |
| }
|
f|	 }	|	S )N)r   r   rq   r   r   r   )r   r   r   r0   r   r   r/   r   )r5   r8   r   r   r   rq   r   self_attention_outputsr   r   layer_outputr   r   r   r:     s*   	







zData2VecVisionLayer.forward)Nr   r   )r   r   r   r   r   r   r   r=   r4   r)   r>   r   r   r   r:   r@   r   r   r6   r   r     s@    r   c                       sf   e Zd Zdededdf fddZedddeeef dej	fd	d
Z
ddedej	fddZ  ZS )r   rB   r   r#   Nc                    sR   t    || _d|d  d d|d  d  d | _tt| j|j| _	d S )Nr[   r   r   r   )
r3   r4   r   num_relative_distancer   rC   r)   rD   r   relative_position_bias_tabler   r6   r   r   r4   @  s   
&
z+Data2VecVisionRelativePositionBias.__init__
   )maxsizec           	      C   s  d|d  d d|d  d  d }|d |d  }t jt |d t |d dd}t |}t |d}|dddddf |dddddf  }|ddd }|dddddf  |d d 7  < |dddddf  |d d 7  < |dddddf  d|d  d 9  < t j|d fd |jd}|	d	|ddddf< |d |dddf< |d |dddf< |d |d
< |S )z
        This method creates the relative position index, modified to support arbitrary window sizes,
        as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
        r[   r   r   r   ij)indexingN)r^   r%   rZ   )r   r   )
r)   meshgridarangestackr   rf   r   rD   r%   sum)	r5   r   r   window_areagridcoordscoords_flattenrelative_coordsrelative_position_indexr   r   r    generate_relative_position_indexI  s    $$
,&&*zCData2VecVisionRelativePositionBias.generate_relative_position_indexFrq   c                 C   sf  d| j d  d }d| j d  d }d|d  d }d|d  d }| j}| j}	|| d }
|d|	d  }|d||ddddd}tjj|t|t|fdd}|dddd|
d d}t	
|||	d d g}| |}||d }||d |d  d |d |d  d d}|ddd }|rtjj|d||fdd	d
d}|dS )zu
        Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
        r[   r   r   r   NrZ   bilinear)r^   r_   Fr]   )r   r   r   re   rf   r   rg   rh   r   r)   rj   r  ri   r   rw   squeeze)r5   r   rq   r   
old_height	old_widthrn   ro    old_relative_position_bias_tableold_num_relative_distancenew_num_relative_distanceold_sub_tablenew_sub_table new_relative_position_bias_tabler  r   r   r   r   r:   b  s@   
&
z*Data2VecVisionRelativePositionBias.forward)FN)r   r   r   r   r   r4   r   r   r)   r>   r  r   r:   r@   r   r   r6   r   r   ?  s
    	r   c                       s   e Zd Zddedee ddf fddZ						dd	ejd
eej de	de	de	deee
e
f  de	deeef fddZ  ZS )Data2VecVisionEncoderNrB   r   r#   c                    sz   t     | _ j| _| jrt d| _dd tjd j	 j
ddD t fddt j
D | _d| _d S )	Nr   c                 S   s   g | ]}|  qS r   )item.0xr   r   r   
<listcomp>  s    z2Data2VecVisionEncoder.__init__.<locals>.<listcomp>r   cpu)r&   c                    s(   g | ]}t   jrnd | dqS )N)r   r   )r   use_relative_position_biasr  irB   dprr   r   r   r    s    F)r3   r4   rB   !use_shared_relative_position_biasr   r   r   r)   linspacer   num_hidden_layersr   
ModuleListrangelayergradient_checkpointingr   r6   r  r   r4     s   
 

zData2VecVisionEncoder.__init__FTr8   r   r   output_hidden_statesrq   r   return_dictc              	   C   s   |rdnd }|r
dnd }	t | jD ]M\}
}|r||f }| jr;|\}}|| jj || jj f}| j|||jd d}nd }|d urE||
 nd }|||||||d}|d }|r^|	|d f }	q|rf||f }|sttdd |||	fD S t|||	dS )	Nr   r   )rq   r   )r   r   r   rq   r   r   c                 s   s    | ]	}|d ur|V  qd S r2   r   )r  vr   r   r   	<genexpr>  s    z0Data2VecVisionEncoder.forward.<locals>.<genexpr>)last_hidden_stater8   
attentions)		enumerater!  r   rB   rK   r   r'   r   r
   )r5   r8   r   r   r#  rq   r   r$  all_hidden_statesall_self_attentionsr  layer_modulerX   rY   r   r   layer_head_masklayer_outputsr   r   r   r:     sD   

	
zData2VecVisionEncoder.forwardr2   )NFFFNT)r   r   r   r   r   r   r4   r)   r>   r   r   r   r
   r:   r@   r   r   r6   r   r    s2     
	r  c                   @   s:   e Zd ZU eed< dZdZdZdgZdgZ	dZ
dd Zd	S )
Data2VecVisionPreTrainedModelrB   data2vec_visionrr   Tr   z.*relative_position_index.*c                 C   sb  t |tjtjtjfr%|jjjd| jj	d |j
dur#|j
j  dS dS t |tjrH|jjjd| jj	d |jdurF|jj|j   dS dS t |tjr]|j
j  |jjd dS t |tr|jj  |jdurs|jj  |jdur|jj  dS dS t |tr|jj  dS t |tr|jdur|jj| jj |jj| jj dS dS dS )zInitialize the weightsr   )meanstdNg      ?)rL   r   r   r   ConvTranspose2dweightdatanormal_rB   initializer_ranger   zero_	Embeddingpadding_idxr   fill_rA   rF   rH   rS   r   r   r   r   r   r   )r5   moduler   r   r   _init_weights  s8   







z+Data2VecVisionPreTrainedModel._init_weightsN)r   r   r   r   __annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules"_keys_to_ignore_on_load_unexpected_supports_sdpar=  r   r   r   r   r/    s   
 r/  c                       s   e Zd Zddededdf fddZdd	 Zd
d Ze						dde	j
dee	j dee	j
 dee dee dedee deeef fddZ  ZS )Data2VecVisionModelFrB   add_pooling_layerr#   Nc                    sp   t  | || _t|| _t|| jjjd| _|j	rt
 nt
j|j|jd| _|r/t|nd| _|   dS )zw
        add_pooling_layer (bool, *optional*, defaults to `False`):
            Whether to add a pooling layer
        r   r   N)r3   r4   rB   rA   rW   r  rJ   r   encoderuse_mean_poolingr   r   r   rE   r   	layernormData2VecVisionPoolerpooler	post_init)r5   rB   rF  r6   r   r   r4   
  s   
zData2VecVisionModel.__init__c                 C   s   | j jS r2   )rW   rJ   r;   r   r   r   get_input_embeddings  s   z(Data2VecVisionModel.get_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrG  r!  r   r   )r5   heads_to_pruner!  r   r   r   r   _prune_heads   s   z Data2VecVisionModel._prune_headsrr   rs   r   r   r#  rq   r$  c              	   C   s   |dur|n| j j}|dur|n| j j}|dur|n| j j}| || j j}| j||d\}}	|jdd }
| j|||||
||d}|d }| 	|}| j
durU| 
|nd}|sl|dura||fn|f}||dd  S t|||j|jdS )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        N)rs   r[   )r   r   r#  r   r$  rq   r   r   )r'  pooler_outputr8   r(  )rB   r   r#  use_return_dictget_head_maskr  rW   r'   rG  rI  rK  r   r8   r(  )r5   rr   rs   r   r   r#  rq   r$  embedding_outputry   r   encoder_outputssequence_outputpooled_outputhead_outputsr   r   r   r:   (  s8   	
zData2VecVisionModel.forward)F)NNNNFN)r   r   r   r   r   r4   rM  rP  r   r)   r>   r   r   r   r   r   r:   r@   r   r   r6   r   rE    s8    
	rE  c                       r   )	rJ  rB   r#   Nc                    s2   t    |jrtj|j|jd| _d S d | _d S )Nr   )r3   r4   rH  r   r   rE   r   rI  r   r6   r   r   r4   b  s
   
zData2VecVisionPooler.__init__r8   c                 C   sL   | j d ur|d d dd d d f }|  |d}|S |d d df }|S )Nr   r   )rI  r1  )r5   r8   patch_tokensrW  r   r   r   r:   h  s   
zData2VecVisionPooler.forwardr   r   r   r6   r   rJ  a  s    rJ  z
    Data2VecVision Model transformer with an image classification head on top (a linear layer on top of the average of
    the final hidden states of the patch tokens) e.g. for ImageNet.
    c                       s   e Zd Zdeddf fddZe							ddeej deej d	eej d
ee	 dee	 de	dee	 de
eef fddZ  ZS )$Data2VecVisionForImageClassificationrB   r#   Nc                    sR   t  | |j| _t|dd| _|jdkrt|j|jnt | _	| 
  d S )NTrF  r   )r3   r4   
num_labelsrE  r0  r   r   rE   r   
classifierrL  r   r6   r   r   r4   |  s
   $z-Data2VecVisionForImageClassification.__init__Frr   r   labelsr   r#  rq   r$  c                 C   s   |dur|n| j j}| j||||||d}|r|jn|d }	| |	}
d}|dur1| ||
| j }|sG|
f|dd  }|durE|f| S |S t||
|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr   r   r#  rq   r$  r   r[   losslogitsr8   r(  )	rB   rR  r0  rQ  r]  loss_functionr   r8   r(  )r5   rr   r   r^  r   r#  rq   r$  r   rW  rb  ra  r/   r   r   r   r:     s.   	
z,Data2VecVisionForImageClassification.forwardNNNNNFN)r   r   r   r   r4   r   r   r)   r>   r   r   r   r   r:   r@   r   r   r6   r   rZ  t  s6    
	rZ  c                       s   e Zd ZdZ			ddededeeeeef f deeeeef ef d	ed
eeeeef f ddf fddZ	de
jde
jfddZ  ZS )Data2VecVisionConvModuleaD  
    A convolutional block that bundles conv/norm/activation layers. This block simplifies the usage of convolution
    layers, which are commonly used with a norm layer (e.g., BatchNorm) and activation layer (e.g., ReLU).

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r   Fr   in_channelsout_channelsr   paddingr   dilationr#   Nc                    s<   t    tj||||||d| _t|| _t | _d S )N)rf  rg  r   rh  r   ri  )	r3   r4   r   r   convBatchNorm2dbnReLU
activation)r5   rf  rg  r   rh  r   ri  r6   r   r   r4     s   
	z!Data2VecVisionConvModule.__init__r    c                 C   s"   |  |}| |}| |}|S r2   )rj  rl  rn  )r5   r    r/   r   r   r   r:     s   


z Data2VecVisionConvModule.forward)r   Fr   )r   r   r   r   r   r   r   r?   r   r4   r)   r>   r:   r@   r   r   r6   r   re    s*    re  c                       sD   e Zd Zdedededdf fddZdejdejfd	d
Z  ZS )!Data2VecVisionPyramidPoolingBlock
pool_scalerf  channelsr#   Nc                    sL   t    t|t||ddg| _t| jD ]\}}| t|| qd S )Nr   r   )	r3   r4   r   AdaptiveAvgPool2dre  layersr)  
add_moduler?   )r5   rp  rf  rq  r  r!  r6   r   r   r4     s   
z*Data2VecVisionPyramidPoolingBlock.__init__r    c                 C   s   |}| j D ]}||}q|S r2   )rt  )r5   r    hidden_stater!  r   r   r   r:     s   

z)Data2VecVisionPyramidPoolingBlock.forward)	r   r   r   r   r4   r)   r>   r:   r@   r   r   r6   r   ro    s    	ro  c                
       sX   e Zd ZdZdeedf dedededdf
 fd	d
Zdej	de
ej	 fddZ  ZS )"Data2VecVisionPyramidPoolingModulea  
    Pyramid Pooling Module (PPM) used in PSPNet.

    Args:
        pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
            Module.
        in_channels (int): Input channels.
        channels (int): Channels after modules, before conv_seg.
        align_corners (bool): align_corners argument of F.interpolate.

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    pool_scales.rf  rq  r`   r#   Nc                    sh   t    || _|| _|| _|| _g | _t|D ]\}}t|||d}| j	| | 
t|| qd S )N)rp  rf  rq  )r3   r4   rx  r`   rf  rq  blocksr)  ro  appendru  r?   )r5   rx  rf  rq  r`   r  rp  blockr6   r   r   r4     s   
z+Data2VecVisionPyramidPoolingModule.__init__r  c                 C   sH   g }| j D ]}||}tjj|| dd  d| jd}|| q|S )Nr[   r  r]   )ry  r   rg   rh   r^   r`   rz  )r5   r  ppm_outsppmppm_outupsampled_ppm_outr   r   r   r:     s   
z*Data2VecVisionPyramidPoolingModule.forward)r   r   r   r   r   r   r   r4   r)   r>   listr:   r@   r   r   r6   r   rw    s    *"rw  c                       sH   e Zd ZdZdeddf fddZdd Zd	ejdejfd
dZ	  Z
S )Data2VecVisionUperHeadz
    Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
    [UPerNet](https://huggingface.co/papers/1807.10221).

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    rB   r#   Nc                    s  t    |j| _|jgd | _|j| _d| _tj| j|j	dd| _
t| j| jd | j| jd| _t| jd t| j| j  | jddd| _t | _t | _| jd d D ] }t|| jdd}t| j| jddd}| j| | j| qWtt| j| j | jddd| _d S )	N   Fr   rr  rZ   )r`   r   r   rh  )r3   r4   rx  rE   rf  rq  r`   r   r   r\  r]  rw  psp_modulesre  r   
bottleneckr  lateral_convs	fpn_convsrz  fpn_bottleneck)r5   rB   rf  l_convfpn_convr6   r   r   r4      s>   


zData2VecVisionUperHead.__init__c                 C   s:   |d }|g}| | | tj|dd}| |}|S )NrZ   r   ra   )extendr  r)   rj   r  )r5   inputsr  psp_outsr/   r   r   r   psp_forwardF  s   
z"Data2VecVisionUperHead.psp_forwardencoder_hidden_statesc                    s   fddt jD   t}t|d ddD ]$}|d  jdd  }|d  tjj	| |dj
d |d < q fd	dt|d D }|d  t|d ddD ]}tjj	|| |d jdd  dj
d||< qbtj|dd
}|}|}|S )Nc                    s   g | ]
\}}| | qS r   r   )r  r  lateral_conv)r  r   r   r  Q  s    z2Data2VecVisionUperHead.forward.<locals>.<listcomp>r   r   rZ   r[   r  r]   c                    s   g | ]}j |  | qS r   )r  r  )lateralsr5   r   r   r  ^  s    ra   )r)  r  rz  r  r   r   r'   r   rg   rh   r`   r)   rj   r  r]  )r5   r  used_backbone_levelsr  
prev_shapefpn_outsr/   r   )r  r  r5   r   r:   O  s$   

zData2VecVisionUperHead.forward)r   r   r   r   r   r4   r  r)   r>   r:   r@   r   r   r6   r   r    s
    &	r  c                       sd   e Zd ZdZ			ddedededeeeeef f d	d
f
 fddZde	j
d	e	j
fddZ  ZS )Data2VecVisionFCNHeada  
    Fully Convolution Networks for Semantic Segmentation. This head is implemented of
    [FCNNet](https://huggingface.co/papers/1411.4038>).

    Args:
        config (Data2VecVisionConfig): Configuration.
        in_channels
        kernel_size (int): The kernel size for convs in the head. Default: 3.
        dilation (int): The dilation rate for convs in the head. Default: 1.


    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r[   r   r   rB   in_indexr   ri  r#   Nc              
      s   t    |j| _|j| _|j| _|j| _	|| _
|d | }g }|t| j| j|||d t| jd D ]}|t| j| j|||d q5| jdkrQt | _ntj| | _| j	rjt| j| j | j||d d| _tj| j|jdd| _d S )Nr[   )r   rh  ri  r   r   r  rr  )r3   r4   rE   rf  auxiliary_channelsrq  auxiliary_num_convs	num_convsauxiliary_concat_inputconcat_inputr  rz  re  r   r   r   convs
Sequentialconv_catr   r\  r]  )r5   rB   r  r   ri  conv_paddingr  r  r6   r   r   r4   }  s6   

zData2VecVisionFCNHead.__init__r  c                 C   s@   || j  }| |}| jr| tj||gdd}| |}|S )Nr   ra   )r  r  r  r  r)   rj   r]  )r5   r  r8   r/   r   r   r   r:     s   


zData2VecVisionFCNHead.forward)r[   r   r   )r   r   r   r   r   r   r   r   r4   r)   r>   r:   r@   r   r   r6   r   r  n  s"    &r  c                       s   e Zd Zdeddf fddZdd Ze							dd	eej	 d
eej	 deej	 dee
 dee
 de
dee
 deeef fddZ  ZS )%Data2VecVisionForSemanticSegmentationrB   r#   Nc                    s   t  | |j| _t|dd| _t| jjdkrtdt	
t	j|j|jdddt	|jt	 t	j|j|jddd| _t	
t	j|j|jddd| _t	 | _t	jddd| _t|| _|jrft|nd | _|   d S )NFr[  r  zData2VecVisionForSemanticSegmentation requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.r[   r   )r3   r4   r\  rE  r0  r   rB   out_indicesr   r   r  r3  rE   rk  GELUfpn1fpn2r   fpn3	MaxPool2dfpn4r  decode_headuse_auxiliary_headr  auxiliary_headrL  r   r6   r   r   r4     s*   


z.Data2VecVisionForSemanticSegmentation.__init__c           
      C   s   t jj||jdd  ddd}|d ur"t jj||jdd  ddd}t| jjd}|||}|}|d urA|||}	|| jj|	 7 }|S )Nr   r  Fr]   )ignore_index)r   rg   rh   r'   r   rB   semantic_loss_ignore_indexauxiliary_loss_weight)
r5   rb  auxiliary_logitsr^  upsampled_logitsupsampled_auxiliary_logitsloss_fct	main_lossra  auxiliary_lossr   r   r   compute_loss  s   

z2Data2VecVisionForSemanticSegmentation.compute_lossFrr   r   r^  r   r#  rq   r$  c                    s  |dur|nj j}|dur|nj j}|dur"j jdkr"tdj|||d||d}|r2|jn|d }	fddt|	D }
|jd  j j	j j
  fd	d|
D }
jjjjg}tt|
D ]}|| |
| |
|< qh|
}d}jdur|
}d}|dur|||}|s|r|f|dd  }n	|f|d
d  }|dur|f| S |S t|||r|jnd|jdS )a@  
        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, Data2VecVisionForSemanticSegmentation
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/data2vec-vision-base")
        >>> model = Data2VecVisionForSemanticSegmentation.from_pretrained("facebook/data2vec-vision-base")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> # logits are of shape (batch_size, num_labels, height, width)
        >>> logits = outputs.logits
        ```Nr   z/The number of labels should be greater than oneTr_  c                    s$   g | ]\}}|d   j jv r|qS r$   )rB   r  )r  idxfeaturer;   r   r   r    s   $ zAData2VecVisionForSemanticSegmentation.forward.<locals>.<listcomp>r   c                    s<   g | ]}|d d dd d d f  ddd dqS )Nr   r   r[   rZ   )rf   re   r  )r|   patch_resolutionr   r   r    s    0r[   r`  )rB   rR  r#  r\  r   r0  r8   r)  r'   rM   rK   r  r  r  r  r   r   r  r  r  r   r(  )r5   rr   r   r^  r   r#  rq   r$  r   r  featuresopsr  rb  r  ra  r/   r   )r|   r  r5   r   r:     sR   "	



z-Data2VecVisionForSemanticSegmentation.forwardrd  )r   r   r   r   r4   r  r   r   r)   r>   r   r   r   r   r:   r@   r   r   r6   r   r    s8     
	r  )rZ  r  rE  r/  )r   F)Dr   collections.abcrN   r   rt   dataclassesr   typingr   r   r)   r   torch.nnr   activationsr   modeling_layersr	   modeling_outputsr
   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   configuration_data2vec_visionr   
get_loggerr   r   r   r>   r=   r   r0   Moduler1   rA   rI   r   r   r   r   r   r   r   r   r   r  r/  rE  rJ  rZ  re  ro  rw  r  r  r  __all__r   r   r   r   <module>   sp   
 
g'ZK/DTL&X=&(V? 