o
    پi-                     @   sl   d dl mZ d dlZd dlmZ d dlmZmZ d dlmZ dd ZG dd	 d	ejZ	G d
d dejZ
dS )    )OptionalN)nn)r   Tensor)_get_activation_fnc                 C   s   t | drt | drt | _| `| j}| j}t||d| _nCt | dr>t | dr>t | _| `| j}| j}t||d| _n$d| 	 v sJd| 	 v rZ| `
| j}| j}t||d| _
ntd td t | d	rjd
| _| S )Nglobal_poolfc)num_classesinitial_num_features
classifierRegNetTResNetz;Model code-writing is not aligned currently with ml-decoder	drop_rater   )hasattrr   Identityr   r   r   num_features	MLDecoderr
   	_get_nameheadprintexitr   )modelr   r    r   J/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/ml_decoder.pyadd_ml_decoder_head	   s,   


r   c                       sr   e Zd Z		d	d fdd	Z fd
dZ				ddededee dee dee dee defddZ  ZS )TransformerDecoderLayerOptimal      皙?reluh㈵>returnNc                    s   t t|   tj||d| _t|| _t|| _t|| _	t|| _
tj|||d| _t||| _t||| _tj||d| _tj||d| _t|| _d S )N)eps)dropout)superr   __init__r   	LayerNormnorm1Dropoutr#   dropout1dropout2dropout3MultiheadAttentionmultihead_attnLinearlinear1linear2norm2norm3r   
activation)selfd_modelnheaddim_feedforwardr#   r3   layer_norm_eps	__class__r   r   r%   $   s   z'TransformerDecoderLayerOptimal.__init__c                    s*   d|vrt jjj|d< tt| | d S )Nr3   )torchr   
functionalr   r$   r   __setstate__)r4   stater9   r   r   r=   8   s   z+TransformerDecoderLayerOptimal.__setstate__tgtmemorytgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskc              	   C   sz   ||  | }| |}| |||d }|| | }| |}| | | | |}|| 	| }| 
|}|S )Nr   )r)   r'   r-   r*   r1   r0   r#   r3   r/   r+   r2   )r4   r?   r@   rA   rB   rC   rD   tgt2r   r   r   forward=   s   


z&TransformerDecoderLayerOptimal.forward)r   r   r   r   r    )r!   N)NNNN)	__name__
__module____qualname__r%   r=   r   r   rF   __classcell__r   r   r9   r   r   #   s(    r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )	r   r      r   c           
         s   t t|   |dk rdn|}||kr|}|| _|dk rdn|}t||| _d}d}d}t|||d}	tj|	|d| _	t
||| _| jd	 || _t|| d
 | _tjt||| j| _tjt|| _tjj| j tjj| jd d S )Nr   d   rK   r      r   )r5   r7   r#   )
num_layersFg+?)r$   r   r%   embed_len_decoderr   r.   embed_standartr   TransformerDecoderdecoder	Embeddingquery_embedrequires_grad_r   intduplicate_factorr;   	Parameterr   duplicate_poolingduplicate_pooling_biasinitxavier_normal_	constant_)
r4   r   num_of_groupsdecoder_embeddingr	   rO   decoder_dropoutnum_layers_decoderr7   layer_decoder9   r   r   r%   [   s0   zMLDecoder.__init__c                 C   s8  t |jdkr|ddd}n|}| |}tjjj|dd}|jd }| j	j
}|dd|d}| ||dd}|dd}tj|jd |jd | j|j|jd}t| jD ](}	|d d |	d d f }
| j|	d d d d f }t|
||d d |	d d f< q\|dd d d | jf }|| j7 }|}|S )	N      rM   T)inplacer   r   )devicedtype)lenshapeflatten	transposerP   r;   r   r<   r   rT   weight	unsqueezeexpandrR   zerosrW   rf   rg   rangerO   rY   matmulr   rZ   )r4   xembedding_spatialembedding_spatial_786bsrT   r?   h
out_extrapih_iw_ih_outlogitsr   r   r   rF   {   s&   

& 
zMLDecoder.forward)r   rK   r   )rG   rH   rI   r%   rF   rJ   r   r   r9   r   r   Z   s     r   )typingr   r;   r   r   torch.nn.modules.transformerr   r   Moduler   r   r   r   r   r   <module>   s    7