o
    پi@                     @   s|   d dl mZmZ d dlZd dlmZ d dlm  mZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ G dd	 d	ejZdS )
    )OptionalTypeN   )maybe_add_mask)use_fused_attn)Mlp)trunc_normal_tf_c                       s   e Zd ZU dZejje ed< dddddddddd	d
de	j
dfdededededee dedededededededeee	j  deee	j  def fddZdd Zd"deej fd d!Z  ZS )#AttentionPoolLatentz' Attention pooling w/ latent query
    
fused_attnN   g      @TFr    tokeng        in_featuresout_features	embed_dim	num_heads	feat_size	mlp_ratioqkv_biasqk_norm
latent_len
latent_dim	pos_embed	pool_type
norm_layer	act_layerdropc                    sn  t    |p|}|p|}|| dksJ || _|| | _|| _| jd | _|| _t | _|dkrB|d us7J t	
t||| _nd | _|
pH|| _|	| _t	
td| j|| _t	j|||d| _t	j||d |d| _|r|pst	j}|| j| _|| j| _n
t	 | _t	 | _t	||| _t	|| _|d ur||nt	 | _t|t|| |d| _|   d S )Nr         absr   )bias   )r   ) super__init__r   head_dimr   scalepoolr   r
   nn	Parametertorchzerosr   r   r   latentLinearqkv	LayerNormq_normk_normIdentityprojDropout	proj_dropnormr   intmlpinit_weights)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   qk_norm_layer	__class__ N/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/attention_pool.pyr"      s<   





zAttentionPoolLatent.__init__c                 C   s<   | j d urt| j | j jd d d t| j| jd d d S )Nr   r   )std)r   r   shaper*   r   )r9   r=   r=   r>   r8   J   s   
z AttentionPoolLatent.init_weights	attn_maskc                 C   sx  |j \}}}| jd ur|| jd|j }| j|dd}| ||| j	| j
| jdd}| |||d| j
| jddddd}|d\}	}
| || |	}}	| jretj||	|
|d}n|| j }||	dd }t||}|jdd	}||
 }|dd|| j	|}| |}| |}|| | | }| jd
kr|d d df }|S | jdkr|d}|S )Nr   r   r          )rA   )dimr   avg)r@   r   	unsqueezetodtyper*   expandr,   reshaper   r   r#   	transposer-   permuteunbindr/   r0   r
   Fscaled_dot_product_attentionr$   r   softmaxr2   r4   r7   r5   r%   mean)r9   xrA   BNCq_latentr,   r-   kvattnr=   r=   r>   forwardO   s2   
$*






zAttentionPoolLatent.forward)N)__name__
__module____qualname____doc__r(   jitFinalbool__annotations__r&   GELUr6   r   floatstrr   Moduler"   r8   Tensorr\   __classcell__r=   r=   r;   r>   r	      sd   
 	
8r	   )typingr   r   r(   torch.nnr&   torch.nn.functional
functionalrP   	attentionr   configr   r7   r   weight_initr   rh   r	   r=   r=   r=   r>   <module>   s    