o
    i_                     @   s   d dl Z d dl mZ d dlmZmZ d dlmZmZ d dlm	Z	 d dl
mZ dd ZG d	d
 d
eZG dd deZG dd deZG dd deZedkrseddddddddddedddZe ddddZeeZejd ksuJ dS dS )!    N)nn)Module
ModuleList)	rearrangerepeat)	Rearrange)mHCc                 C   s   t | tr| S | | fS N)
isinstancetuplet r   I/home/ubuntu/.local/lib/python3.10/site-packages/hyper_connections/vit.pypair   s   r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )FeedForward        c              
      sJ   t    tt|t||t t|t||t|| _d S r	   )	super__init__r   
Sequential	LayerNormLinearGELUDropoutnet)selfdim
hidden_dimdropout	__class__r   r   r      s   



zFeedForward.__init__c                 C   s
   |  |S r	   )r   )r   xr   r   r   forward   s   
zFeedForward.forward)r   __name__
__module____qualname__r   r"   __classcell__r   r   r   r   r      s    r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )		Attention   @   r   c                    s   t    || }|dko||k }|| _|d | _t|| _tjdd| _t	|| _
tj||d dd| _|rJtt||t	|| _d S t | _d S )N   g      r      F)bias)r   r   headsscaler   r   normSoftmaxattendr   r   r   to_qkvr   Identityto_out)r   r   r0   dim_headr   	inner_dimproject_outr   r   r   r   !   s    



zAttention.__init__c           	         s     |} |jddd}t fdd|\}}}t||dd j } |} 	|}t||}t
|d} |S )Nr.   r,   r-   c                    s   t | d jdS )Nzb n (h d) -> b h n d)h)r   r0   r   r   r   r   <lambda>9   s    z#Attention.forward.<locals>.<lambda>zb h n d -> b n (h d))r2   r5   chunkmaptorchmatmul	transposer1   r4   r   r   r7   )	r   r!   qkvqkvdotsattnoutr   r<   r   r"   5   s   




zAttention.forward)r)   r*   r   r#   r   r   r   r   r(       s    r(   c                       s.   e Zd Zdde f fdd	Zdd Z  ZS )Transformerr      c	                    s   t    t|| _tg | _tj|fi |\}	| _	| _
t|D ]}
| jt|	|t||||dd|	|t|||ddg q"d S )N)r0   r8   r   )r   branch)r   )r   r   r   r   r2   r   layersr   +get_init_and_expand_reduce_stream_functionsexpand_streamsreduce_streamsrangeappendr(   r   )r   r   depthr0   r8   mlp_dimr   num_residual_streams
mhc_kwargsinit_hyper_conn_r   r   r   r   E   s   


zTransformer.__init__c                 C   s>   |  |}| jD ]\}}||}||}q| |}| |S r	   )rP   rN   rQ   r2   )r   r!   rI   ffr   r   r   r"   R   s   



zTransformer.forwardr$   r%   r&   dictr   r"   r'   r   r   r   r   rK   D   s    rK   c                	       s<   e Zd Zddddddeddd fd	d

Zdd Z  ZS )ViTclsr.   r*   r   rL   r+   )num_dynamic_alpha_proposals)poolchannelsr8   r   emb_dropoutrV   rW   c             	      s   t    t|\}}t|\}}|| dkr|| dks!J d|| ||  }|	| | }|dv s7J d|dkr=dnd}ttd||dt|t||t|| _t	t
||| _t	t
|| || _t|| _t||||
||||| _|| _t | _t||| _d S )	Nr   z5Image dimensions must be divisible by the patch size.>   r^   meanz?pool type must be either cls (cls token) or mean (mean pooling)r^   r+   z&b c (h p1) (w p2) -> b (h w) (p1 p2 c))p1p2)r   r   r   r   r   r   r   r   to_patch_embedding	ParameterrA   randn	cls_tokenpos_embeddingr   r   rK   transformerr`   r6   	to_latentmlp_head)r   
image_size
patch_sizenum_classesr   rT   r0   rU   r`   ra   r8   r   rb   rV   rW   image_heightimage_widthpatch_heightpatch_widthnum_patches	patch_dimnum_cls_tokensr   r   r   r   _   s*   
 

zViT.__init__c                 C   s   |j d }| |}t| jd|d}tj||fdd}|j d }|| jd |  }| |}| |}| j	dkr>|j
ddn|d d df }| |}| |S )Nr   z... d -> b ... d)br+   r-   rc   )shaperf   r   ri   rA   catrj   r   rk   r`   rc   rl   rm   )r   imgbatchr!   
cls_tokensseqr   r   r   r"      s   




&

zViT.forwardr[   r   r   r   r   r]   ^   s    $ r]   __main__         i         i   g?rL   F)use_triton_sinkhorn)rn   ro   rp   r   rT   r0   rU   r   rb   rV   rW   r+   r.   )r+   r   )rA   r   torch.nnr   r   einopsr   r   einops.layers.torchr   hyper_connections.mHCv2r   r   r   r(   rK   r]   r$   r\   rG   rh   r{   predsry   r   r   r   r   <module>   s@    $4