o
    XiS                     @   s  d dl Z d dl mZmZ d dlm  mZ d dlmZ d dlm	Z	 dd Z
dd Zd	d
 ZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZdS )    N)nneinsum	rearrange)	Rearrangec                 C   s   | d uS N )valr   r   G/home/ubuntu/.local/lib/python3.10/site-packages/conformer/conformer.pyexists
   s   r   c                 C   s   t | r| S |S r   )r   )r	   dr   r   r
   default   s   r   c                 C   s   | d }||| d d  fS )N      r   )kernel_sizepadr   r   r
   calc_same_padding   s   r   c                   @   s   e Zd Zdd ZdS )Swishc                 C   s   ||   S r   )sigmoidselfxr   r   r
   forward   s   zSwish.forwardN)__name__
__module____qualname__r   r   r   r   r
   r      s    r   c                       $   e Zd Z fddZdd Z  ZS )GLUc                    s   t    || _d S r   )super__init__dim)r   r    	__class__r   r
   r      s   

zGLU.__init__c                 C   s    |j d| jd\}}||  S )Nr   r    )chunkr    r   )r   r   outgater   r   r
   r      s   zGLU.forwardr   r   r   r   r   __classcell__r   r   r!   r
   r      s    r   c                       r   )DepthWiseConv1dc                    s(   t    || _tj||||d| _d S )N)groups)r   r   paddingr   Conv1dconv)r   chan_inchan_outr   r+   r!   r   r
   r   $   s   
zDepthWiseConv1d.__init__c                 C   s   t || j}| |S r   )Fr   r+   r-   r   r   r   r
   r   )   s   
zDepthWiseConv1d.forwardr'   r   r   r!   r
   r)   #       r)   c                       r   )Scalec                    s   t    || _|| _d S r   )r   r   fnscale)r   r4   r3   r!   r   r
   r   0   s   

zScale.__init__c                 K   s   | j |fi || j S r   )r3   r4   r   r   kwargsr   r   r
   r   5   s   zScale.forwardr'   r   r   r!   r
   r2   /   r1   r2   c                       r   )PreNormc                    s    t    || _t|| _d S r   )r   r   r3   r   	LayerNormnorm)r   r    r3   r!   r   r
   r   9   s   
zPreNorm.__init__c                 K   s   |  |}| j|fi |S r   )r9   r3   r5   r   r   r
   r   >   s   
zPreNorm.forwardr'   r   r   r!   r
   r7   8   r1   r7   c                       s6   e Zd Z				d
 fdd	Z			ddd	Z  ZS )	Attention   @              c                    s   t    || }|| _|d | _tj||dd| _tj||d dd| _t||| _|| _	t
d| d || _t|| _d S )Ng      F)biasr   r   )r   r   headsr4   r   Linearto_qto_kvto_outmax_pos_emb	Embeddingrel_pos_embDropoutdropout)r   r    r@   dim_headrI   rE   	inner_dimr!   r   r
   r   C   s   

zAttention.__init__Nc                    s  j d j| j| jt f\}}}t  | g|  jdddR \}}	}
t	fdd||	|
f\}}	}
t
d||	| j }tj|d}t|d	t|d
 }|| || }| ||}t
d||| j }|| }t|s|t|rt|fdd}|st||n	t| fdd}t|jj }t|dt|d }|| | |jdd}t
d||
}t|d}| |}| |S )Nr   r#   c                    s   t | d dS )Nzb n (h d) -> b h n dhr   )trN   r   r
   <lambda>c   s    z#Attention.forward.<locals>.<lambda>zb h i d, b h j d -> b h i j)devicez	i -> i ()z	j -> () jzb h n d, n r d -> b h n rc                      s   t jjd d d iS Nr   rR   torchonesshaper   )rR   r   r   r
   rQ   q       c                      s   t j jd d diS rS   rT   r   )contextrR   r   r
   rQ   r   rX   zb i -> b () i ()zb j -> b () () jzb h i j, b h j d -> b h i dzb h n d -> b n (h d))rW   rR   r@   rE   r   r   rB   rC   r$   mapr   r4   rU   aranger   clamprG   tofinfodtypemaxmasked_fill_softmaxrD   rI   )r   r   rY   maskcontext_masknrE   has_contextqkvdotsseqdistrG   pos_attn
mask_valueattnr%   r   )rY   rR   rO   r   r
   r   X   s,   (
("


zAttention.forward)r;   r<   r=   r>   )NNNr'   r   r   r!   r
   r:   B   s    r:   c                       s*   e Zd Z		d fdd	Zdd Z  ZS )FeedForward   r=   c              	      sH   t    tt||| t t|t|| |t|| _d S r   )r   r   r   
SequentialrA   r   rH   net)r   r    multrI   r!   r   r
   r      s   

zFeedForward.__init__c                 C   
   |  |S r   rs   r   r   r   r
   r         
zFeedForward.forward)rq   r=   r'   r   r   r!   r
   rp   ~   s
    rp   c                       s.   e Zd Z				d	 fdd	Zdd Z  ZS )
ConformerConvModuleFr      r=   c                    s   t    || }|st|n|d df}tt|tdt||d dtddt	||||d|s8t
|nt t t||dtdt|
| _d S )Nr   r   zb n c -> b c nr   r#   )r   r+   zb c n -> b n c)r   r   r   r   rr   r8   r   r,   r   r)   BatchNorm1dIdentityr   rH   rs   )r   r    causalexpansion_factorr   rI   rK   r+   r!   r   r
   r      s   

zConformerConvModule.__init__c                 C   ru   r   rv   r   r   r   r
   r      rw   zConformerConvModule.forward)Fr   ry   r=   r'   r   r   r!   r
   rx      s    rx   c                
       s<   e Zd Zdddddddddd	 fd	d

ZdddZ  ZS )ConformerBlockr<   r;   rq   r   ry   r=   F	rJ   r@   ff_multconv_expansion_factorconv_kernel_sizeattn_dropout
ff_dropoutconv_dropoutconv_causalc       
            s   t    t|||d| _t||||d| _t||
|||	d| _t|||d| _t	|| j| _t
dt	|| j| _t
dt	|| j| _t|| _d S )N)r    rt   rI   )r    rJ   r@   rI   )r    r|   r}   r   rI   g      ?)r   r   rp   ff1r:   ro   rx   r-   ff2r7   r2   r   r8   	post_norm)r   r    rJ   r@   r   r   r   r   r   r   r   r!   r   r
   r      s   
zConformerBlock.__init__Nc                 C   sJ   |  || }| j||d| }| || }| || }| |}|S )N)rc   )r   ro   r-   r   r   )r   r   rc   r   r   r
   r      s   
zConformerBlock.forwardr   r'   r   r   r!   r
   r~      s    r~   c                
       s:   e Zd Zdddddddddd	 fd	d

Zdd Z  ZS )	Conformerr<   r;   rq   r   ry   r=   Fr   c       
            sL   t    || _tg | _t|D ]}| jt|||||||d qd S )N)r    rJ   r@   r   r   r   r   )	r   r   r    r   
ModuleListlayersrangeappendr~   )r   r    depthrJ   r@   r   r   r   r   r   r   r   _r!   r   r
   r      s   

zConformer.__init__c                 C   s   | j D ]}||}q|S r   )r   )r   r   blockr   r   r
   r      s   

zConformer.forwardr'   r   r   r!   r
   r      s    r   )rU   r   r   torch.nn.functional
functionalr0   einopsr   einops.layers.torchr   r   r   r   Moduler   r   r)   r2   r7   r:   rp   rx   r~   r   r   r   r   r
   <module>   s$    		
< %