o
    im-                     @   s  d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ	 d dl
mZ d dlmZ ddlmZ G d	d
 d
ejZG dd deZG dd dejZG dd dejZG dd dejZdd Zg dg dg ddZG dd dejZG dd dejZ	dBdeded ed!ed"ejf
d#d$Z	%dCdeded ed&ed"ejf
d'd(Zd)d* Zd+d, ZG d-d. d.ejZ dDd0d1Z!G d2d3 d3ejZ"d4Z#d5d6 Z$e$dEd7d8Z%e$d9d, ZG d:d; d;ej&Z'G d<d= d=ejZ(ej)j*d>d? Z+G d@dA dAejZ,dS )F    )reduceN)nn)
functional)
sdp_kernel)version   )Snake1dc                       &   e Zd Zd fdd	Zdd Z  ZS )ResidualBlockNc                    s2   t    tj| | _|r|| _d S t | _d S N)super__init__r   
SequentialmainIdentityskip)selfr   r   	__class__ c/home/ubuntu/.local/lib/python3.10/site-packages/solospeech/vae_modules/stable_vae/models/blocks.pyr      s   
zResidualBlock.__init__c                 C   s   |  || | S r   )r   r   r   inputr   r   r   forward   s   zResidualBlock.forwardr   __name__
__module____qualname__r   r   __classcell__r   r   r   r   r
      s    r
   c                       s   e Zd Zd fdd	Z  ZS )ResConvBlockF   Tc           	         s   ||krd nt j||ddd}t t j||||d |dt d||r(t|nt  t j||||d |d|s>t d|nt  |sM|rIt|nt  nt  g| d S )Nr   Fbias   )paddingr"   )r   Conv1dr   r   	GroupNormr   GELUr   )	r   c_inc_midc_outis_lastkernel_size	conv_bias	use_snaker   r   r   r   r      s   
zResConvBlock.__init__)Fr    TF)r   r   r   r   r   r   r   r   r   r      s    r   c                       &   e Zd Zd fdd	Zdd Z  ZS )SelfAttention1dr           c                    s   t    || dksJ td|| _|| _t||d d| _t||d| _tj	|dd| _
tj o@ttjtdk| _| jsGd S tjtd}|jdkr_|jdkr_d	| _d S d
| _d S )Nr   r      T)inplacez2.0.0cuda   )TFF)FTT)r   r   r   r&   normn_headr%   qkv_projout_projDropoutdropouttorchr4   is_availabler   parse__version__	use_flashget_device_propertiesdevicemajorminorsdp_kernel_config)r   r(   r7   dropout_ratedevice_propertiesr   r   r   r   &   s   
"

zSelfAttention1d.__init__c                 C   s
  |j \}}}| | |}||| jd || j |gdd}|jddd\}}}|j d d }	| jr\t| j	  t
j|||dd |||g}
W d    n1 sVw   Y  n||	 |dd|	  d}|| dd |||g}
|| | |
 S )Nr2   r#   r   dimg      пF)	is_causal)shaper8   r6   viewr7   	transposechunkr@   r   rE   Fscaled_dot_product_attention
contiguoussoftmaxr;   r9   )r   r   ncsqkvqkvscaleyattr   r   r   r   =   s"   $ zSelfAttention1d.forward)r   r1   r   r   r   r   r   r0   %   s    r0   c                       s$   e Zd Z fddZdd Z  ZS )	SkipBlockc                    s   t    tj| | _d S r   )r   r   r   r   r   )r   r   r   r   r   r   Q   s   
zSkipBlock.__init__c                 C   s   t j| ||gddS )Nr   rH   )r<   catr   r   r   r   r   r   U      zSkipBlock.forwardr   r   r   r   r   r]   P   s    r]   c                       r	   )FourierFeatures      ?c                    s<   t    |d dksJ tt|d |g| | _d S )Nr#   r   r   r   r   	Parameterr<   randnweight)r   in_featuresout_featuresstdr   r   r   r   Z   s   

zFourierFeatures.__init__c                 C   s0   dt j | | jj }tj| | gddS Nr#   rH   )mathpire   Tr<   r^   cossin)r   r   fr   r   r   r   `   s   zFourierFeatures.forward)ra   r   r   r   r   r   r`   Y   s    r`   c                 C   s   | d  dd|d gS )N).Nr   r#   )repeat)r   rK   r   r   r   expand_to_planese   r_   rr   )      ?      ?rt   rs   )                  ?     ?rx   rw   rv   ru   )    8n?   Վ?   @Dh   $   <X?   ?r~   r}   r|   r{   rz   ry   )linearcubiclanczos3c                       &   e Zd Zd fdd	Zdd Z  ZS )	Downsample1dr   reflectFc                    sH   t    || _tt| }|jd d d | _| d| || _	d S )Nr   r#   r   kernel
r   r   pad_moder<   tensor_kernelsrK   padregister_bufferchannels_lastr   r   r   r   	kernel_1dr   r   r   r   w   s   

zDownsample1d.__init__c                 C   s   | j r
|ddd}t|| jfd | j}||jd |jd | jjd g}tj	|jd |j
d}| j||||f< tj||dd}| j rO|ddd}|S )Nr   r#   r   rB   )stride)r   permuterO   r   r   	new_zerosrK   r   r<   arangerB   toconv1dr   xre   indicesr   r   r   r      s   $zDownsample1d.forwardr   r   Fr   r   r   r   r   r   v       r   c                       r   )	
Upsample1dr   r   Fc                    sL   t    || _tt| d }|jd d d | _| d| || _	d S )Nr#   r   r   r   r   r   r   r   r   r      s   

zUpsample1d.__init__c                 C   s   | j r
|ddd}t|| jd d fd | j}||jd |jd | jjd g}tj	|jd |j
d}| j||||f< tj||d| jd d d}| j rY|ddd}|S )Nr   r#   r   r   )r   r$   )r   r   rO   r   r   r   rK   r   r<   r   rB   r   conv_transpose1dr   r   r   r   r      s    $zUpsample1d.forwardr   r   r   r   r   r   r      r   r   r#   in_channelsout_channelsfactorkernel_multiplierreturnc                 C   s8   |d dks
J dt j| ||| d |||d  dS )Nr#   r   zKernel multiplier must be evenr   )r   r   r,   r   r$   )r   r%   )r   r   r   r   r   r   r   Downsample1d_2   s   

r   Fuse_nearestc              	   C   sl   |dkrt j| |dddS |r!t t j|ddt j| |dddS t j| ||d ||d |d  |d dS )Nr   r2   )r   r   r,   r$   nearest)scale_factormoder#   )r   r   r,   r   r$   output_padding)r   r%   r   UpsampleConvTranspose1d)r   r   r   r   r   r   r   Upsample1d_2   s,   
r   c                 C   s*   t j| j | jd urt j| j | S r   )r   initzeros_re   r"   )layerr   r   r   	zero_init   s   
r   c                 C   Z   t tj| j|jtjf}tj| |d ddd}||t||  }| || j S Nr#   rj   TrI   keepdimr   r<   promote_typesdtypefloat32meanr   rsqrtr   rZ   epsr   mean_sqr   r   r   rms_norm   s   r   c                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	
AdaRMSNormư>c                    s*   t    || _ttj||dd| _d S )NFr!   )r   r   r   r   r   Linearr   )r   featurescond_featuresr   r   r   r   r      s   
zAdaRMSNorm.__init__c                 C   s   d| j  dS )Nzeps=,)r   r   r   r   r   
extra_repr   s   zAdaRMSNorm.extra_reprc                 C   s*   t || |d d d d d f d | jS )Nr   )r   r   r   )r   r   condr   r   r   r      s   *zAdaRMSNorm.forward)r   r   r   r   r   r   r   r   r   r   r   r   r      s    r   -C6?c                 C   sL   t td| j}tjj| |dd}t| |   }| tj	|||d S )Nr   Tr   )alpha)
listrangendimr<   linalgvector_normnpsqrtnumeladd)r   r   rI   rS   r   r   r   r   	normalize   s   r   c                       r	   )ForcedWNConv1dr   c                    s&   t    tt|||g| _d S r   rb   )r   r   r   r,   r   r   r   r      s   
zForcedWNConv1d.__init__c                 C   sr   | j r t  | jt| j W d    n1 sw   Y  | jd  }t| jt| }t	j
||ddS )Nr   same)r$   )trainingr<   no_gradre   copy_r   r   rk   r   rO   r   )r   r   fan_inwr   r   r   r      s   
zForcedWNConv1d.forward)r   r   r   r   r   r   r          r   Tc                 O   s:   t s| S ztj| g|R i |W S  ty   |  Y S w r   )use_compiler<   compileRuntimeError)functionargskwargsr   r   r   r     s   r   c                 C   s:   | |j  } |d ur| | } | jddd\} }| t| S ri   )mTrN   rO   gelu)r   re   r"   gater   r   r   linear_geglu  s
   
r   c                 C   r   r   r   r   r   r   r   r     s   c                       r	   )LinearGEGLUTc                    s    t  j||d |d || _d S )Nr#   r!   )r   r   rg   )r   rf   rg   r"   r   r   r   r   "  s   
zLinearGEGLU.__init__c                 C      t || j| jS r   )r   re   r"   r   r   r   r   r   r   &     zLinearGEGLU.forward)Tr   r   r   r   r   r   !  r   r   c                       s.   e Zd Zd	 fdd	Zdd Zdd Z  ZS )
RMSNormFr   c                    s@   t    || _|r| dt| d S tt|| _d S )NrZ   )	r   r   r   r   r<   onesr   rc   rZ   )r   rK   	fix_scaler   r   r   r   r   +  s
   
zRMSNorm.__init__c                 C   s   dt | jj d| j S )Nzshape=z, eps=)tuplerZ   rK   r   r   r   r   r   r   4  s   zRMSNorm.extra_reprc                 C   r   r   )r   rZ   r   r   r   r   r   r   7  r   zRMSNorm.forward)Fr   r   r   r   r   r   r   *  s    	r   c                 C   s$   | d|d  t t| | d  S )Nra   g&.>r#   )powr<   ro   )r   r   betar   r   r   
snake_beta<  s   $r   c                       r/   )	SnakeBetara   Tc                    s   t t|   || _|| _| jr'tt|| | _	tt|| | _
ntt|| | _	tt|| | _
|| j	_|| j
_d S r   )r   r   r   rf   alpha_logscaler   rc   r<   zerosr   r   r   requires_grad)r   rf   r   alpha_trainabler   r   r   r   r   J  s   zSnakeBeta.__init__c                 C   sN   | j dd}| jdd}| jrt|}t|}t|||}|S )Nr   rj   )r   	unsqueezer   r   r<   expr   )r   r   r   r   r   r   r   r   ^  s   

zSnakeBeta.forward)ra   TTr   r   r   r   r   r   H  s    r   )r#   )F)r   r   )-	functoolsr   rk   numpyr   r<   r   torch.nnr   rO   torch.backends.cudar   	packagingr   	nn.layersr   Moduler
   r   r0   r]   r`   rr   r   r   r   intr   boolr   r   r   r   r   r   r   r   r   r   r   r   jitscriptr   r   r   r   r   r   <module>   sz    
+	


	
		
