o
    ߥi#                     @   s   d dl Z d dlm  mZ d dl mZmZ d dlmZ dd Zdd Z	dd	 Z
G d
d dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZdS )    N)einsumnn)MossFormerConvModulec                 C   s   | d uS N )valr   r   g/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/audio/separation/mossformer_block.pyexists   s   r	   c                 C   s   t | r| S |S r   )r	   )r   dr   r   r   default   s   r   c                 C   s   | | }|dkr
dS || S )Nr   r   )nmult	remainderr   r   r   padding_to_multiple_of   s   r   c                       &   e Zd Zd fdd	Zdd Z  ZS )	ScaleNormh㈵>c                    s0   t    |d | _|| _ttd| _d S )Ng         )	super__init__scaleepsr   	Parametertorchonesg)selfdimr   	__class__r   r   r      s   

zScaleNorm.__init__c                 C   s.   t j|ddd| j }||j| jd | j S )NT)r   keepdim)min)r   normr   clampr   r   )r   xr#   r   r   r   forward"   s   zScaleNorm.forward)r   __name__
__module____qualname__r   r&   __classcell__r   r   r   r   r          r   c                       s$   e Zd Z fddZdd Z  ZS )ScaledSinuEmbeddingc                    sJ   t    ttd| _ddtd|d |   }| 	d| d S )Nr         ?i'  r      inv_freq)
r   r   r   r   r   r   r   arangefloatregister_buffer)r   r   r0   r   r   r   r   )   s   
zScaledSinuEmbedding.__init__c                 C   sZ   |j d |j}}tj||d| j}td|| j}tj| |	 fdd}|| j
 S )Nr   )devicezi , j -> i jr    r   )shaper4   r   r1   type_asr0   r   catsincosr   )r   r%   r   r4   tsinuembr   r   r   r&   /   s
   
zScaledSinuEmbedding.forwardr'   r   r   r   r   r-   '   s    r-   c                       r   )OffsetScaler   c                    sH   t    tt||| _tt||| _tj	j
| jdd d S )Ng{Gz?)std)r   r   r   r   r   r   gammazerosbetainitnormal_)r   r   headsr   r   r   r   9   s   
zOffsetScale.__init__c                 C   s    t d|| j| j }|jddS )Nz... d, h d -> ... h dr5   )r   r@   rB   unbind)r   r%   outr   r   r   r&   ?   s   zOffsetScale.forward)r   r'   r   r   r   r   r>   7   r,   r>   c                       s,   e Zd Zejdf fdd	Zdd Z  ZS )FFConvM皙?c              	      s<   t    t||t||t t|t|| _d S r   )	r   r   r   
SequentialLinearSiLUr   Dropoutmdl)r   dim_indim_out
norm_klassdropoutr   r   r   r   F   s
   

zFFConvM.__init__c                 C   s   |  |}|S r   )rO   )r   r%   outputr   r   r   r&   L   s   
zFFConvM.forward)r(   r)   r*   r   	LayerNormr   r&   r+   r   r   r   r   rI   D   s    rI   c                       sB   e Zd Zddddddejdf fdd		Zd
d ZdddZ  ZS )MossFormerBlock      r.   FrJ   NTc
                    s   t    t|| }
|| _|| _|	| _|| _t|| _	t
||
||d| _t
||||d| _t|dd| _t
|d |||d| _t | _d S )N)rP   rQ   rR   rS      )rE   r/   )r   r   int
group_sizecausalshift_tokensrotary_pos_embr   rN   rS   rI   	to_hiddento_qkr>   qk_offset_scaleto_outSigmoidgateActivate)r   r   r[   query_key_dimexpansion_factorr\   rS   r^   rR   r]   
hidden_dimr   r   r   r   S   s6   

zMossFormerBlock.__init__c              	   C   s   |}| j r|jddd\}}tj|ddd}tj||fdd}| |jddd\}}| |}| |\}}	}
}| 	|||	|
|||\}}|| | 
||  }|| | }|S )Nr/   r    r5   )r   r   r   r            value)r]   chunkFpadr   r8   r_   r`   ra   cal_attentionrd   rb   )r   r%   normed_xx_shiftx_passvuqkquad_qlin_qquad_klin_katt_vatt_urH   r   r   r   r&   |   s   
zMossFormerBlock.forwardc	                    s  |j d |j d |jjf\}	 }
}ddlm t|r(|d}|| d}tjr<tjj	||||f\}}}}t
 |dkrrtfdd||||||f\}}}}}}t|tj|	 f|
tjd}tj|dfd	d
}tfdd||||||f\}}}}}}t|r|d|d}td||| }t|d }|}t|r|| d}jrtj||ftj|
dd}||d}td||}td||}jrtd||| }|jdd}tj|ddd
}td||}td||| }|jdd}tj|ddd
}td||}ntd||  }td||}td||  }td||}t fdd||f\}}t fdd||f\}}|| || fS )Nr   rF   )	rearrangez... -> ... 1rh   c                    s   t j| ddd fddS )Nr   rh   ri   )rl   rm   r;   )paddingr   r   <lambda>   s    z/MossFormerBlock.cal_attention.<locals>.<lambda>)r4   dtypeFri   c                    s    | dj dS )Nzb (g n) d -> b g n d)r   )r[   r|   )r{   r   r   r   r~      s    zb (g j) -> b g 1 j)jz... i d, ... j d -> ... i jr/   )r   r4   r   z... i j, ... j d -> ... i dzb g n d, b g n e -> b g d er5   )r   r   r   r   r   r    zb g d e, b g n d -> b g n ezb g n d, b g n e -> b d ezb g n d, b d e -> b g n ec                       | dd d d  f S Nzb g n d -> b (g n) dr   r|   r   r{   r   r   r~          c                    r   r   r   r|   r   r   r   r~      r   )r6   r4   r[   einopsr{   r	   masked_fillr^   maprotate_queries_or_keysr   r   r   r   boolrl   rm   r   relurS   r\   triucumsum)r   r%   ru   rv   rw   rx   rr   rs   maskbr4   r   lin_masksimattncausal_mask
quad_out_v
quad_out_ulin_kv	lin_out_vlin_ku	lin_out_uquad_attn_out_vlin_attn_out_vquad_attn_out_ulin_attn_out_ur   )r   r}   r{   r   r   rn      s|   $





zMossFormerBlock.cal_attentionr   )	r(   r)   r*   r   rU   r   r&   rn   r+   r   r   r   r   rV   Q   s    )rV   c                       s4   e Zd Z							d fdd		Zd
d Z  ZS )MossFormerModulerW   rX         @FrJ   	scalenormTc
                    s   t    |dv sJ d|dkrtn|dkrtjddlm}
 |
tddt f	d	d
t	|D | _
d S )N)r   	layernormz/norm_type must be one of scalenorm or layernormr   r   r   )RotaryEmbedding    r5   c                    s&   g | ]}t  d 	qS ))	r   r[   re   rf   r\   rS   r^   rR   r]   )rV   ).0_	attn_dropoutr\   r   rf   r[   rR   re   r^   r]   r   r   
<listcomp>   s    
z-MossFormerModule.__init__.<locals>.<listcomp>)r   r   r   r   rU   rotary_embedding_torchr   r"   
ModuleListrangelayers)r   r   depthr[   re   rf   r\   r   	norm_typer]   r   r   r   r   r      s   



zMossFormerModule.__init__c                 C   s   | j D ]}||}q|S r   )r   )r   r%   mossformer_layerr   r   r   r&     s   

zMossFormerModule.forward)rW   rX   r   FrJ   r   Tr'   r   r   r   r   r      s    $r   )r   torch.nn.functionalr   
functionalrl   r   9modelscope.models.audio.separation.mossformer_conv_moduler   r	   r   r   Moduler   r-   r>   rI   rV   r   r   r   r   r   <module>   s    