o
    i1                     @   s   d dl Z d dlmZ d dlm  mZ d dlZd dlmZm	Z	 G dd dZ
G dd deje
ZG dd deje
ZG d	d
 d
eje
ZG dd deje
ZdS )    N)OptionalListc                   @   s&   e Zd ZdedededefddZdS )	LoRALayerr
lora_alphalora_dropoutmerge_weightsc                 C   s>   || _ || _|dkrtj|d| _ndd | _d| _|| _d S )N        )pc                 S   s   | S N )xr   r   M/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/lora/layers.py<lambda>   s    z$LoRALayer.__init__.<locals>.<lambda>F)r   r   nnDropoutr   mergedr   )selfr   r   r   r   r   r   r   __init__   s   

zLoRALayer.__init__N)__name__
__module____qualname__intfloatboolr   r   r   r   r   r      s    r   c                   @   sb   e Zd Z			ddededededef
d	d
Zdd ZddefddZdd Zde	j
fddZdS )	Embeddingr      Tnum_embeddingsembedding_dimr   r   r   c                 K   s   t jj| ||fi | tj| ||d|d |dkr=t | j||f| _t | j||f| _| j	| j
 | _d| j_|   d S )Nr   r   r   r   r   F)r   r   r   r   	Parameterweight	new_zeroslora_Alora_Br   r   scalingrequires_gradreset_parameters)r   r   r   r   r   r   kwargsr   r   r   r   #   s   	
zEmbedding.__init__c                 C   s:   t j|  t| drt j| j t j| j d S d S )Nr#   )	r   r   r'   hasattrinitzeros_r#   normal_r$   r   r   r   r   r'   9   s
   
zEmbedding.reset_parametersmodec                 C   sV   t j| | | jr'| jr)| jdkr"| j j| j| j	 j
| j 8  _d| _d S d S d S )Nr   F)r   r   trainr   r   r   r!   datar$   r#   Tr%   r   r.   r   r   r   r/   @   s   
 
zEmbedding.trainc                 C   sR   t j|  | jr%| js'| jdkr | j j| j| j	 | j
 7  _d| _d S d S d S )Nr   Tr   Linearevalr   r   r   r!   r0   r$   r#   r%   r-   r   r   r   r5   H   s   

zEmbedding.evalr   c              	   C   st   | j dkr3| js3tj| |}| j dkr1t|| jj| j	| j
| j| j| j}||| jj | j 7 }|S tj| |S Nr   )r   r   r   r   forwardF	embeddingr#   r1   padding_idxmax_norm	norm_typescale_grad_by_freqsparser$   r%   )r   r   resultafter_Ar   r   r   r7   P   s   
	zEmbedding.forwardN)r   r   TT)r   r   r   r   r   r   r'   r/   r5   torchTensorr7   r   r   r   r   r   !   s&    
r   c                   @   sn   e Zd Z					ddededed	ed
ededefddZdd ZddefddZdd Z	de
jfddZdS )r4   r   r   r	   FTin_featuresout_featuresr   r   r   fan_in_fan_outr   c           	      K   s   t jj| ||fi | tj| ||||d || _|dkr@t | j||f| _t | j||f| _	| j
| j | _d| j_|   |rO| jjj| j_d S d S Nr   r   F)r   r4   r   r   rF   r    r!   r"   r#   r$   r   r   r%   r&   r'   r0   r1   )	r   rD   rE   r   r   r   rF   r   r(   r   r   r   r   e   s   
zLinear.__init__c                 C   D   t j|  t| dr t jj| jtdd t j	| j
 d S d S Nr#      )ar   r4   r'   r)   r*   kaiming_uniform_r#   mathsqrtr+   r$   r-   r   r   r   r'      
   
zLinear.reset_parametersr.   c                    sd    fdd}t j |  jr. jr0 jdkr) j j| j j	  j
 8  _d _d S d S d S )Nc                        j r| jS | S r   rF   r1   wr-   r   r   r1         zLinear.train.<locals>.Tr   F)r   r4   r/   r   r   r   r!   r0   r$   r#   r%   )r   r.   r1   r   r-   r   r/      s   
"
zLinear.trainc                    sb    fdd}t j   jr- js/ jdkr( j j| j j	  j
 7  _d _d S d S d S )Nc                    rQ   r   rR   rS   r-   r   r   r1      rU   zLinear.eval.<locals>.Tr   Tr3   )r   r1   r   r-   r   r5      s   
"
zLinear.evalr   c                    s~    fdd} j dkr3 js3tj|| j jd} j dkr1| | jj  j	j  j
 7 }|S tj|| j jdS )Nc                    rQ   r   rR   rS   r-   r   r   r1      rU   zLinear.forward.<locals>.Tr   bias)r   r   r8   linearr!   rW   r   r#   r1   r$   r%   )r   r   r1   r?   r   r-   r   r7      s   
$zLinear.forwardN)r   r   r	   FTrA   r   r   r   r   r   r   r   r'   r/   r5   rB   rC   r7   r   r   r   r   r4   c   s2    
r4   c                   @   s   e Zd Zddddgddfdededed	ed
edee dedefddZdd Zdd Z	ddefddZ
dd ZdejfddZdS )MergedLinearr   r   r	   FTrD   rE   r   r   r   enable_lorarF   r   c	           
      K   s"  t jj| ||fi |	 tj| ||||d |t| dks"J d|| _|| _|dkrt|rt | j	
|t| |f| _t | j	
|t| t| |f| _| j| j | _d| j	_| j	j
|ftjdt|d| _d| j|d d f< | jd| _|   |r| j	jj| j	_d S d S )Nr   r   z2The length of enable_lora must divide out_featuresF)dtypeT)r   r4   r   r   lenr[   rF   anyr    r!   r"   sumr#   r$   r   r   r%   r&   rB   r   viewlora_indr'   r0   r1   )
r   rD   rE   r   r   r   r[   rF   r   r(   r   r   r   r      s2   
 zMergedLinear.__init__c                 C   rH   rI   rL   r-   r   r   r   r'      rP   zMergedLinear.reset_parametersc                 C   s~   | g |jd d | jR }|d| j}|d| jt| j t| j |d d | jf< |g |jd d | jR S )Nr]   )	r"   shaperE   ra   reshaper^   r[   r`   rb   )r   r   r?   r   r   r   zero_pad   s     zMergedLinear.zero_padr.   c                    s    fdd}t j |  jrJ jrL jdkrEt jrEtj	 j
jd jjdt jdd} j j || j 8  _d _d S d S d S )Nc                    rQ   r   rR   rS   r-   r   r   r1      rU   zMergedLinear.train.<locals>.Tr   r]   groupsF)r   r4   r/   r   r   r   r_   r[   r8   conv1dr#   r0   	unsqueezer$   r`   squeezer!   re   r%   )r   r.   r1   delta_wr   r-   r   r/      s    
zMergedLinear.trainc                    s    fdd}t j   jrI jsK jdkrDt jrDtj	 j
jd jjdt jdd} j j || j 7  _d _d S d S d S )Nc                    rQ   r   rR   rS   r-   r   r   r1      rU   zMergedLinear.eval.<locals>.Tr   r]   rf   T)r   r4   r5   r   r   r   r_   r[   r8   rh   r#   r0   ri   r$   r`   rj   r!   re   r%   )r   r1   rk   r   r-   r   r5      s    
zMergedLinear.evalr   c                    s    fdd} j rtj|| j jdS tj|| j jd} jdkrQt | j}tj|	dd j
dt jd	dd}| | j 7 }|S )Nc                    rQ   r   rR   rS   r-   r   r   r1     rU   zMergedLinear.forward.<locals>.TrV   r   r]   rf   )r   r8   rX   r!   rW   r   r   r#   rh   	transposer$   ri   r`   r[   re   r%   )r   r   r1   r?   r@   after_Br   r-   r   r7     s   


zMergedLinear.forwardNrA   )r   r   r   r   r   r   r   r   r'   re   r/   r5   rB   rC   r7   r   r   r   r   rZ      s:    	
(rZ   c                   @   sl   e Zd Z				ddedededed	ed
edefddZdd ZddefddZdd Z	de
jfddZdS )Conv2dr   r   r	   Tin_channelsout_channelskernel_sizer   r   r   r   c           	      K   s   t jj| |||fi | tj| ||||d t|tu sJ |dkrNt | j|| || f| _	t | j|| || f| _
| j| j | _d| j_|   d S rG   )r   ro   r   r   typer   r    r!   r"   r#   r$   r   r   r%   r&   r'   )	r   rp   rq   rr   r   r   r   r   r(   r   r   r   r     s   
zConv2d.__init__c                 C   rH   rI   )r   ro   r'   r)   r*   rM   r#   rN   rO   r+   r$   r-   r   r   r   r'   6  rP   zConv2d.reset_parametersr.   c                 C   sT   t j| | | jr&| jr(| j j| j| j 	| jj
| j 8  _d| _d S d S d S )NF)r   ro   r/   r   r   r!   r0   r$   r#   ra   rc   r%   r2   r   r   r   r/   =  s
   (
zConv2d.trainc                 C   sR   t j|  | jr%| js'| j j| j| j 	| jj
| j 7  _d| _d S d S d S )NT)r   ro   r5   r   r   r!   r0   r$   r#   ra   rc   r%   r-   r   r   r   r5   D  s
   (
zConv2d.evalr   c              	   C   s\   | j dkr'| js't|| j| j| j | jj| j	  | j
| j| j| j| jS tj| |S r6   )r   r   r8   conv2dr!   r$   r#   ra   rc   r%   rW   stridepaddingdilationrg   r   ro   r7   )r   r   r   r   r   r7   K  s    	zConv2d.forwardN)r   r   r	   TrA   rY   r   r   r   r   ro     s0    
ro   )rB   torch.nnr   torch.nn.functional
functionalr8   rN   typingr   r   r   r   r4   rZ   ro   r   r   r   r   <module>   s   BHl