o
    ߥi0                     @   s   d dl Z d dlmZ d dlm  mZ d dlZd dlmZm	Z	 G dd dZ
G dd deje
ZG dd deje
ZG d	d
 d
eje
ZG dd deje
ZG dd deZG dd deZG dd deZdS )    N)OptionalListc                   @   s&   e Zd ZdedededefddZdS )	LoRALayerr
lora_alphalora_dropoutmerge_weightsc                 C   s>   || _ || _|dkrtj|d| _ndd | _d| _|| _d S )N        )pc                 S   s   | S N )xr   r   B/home/ubuntu/.local/lib/python3.10/site-packages/loralib/layers.py<lambda>   s    z$LoRALayer.__init__.<locals>.<lambda>F)r   r   nnDropoutr   mergedr   )selfr   r   r   r   r   r   r   __init__   s   

zLoRALayer.__init__N)__name__
__module____qualname__intfloatboolr   r   r   r   r   r      s    r   c                   @   sZ   e Zd Z			ddededededef
d	d
Zdd ZddefddZdej	fddZ
dS )	Embeddingr      Tnum_embeddingsembedding_dimr   r   r   c                 K   s   t jj| ||fi | tj| ||d|d |dkr=t | j||f| _t | j||f| _| j	| j
 | _d| j_|   d S )Nr   r   r   r   r   F)r   r   r   r   	Parameterweight	new_zeroslora_Alora_Br   r   scalingrequires_gradreset_parameters)r   r   r   r   r   r   kwargsr   r   r   r   "   s   	zEmbedding.__init__c                 C   s:   t j|  t| drt j| j t j| j d S d S )Nr#   )	r   r   r'   hasattrinitzeros_r#   normal_r$   r   r   r   r   r'   7   s
   
zEmbedding.reset_parametersmodec                 C   s   t j| | |r0| jr,| jr.| jdkr'| j j| j| j	 
dd| j 8  _d| _d S d S d S | jrS| jsU| jdkrN| j j| j| j	 
dd| j 7  _d| _d S d S d S )Nr   r   FT)r   r   trainr   r   r   r!   datar$   r#   	transposer%   r   r.   r   r   r   r/   >   s   
&

&
zEmbedding.trainr   c              	   C   sv   | j dkr4| js4tj| |}t|| jdd| j	| j
| j| j| j}||| jdd | j 7 }|S tj| |S Nr   r   )r   r   r   r   forwardF	embeddingr#   r1   padding_idxmax_norm	norm_typescale_grad_by_freqsparser$   r%   )r   r   resultafter_Ar   r   r   r4   M   s   zEmbedding.forwardN)r   r   TT)r   r   r   r   r   r   r'   r/   torchTensorr4   r   r   r   r   r       s$    
r   c                   @   sf   e Zd Z					ddededed	ed
ededefddZdd ZddefddZde	j
fddZdS )Linearr   r   r	   FTin_featuresout_featuresr   r   r   fan_in_fan_outr   c           	      K   s   t jj| ||fi | tj| ||||d || _|dkr@t | j||f| _t | j||f| _	| j
| j | _d| j_|   |rR| jjdd| j_d S d S )Nr   r   Fr   )r   rA   r   r   rD   r    r!   r"   r#   r$   r   r   r%   r&   r'   r0   r1   )	r   rB   rC   r   r   r   rD   r   r(   r   r   r   r   \   s   zLinear.__init__c                 C   D   t j|  t| dr t jj| jtdd t j	| j
 d S d S Nr#      )ar   rA   r'   r)   r*   kaiming_uniform_r#   mathsqrtr+   r$   r-   r   r   r   r'   w   
   
zLinear.reset_parametersr.   c                    s    fdd}t j | |r4 jr0 jr2 jdkr+ j j| j j	  j
 8  _d _d S d S d S  jrU jsW jdkrP j j| j j	  j
 7  _d _d S d S d S )Nc                        j r	| ddS | S r3   rD   r1   wr-   r   r   T      zLinear.train.<locals>.Tr   FT)r   rA   r/   r   r   r   r!   r0   r$   r#   r%   r   r.   rR   r   r-   r   r/   ~   s   
"

"
zLinear.trainr   c                    s    fdd} j dkr4 js4tj|| j jd}| | jdd  j	dd  j
 7 }|S tj|| j jdS )Nc                    rN   r3   rO   rP   r-   r   r   rR      rS   zLinear.forward.<locals>.Tr   biasr   )r   r   r5   linearr!   rV   r   r#   r1   r$   r%   r   r   rR   r<   r   r-   r   r4      s   0zLinear.forwardN)r   r   r	   FTr>   )r   r   r   r   r   r   r   r'   r/   r?   r@   r4   r   r   r   r   rA   Z   s0    
rA   c                   @   s   e Zd Zddddgddfdededed	ed
edee dedefddZdd Zdd Z	dd Z
ddefddZdejfddZdS )MergedLinearr   r   r	   FTrB   rC   r   r   r   enable_lorarD   r   c	           
      K   s(  t jj| ||fi |	 tj| ||||d |t| dks"J d|| _|| _|dkrt|rt | j	
|t| |f| _t | j	
|t| t| |f| _| j| j | _d| j	_| j	j
|ftjdt|d| _d| j|d d f< | jd| _|   |r| j	jdd| j	_d S d S )	Nr   r   z2The length of enable_lora must divide out_featuresF)dtypeTr   )r   rA   r   r   lenrZ   rD   anyr    r!   r"   sumr#   r$   r   r   r%   r&   r?   r   viewlora_indr'   r0   r1   )
r   rB   rC   r   r   r   rZ   rD   r   r(   r   r   r   r      s:   zMergedLinear.__init__c                 C   rE   rF   rI   r-   r   r   r   r'      rM   zMergedLinear.reset_parametersc                 C   s0   | t| jg|jdd  R }||| j< |S )Nr   )r"   r]   ra   shape)r   r   r<   r   r   r   zero_pad   s   "
zMergedLinear.zero_padc                    sF    fdd}t j jd jdt jdd}| |S )Nc                    rN   r3   rO   rP   r-   r   r   rR      rS   z MergedLinear.merge_AB.<locals>.Tr   r\   )groups)	r5   conv1dr#   	unsqueezer$   r_   rZ   squeezerc   )r   rR   delta_wr   r-   r   merge_AB   s   

zMergedLinear.merge_ABr.   c                    s    fdd}t j | |r5 jr1 jr3 jdkr,t jr, j j	 
  j 8  _	d _d S d S d S  jrW jsY jdkrRt jrR j j	 
  j 7  _	d _d S d S d S )Nc                    rN   r3   rO   rP   r-   r   r   rR      rS   zMergedLinear.train.<locals>.Tr   FT)r   rA   r/   r   r   r   r^   rZ   r!   r0   ri   r%   rT   r   r-   r   r/      s   

zMergedLinear.trainr   c                    sr    fdd} j rtj|| j jdS tj|| j jd} jdkr7| ||  j  j	 7 }|S )Nc                    rN   r3   rO   rP   r-   r   r   rR      rS   zMergedLinear.forward.<locals>.TrU   r   )
r   r5   rW   r!   rV   r   r   ri   rR   r%   rX   r   r-   r   r4      s   
"zMergedLinear.forwardNr>   )r   r   r   r   r   r   r   r   r'   rc   ri   r/   r?   r@   r4   r   r   r   r   rY      s:    	
'
rY   c                       s<   e Zd Zd fdd	Zdd Zd fd	d
	Zdd Z  ZS )ConvLoRAr   r   r	   Tc	           
         s   t t|   ||||fi |	| _tj| ||||d t|ts#J |dkrYt| jj	
|| || f| _t| jj	
|| jj | || f| _| j| j | _d| jj	_|   d| _d S )Nr   r   F)superrj   r   convr   
isinstancer   r   r    r!   r"   r#   rd   r$   r   r   r%   r&   r'   r   )
r   conv_modulein_channelsout_channelskernel_sizer   r   r   r   r(   	__class__r   r   r      s    

zConvLoRA.__init__c                 C   sB   | j   t| drtjj| jtdd tj	| j
 d S d S rF   )rl   r'   r)   r   r*   rJ   r#   rK   rL   r+   r$   r-   r   r   r   r'   	  s
   

zConvLoRA.reset_parametersc                    s   t t| | |r4| jr0| jr2| jdkr+| jj j| j	| j
 | jjj| j 8  _d| _d S d S d S | jrZ| js\| jdkrU| jj j| j	| j
 | jjj| j 7  _d| _d S d S d S )Nr   FT)rk   rj   r/   r   r   r   rl   r!   r0   r$   r#   r`   rb   r%   r2   rr   r   r   r/     s   
,

,
zConvLoRA.trainc                 C   sP   | j dkr#| js#| j|| jj| j| j | jjj| j	  | jj
S | |S )Nr   )r   r   rl   _conv_forwardr!   r$   r#   r`   rb   r%   rV   )r   r   r   r   r   r4     s   $
zConvLoRA.forward)r   r   r	   Tr>   )r   r   r   r   r'   r/   r4   __classcell__r   r   rr   r   rj      s
    rj   c                          e Zd Z fddZ  ZS )Conv2dc                    $   t t| jtjg|R i | d S r   )rk   rw   r   r   r   argsr(   rr   r   r   r   )     $zConv2d.__init__r   r   r   r   ru   r   r   rr   r   rw   (      rw   c                       rv   )Conv1dc                    rx   r   )rk   r~   r   r   ry   rr   r   r   r   -  r{   zConv1d.__init__r|   r   r   rr   r   r~   ,  r}   r~   c                       rv   )Conv3dc                    rx   r   )rk   r   r   r   ry   rr   r   r   r   3  r{   zConv3d.__init__r|   r   r   rr   r   r   2  r}   r   )r?   torch.nnr   torch.nn.functional
functionalr5   rK   typingr   r   r   r   rA   rY   Modulerj   rw   r~   r   r   r   r   r   <module>   s   :@[3