o
    Ni !                     @   s   d dl mZ d dlZd dlm  mZ d dlmZ d dlmZm	Z	 d dl
mZ G dd dejZG dd	 d	eZG d
d deZG dd deZG dd deZG dd deZdS )    )deepcopyN)nn)dequantize_module_weightgather_params_ctx)	transposec                       s\   e Zd Z fddZdejfddZdddd	d
ZddddZde	f fddZ
  ZS )DoraLinearLayerc                    s   t    || _d S N)super__init__fan_in_fan_outselfr   	__class__ I/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/lora/dora.pyr
      s   

zDoraLinearLayer.__init__returnc                 C   s4   t || j}|||  }tjj|dd|j}|S )N   )dim)r   r   torchlinalgnormtodtype)r   weightlora_weightscalingweight_normr   r   r   get_weight_norm   s   zDoraLinearLayer.get_weight_normF)place_on_cpuNc                C   s   |j tjk}|r| }| }t| M |jjdkr!t|}t	|}|j
jdkrG|jd }t|d|g||dg}	|	|j}	n|| }	|rQ|	 }	| ||j|	|}
W d    n1 sfw   Y  |rr|
d}
tj|
dd| _d S )N
Linear4bit   r   cpuT)requires_grad)r   r   float16floatr   
parametersr   __name__r   r   datandimshapemmviewreshapehalfr   r   devicer   	Parameterr   )r   
base_layerlora_Alora_Br   r   dtype_is_fp16r   rr   r   r   r   r   update_layer%   s(   
 
zDoraLinearLayer.update_layerbase_resultc                C   s   t j|jjd |jj|jd}|||j}| j}	t|}
|
|j}
| 	|
|
 |}|
 }|	| dd}|||}d}|durP|j}|durO|| }n
t|t|
| j}|d | || |  }|S )
        For DoRA, calculate the extra output from LoRA with DoRA applied. This should be added on top of the base layer
        output.
        r   )r0   r   r"   N)r   eyer   r+   r0   r   Tr   r   r   detachr-   biasFlinearr   r   )r   xr3   r4   r   r2   r9   x_eyer   	magnituder   r   mag_norm_scalelora_resultr>   result_dorar   r   r   forwardB   s$   zDoraLinearLayer.forwardc                       t   }d| S Nz
lora.dora.r	   __repr__r   repr   r   r   rK   g      
zDoraLinearLayer.__repr__)r   N)r(   
__module____qualname__r
   r   Tensorr   r7   rG   strrK   __classcell__r   r   r   r   r      s    %r   c                       s*   e Zd Zdd Zdef fddZ  ZS )DoraEmbeddingLayerc                C   sV   || j }| j}|j}	| |	| |}
|
 }
||
 }|||||  | }||fS )r:   )r<   r   r   r=   )r   rA   r3   r4   r   r2   embed_fnr   rC   r   r   rD   rF   r   r   r   rG   m   s   
zDoraEmbeddingLayer.forwardr   c                    rH   rI   rJ   rL   r   r   r   rK      rN   zDoraEmbeddingLayer.__repr__)r(   rO   rP   rG   rR   rK   rS   r   r   r   r   rT   l   s    rT   c                       s@   e Zd ZdejfddZddddZdef fdd	Z  Z	S )
_DoraConvNdLayerr   c                 C   s:   |||  }t td| }|jd|dddd}|S )Nr      T)pr   keepdimr   )tupleranger   r   r   )r   r   r   r   r   r   r   r   r   r      s   z _DoraConvNdLayer.get_weight_normNr8   c             	   C   s   |j }|j jd }t|j d|g|j |dg}	|	|j}	| j }
| ||	 |}| }|
| }|du rJ| j||d|j	|j
|j|jd}n|j}|durbdd| d   }||j|  }|d | |||| |  }|S )	r:   r   r"   N)r>   stridepaddingdilationgroups)r   r"   )r   rW   r   )r   r+   r   r,   r-   r.   r   r=   conv_fnr\   r]   r^   r_   r>   r   )r   rA   r3   r4   r   r2   r9   r   r6   r   rC   r   rD   r>   
bias_shaperF   r   r   r   rG      s0   $
 z_DoraConvNdLayer.forwardc                    rH   rI   rJ   rL   r   r   r   rK      rN   z_DoraConvNdLayer.__repr__)
r(   rO   rP   r   rQ   r   rG   rR   rK   rS   r   r   r   r   rV      s    (rV   c                          e Zd Z fddZ  ZS )DoraConv1dLayerc                       t  | tj| _d S r   )r	   r
   r?   conv1dr`   r   r   r   r   r
         zDoraConv1dLayer.__init__r(   rO   rP   r
   rS   r   r   r   r   rc          rc   c                       rb   )DoraConv2dLayerc                    rd   r   )r	   r
   r?   conv2dr`   r   r   r   r   r
      rf   zDoraConv2dLayer.__init__rg   r   r   r   r   ri      rh   ri   c                       rb   )DoraConv3dLayerc                    rd   r   )r	   r
   r?   conv3dr`   r   r   r   r   r
      rf   zDoraConv3dLayer.__init__rg   r   r   r   r   rk      rh   rk   )copyr   r   torch.nn.functionalr   
functionalr?   peft.utils.integrationsr   r   peft.utils.otherr   Moduler   rT   rV   rc   ri   rk   r   r   r   r   <module>   s   S6