o
    Ni&                     @  s   d dl mZ d dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZ G dd	 d	ZG d
d deZG dd deZG dd deZdS )    )annotations)AnyCallableOptionalN)Tensor)lora   )XLoraConfigc                   @  s6   e Zd ZdZdddZ	 edddZ	 dddZdS )
XLoraLayerz
    A XLoraLayer wraps any LoraLayer and performs the XLora operation on the LoRA adaptors specified. Its primary API
    is the forward method, which uses the scalings to execute the XLora algorithm.
    model	nn.Moduletargetlora.LoraLayertarget_forwardCallable[..., Any]layer_numberintconfigr	   returnNonec                 C  s"   || _ || _|| _|| _|| _d S N)r   r   r   r   r   selfr   r   r   r   r    r   K/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/xlora/layer.py__init__!   s
   
zXLoraLayer.__init__xtorch.Tensorscalings_layeradapterc                 C  s$   |d d d d |f  d}| | S )N)	unsqueeze)r   r   r   scalingsr   r   r   apply_scalings_to_x3   s   zXLoraLayer.apply_scalings_to_xc           	      C  s   |d d d d | j d d f }| jjd ur8tj|| jjdd\}}tj|tjd}|d|d |||j	 }| jj
rW|dk}|| td}tj|dd}|| d}|S )	Nr    )kdim)dtypeTr   z-inf)r%   g        )r   r   
top_k_loratorchtopk
zeros_likeboolscatter_tor&   enable_softmax_topkmasked_fillfloatsoftmax)	r   r"   xlora_scalings_topk_indicesmasknonzero_maskfullnew_scalingsr   r   r   get_maybe_topk_scalings?   s   z"XLoraLayer.get_maybe_topk_scalingsN)r   r   r   r   r   r   r   r   r   r	   r   r   )r   r   r   r   r   r   r   r   )r   r   )__name__
__module____qualname____doc__r   staticmethodr#   r9   r   r   r   r   r
      s    
r
   c                      .   e Zd Zd fddZdddddZ  ZS )XLoraLinearLayerr   r   r   lora.Linearr   r   r   r   r   r	   r   r   c                      t  ||||| d S r   superr   r   	__class__r   r   r   W      zXLoraLinearLayer.__init__Nr"   r   r   argsr   r"   Optional[Tensor]kwargsc                O    |j }|dur| |}| jj|g|R i |}| jjs|t| jjD ]X\}}	|	| jj vr0q#| jj	|	 r:t
d| jj|	 }
| jj|	 }| jj|	 }| jj|	 }||
jj }|duri| |||}| jj}n|}d}|||
||| | 7 }q#||}|S 
        This method is designed to be a drop-in-replacement for the LoRA layers' .forward method. To use it, a bound
        method must be created (bound to an instance of the XLoraLayer class).
        N7X-LoRA currently does not support LoRA layers with DoRAr   r&   r9   r   
base_layermerged	enumerateactive_adapterslora_Akeysuse_dora
ValueErrorlora_Blora_dropoutscalingr-   weightr#   r   global_scaling_weightr   r   r"   rI   rK   previous_dtyper2   result	adapter_nactive_adapterrU   rY   dropoutr[   x_modscaling_weightr   r   r   forwarda   s.   


zXLoraLinearLayer.forward)r   r   r   rA   r   r   r   r   r   r	   r   r   
r   r   rI   r   r"   rJ   rK   r   r   r   r:   r;   r<   r   rf   __classcell__r   r   rE   r   r@   V       
r@   c                      r?   )XLoraEmbeddingLayerr   r   r   lora.Embeddingr   r   r   r   r   r	   r   r   c                   rB   r   rC   r   rE   r   r   r      rG   zXLoraEmbeddingLayer.__init__NrH   r   r   rI   r   r"   rJ   rK   c                O  s  |dur	|  |}| jj|g|R i |}| j }| jjst| jjD ]^\}}	|	| jjvr0q%| jj	|	dr<t
d| jj|	 j}
| jj|	 j}| jj|	 }| j||
}|durg| |||}| jj}n|}d}|| | | }|dur|||j }||7 }q%|S )rN   NFrO   r   )r9   r   rQ   _get_embed_scalerR   rS   rT   lora_embedding_ArW   getrX   Tlora_embedding_Br[   _embedr#   r   r]   r-   r&   )r   r   r"   rI   rK   r2   r`   embed_scalera   rb   embedding_Aembedding_Br[   after_Aafter_A_modre   adapter_outputr   r   r   rf      s0   



zXLoraEmbeddingLayer.forward)r   r   r   rl   r   r   r   r   r   r	   r   r   rg   rh   r   r   rE   r   rk      rj   rk   c                      r?   )XLoraConv2dLayerr   r   r   lora.Conv2dr   r   r   r   r   r	   r   r   c                   rB   r   rC   r   rE   r   r   r      rG   zXLoraConv2dLayer.__init__NrH   r   r   rI   r   r"   rJ   rK   c                O  rL   rM   rP   r^   r   r   r   rf      s.   


zXLoraConv2dLayer.forward)r   r   r   rz   r   r   r   r   r   r	   r   r   rg   rh   r   r   rE   r   ry      rj   ry   )
__future__r   typingr   r   r   r(   torch.nnnnr   peft.tunersr   r   r	   r
   r@   rk   ry   r   r   r   r   <module>   s   ;08