o
    8wig#                     @  s   d dl mZ d dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZ G dd	 d	ZG d
d deZG dd deZG dd deZdS )    )annotations)AnyCallableOptionalN)Tensor)lora   )XLoraConfigc                   @  s6   e Zd ZdZdddZ	 edddZ	 dddZdS )
XLoraLayerz
    A XLoraLayer wraps any LoraLayer and performs the XLora operation on the LoRA adaptors specified. Its primary API
    is the forward method, which uses the scalings to execute the XLora algorithm.
    model	nn.Moduletargetlora.LoraLayertarget_forwardCallable[..., Any]layer_numberintconfigr	   returnNonec                 C  s"   || _ || _|| _|| _|| _d S N)r   r   r   r   r   selfr   r   r   r   r    r   T/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/xlora/layer.py__init__!   s
   
zXLoraLayer.__init__xtorch.Tensorscalings_layeradapterc                 C  s$   |d d d d |f  d}| | S )N)	unsqueeze)r   r   r   scalingsr   r   r   apply_scalings_to_x3   s   zXLoraLayer.apply_scalings_to_xc                 C  s   |d d d d | j d d f }| jjd ur8tj|| jjdd\}}tj|tjd}|d|d |||j	 }| jj
rM|dk}tj|| dd}|||< |S )Nr    )kdim)dtypeTr   )r%   )r   r   
top_k_loratorchtopk
zeros_likeboolscatter_tor&   enable_softmax_topksoftmax)r   r"   xlora_scalings_topk_indicesmasknonzero_masksoftmax_res_nonzeror   r   r   get_maybe_topk_scalings?   s   z"XLoraLayer.get_maybe_topk_scalingsN)r   r   r   r   r   r   r   r   r   r	   r   r   )r   r   r   r   r   r   r   r   )r   r   )__name__
__module____qualname____doc__r   staticmethodr#   r6   r   r   r   r   r
      s    
r
   c                      .   e Zd Zd fddZdddddZ  ZS )XLoraLinearLayerr   r   r   lora.Linearr   r   r   r   r   r	   r   r   c                      t  ||||| d S r   superr   r   	__class__r   r   r   U      zXLoraLinearLayer.__init__Nr"   r   r   argsr   r"   Optional[Tensor]kwargsc                O    |j }|dur| |}| jj|g|R i |}| jjs|t| jjD ]X\}}	|	| jj vr0q#| jj	|	 r:t
d| jj|	 }
| jj|	 }| jj|	 }| jj|	 }||
jj }|duri| |||}| jj}n|}d}|||
||| | 7 }q#||}|S 
        This method is designed to be a drop-in-replacement for the LoRA layers' .forward method. To use it, a bound
        method must be created (bound to an instance of the XLoraLayer class).
        N7X-LoRA currently does not support LoRA layers with DoRAr   r&   r6   r   
base_layermerged	enumerateactive_adapterslora_Akeysuse_dora
ValueErrorlora_Blora_dropoutscalingr-   weightr#   r   global_scaling_weightr   r   r"   rF   rH   previous_dtyper0   result	adapter_nactive_adapterrR   rV   dropoutrX   x_modscaling_weightr   r   r   forward_   s.   


zXLoraLinearLayer.forward)r   r   r   r>   r   r   r   r   r   r	   r   r   
r   r   rF   r   r"   rG   rH   r   r   r   r7   r8   r9   r   rc   __classcell__r   r   rB   r   r=   T       
r=   c                      r<   )XLoraEmbeddingLayerr   r   r   lora.Embeddingr   r   r   r   r   r	   r   r   c                   r?   r   r@   r   rB   r   r   r      rD   zXLoraEmbeddingLayer.__init__NrE   r   r   rF   r   r"   rG   rH   c                O  s   |dur	|  |}| jj|g|R i |}| jjsqt| jjD ]P\}}|| jjvr+q | jj|dr7t	d| jj| j
}	| jj| j
}
| jj| }| j||	}|durb| |||}| jj}n|}d}|||
 | | 7 }q |S )rK   NFrL   r   )r6   r   rN   rO   rP   rQ   lora_embedding_ArT   getrU   Tlora_embedding_BrX   _embedr#   r   rZ   )r   r   r"   rF   rH   r0   r]   r^   r_   embedding_Aembedding_BrX   after_Aafter_A_modrb   r   r   r   rc      s(   

zXLoraEmbeddingLayer.forward)r   r   r   ri   r   r   r   r   r   r	   r   r   rd   re   r   r   rB   r   rh      rg   rh   c                      r<   )XLoraConv2dLayerr   r   r   lora.Conv2dr   r   r   r   r   r	   r   r   c                   r?   r   r@   r   rB   r   r   r      rD   zXLoraConv2dLayer.__init__NrE   r   r   rF   r   r"   rG   rH   c                O  rI   rJ   rM   r[   r   r   r   rc      s.   


zXLoraConv2dLayer.forward)r   r   r   rt   r   r   r   r   r   r	   r   r   rd   re   r   r   rB   r   rs      rg   rs   )
__future__r   typingr   r   r   r(   torch.nnnnr   peft.tunersr   r   r	   r
   r=   rh   rs   r   r   r   r   <module>   s   90-