o
    8wiI                     @  s   d dl mZ d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	 ddl
mZmZmZmZmZ dd	lmZmZmZmZmZmZmZ G d
d deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZdS )    )annotations)AnyN)is_xpu_available)nn)	transpose   )DoraConv1dLayerDoraConv2dLayerDoraConv3dLayerDoraEmbeddingLayerDoraLinearLayer)Conv1dConv2dConv3d	EmbeddingLinearLoraVariant_ConvNdc                   @  R   e Zd Zedd	d
ZedddZedddZedddZedddZdS )DoraLinearVariantmoduler   adapter_namestrkwargsr   returnNonec                 K  s   | j s| jd d  d | _tt| ddd}| j| j}| j| j}| jo0|jj	dkp0|jj	dk}| jr[|jj	dv rA|
|j}n|jj	dvrUt rP|
d}n|
d}|
|j}| j| }|j|  ||||d	 || j |< d S )
Nlora_magnitude_vectorfan_in_fan_outFr   cpu)cudaxpur"   r!   )
base_layerlora_Alora_Bscalingplace_on_cpu)r   adapter_layer_namesr   getattrr$   weightr%   ephemeral_gpu_offloaddevicetypetor   r&   update_layerget_base_layer)r   r   r   
dora_layerr$   r%   r'   r&    r2   V/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/lora/variants.pyinit   s.   

zDoraLinearVariant.initactive_adapterorig_weighttorch.Tensorc                 C  s   |j }| |}| j| j|t|| jdd }| | d| | j| j| }t|	dd| j}|||  }|
|}|S Nr   r&   -weight_norm)dtypeget_delta_weightr   get_weight_normr   r   detach_cache_storer*   viewr.   r   r5   r6   
orig_dtypedelta_weightweight_normdora_factor
new_weightr2   r2   r3   
merge_safe;   s   

zDoraLinearVariant.merge_safec                 C  s   |j }| |}| j| j|t|| jdd }| | d| | j| j| }t|	dd| j}||j
|  }||}||_
d S r8   )r<   r=   r   r>   r   r   r?   r@   r*   rA   datar.   rB   r2   r2   r3   merge_unsafeP   s   


zDoraLinearVariant.merge_unsafec                 C  T   |j }| |}| | d}| j| j| }|j|dd | }||}|S )Nr:   r;   r   r<   r=   
_cache_popr   r*   rI   rA   r.   rB   r2   r2   r3   unmergec      

zDoraLinearVariant.unmergexresultc           	   	   C  r   | j | }| j| }| j| }| j| }t|tjs| js |}n||}d }|| j| ||||| 	 |d }|S N)r$   r%   r&   r#   base_result
r$   r%   lora_dropoutr&   
isinstancer   Identitytrainingr   r0   	r   r5   rP   rQ   r$   r%   dropoutr&   rT   r2   r2   r3   forwardm   "   




zDoraLinearVariant.forwardNr   r   r   r   r   r   r   r   r   r   r5   r   r6   r7   r   r7   r   r   r5   r   r6   r7   r   r   
r   r   r5   r   rP   r7   rQ   r7   r   r7   	__name__
__module____qualname__staticmethodr4   rH   rJ   rN   r\   r2   r2   r2   r3   r      s    	r   c                   @  r   )DoraEmbeddingVariantr   r   r   r   r   r   r   r   c                 K  sj   | j d u r| jd d  d | _tdd}| j| }| j| }| j| }|j|  |||d || j |< d S )Nr   Tr   r#   r$   r%   r&   )r   r(   r   lora_embedding_Alora_embedding_Br&   r/   r0   )r   r   r   r1   ri   rj   r&   r2   r2   r3   r4      s   




zDoraEmbeddingVariant.initr5   r6   r7   c                 C  st   |j }| |}| j| j||jdd }| | d| | j| j| }|dd}|||  }|	|}|S r8   )
r<   r=   r   r>   Tr?   r@   r*   rA   r.   rB   r2   r2   r3   rH      s   


zDoraEmbeddingVariant.merge_safec                 C  s|   |j }| |}| j| j||jdd }| | d| | j| j| }|dd}||j	|  }|
|}||_	d S r8   )r<   r=   r   r>   rk   r?   r@   r*   rA   rI   r.   rB   r2   r2   r3   rJ      s   



z!DoraEmbeddingVariant.merge_unsafec                 C  rK   )Nr:   r   r;   rL   rB   r2   r2   r3   rN      rO   zDoraEmbeddingVariant.unmergerP   rQ   c           	      C  sV   | j | j}| j| j}| j| }| j| |||||  | jd\}}|| | }|S )N)r$   r%   r&   r#   embed_fn)ri   rk   rj   r&   r   r0   _embed)	r   r5   rP   rQ   embedding_Aembedding_Br&   mag_norm_scaledora_resultr2   r2   r3   r\      s   

zDoraEmbeddingVariant.forwardN)r   r   r   r   r   r   r   r   )r   r   r5   r   r6   r7   r   r7   )r   r   r5   r   r6   r7   r   r   )
r   r   r5   r   rP   r7   rQ   r7   r   r7   rb   r2   r2   r2   r3   rg      s    	rg   c                   @  r   )_DoraConvNdVariantr   r   r   r   r1   	nn.Moduler   r   c                 C  sd   | j d u r| jd d  d | _| j| j}| j| j}| j| }|j|  |||d || j |< d S )Nr   rh   )r   r(   r$   r*   r%   r&   r/   r0   )r   r   r1   r$   r%   r&   r2   r2   r3   init_convd_variant   s   

z%_DoraConvNdVariant.init_convd_variantr5   r6   r7   c                 C  sp   |j }| |}| j| j||dd }| | d| | j| j| }|j|   ||  }|	|}|S Nr   r9   r:   )
r<   r=   r   r>   r?   r@   r*   rA   _get_dora_factor_viewr.   rB   r2   r2   r3   rH      s   

z_DoraConvNdVariant.merge_safec                 C  sx   |j }| |}| j| j||dd }| | d| | j| j| }|j|   |j	|  }|
|}||_	d S ru   )r<   r=   r   r>   r?   r@   r*   rA   rv   rI   r.   rB   r2   r2   r3   rJ      s   


z_DoraConvNdVariant.merge_unsafec                 C  sV   |j }| |}| | d}| j| j| }|j|j|    | }||}|S )Nr:   )	r<   r=   rM   r   r*   rI   rA   rv   r.   rB   r2   r2   r3   rN   	  s   

z_DoraConvNdVariant.unmergerP   rQ   c           	   	   C  rR   rS   rU   rZ   r2   r2   r3   r\     r]   z_DoraConvNdVariant.forwardN)r   r   r   r   r1   rs   r   r   )r   r   r5   r   r6   r7   r   r7   )r   r   r5   r   r6   r7   r   r   )
r   r   r5   r   rP   r7   rQ   r7   r   r7   )	rc   rd   re   rf   rt   rH   rJ   rN   r\   r2   r2   r2   r3   rr      s    	rr   c                   @     e Zd Zedd	d
ZdS )DoraConv1dVariantr   r   r   r   r   r   r   r   c                 K     t dd}tj| ||d d S NFr   )r1   )r   rr   rt   r   r   r   r1   r2   r2   r3   r4   ,     
zDoraConv1dVariant.initN)r   r   r   r   r   r   r   r   rc   rd   re   rf   r4   r2   r2   r2   r3   rx   +      rx   c                   @  rw   )DoraConv2dVariantr   r   r   r   r   r   r   r   c                 K  ry   rz   )r	   rr   rt   r{   r2   r2   r3   r4   3  r|   zDoraConv2dVariant.initN)r   r   r   r   r   r   r   r   r}   r2   r2   r2   r3   r   2  r~   r   c                   @  rw   )DoraConv3dVariantr   r   r   r   r   r   r   r   c                 K  ry   rz   )r
   rr   rt   r{   r2   r2   r3   r4   :  r|   zDoraConv3dVariant.initN)r   r   r   r   r   r   r   r   r}   r2   r2   r2   r3   r   9  r~   r   c                   @  s`   e Zd Zedd	d
ZedddZedddZedddZedddZedddZ	dS ) QALoraLinearVariantr   r   r   r   r   r   r   r   c           	      K  s   d|vrt d| jdur$| j|d  dkr$t d| j d|d  d|d }d| jvr3| jd | _t| ds;i | _|| j|< | j| }|j}|jj}|jj	}t
j|j| j|  |d	||d
}|| j|< dS )ao  
        Initializes QALoRA specific parameters for a given adapter.

        Args:
            module (Linear): The linear module to be adapted.
            adapter_name (str): The name of the adapter.
            **kwargs: Additional keyword arguments.
                qalora_group_size (int): The size of groups for pooling. This is expected to be passed.
        qalora_group_sizezx`use_qalora=True` requires 'qalora_group_size' to be provided in kwargs. Please ensure it is passed from the LoraConfig.Nr   z1`use_qalora=True` requires `module.in_features` (z)) to bedivisible by 'qalora_group_size' ())r   F)biasr,   r<   )
ValueErrorin_featuresother_param_nameshasattrr   r$   out_featuresr*   r,   r<   r   r   )	r   r   r   r   old_lora_A_layerrr,   r<   new_lora_A_layerr2   r2   r3   r4   A  s8   




zQALoraLinearVariant.initr5   r7   c                 C     t d)Nz;QALoRA for GPTQ layers does not support 'get_delta_weight'.NotImplementedError)r   r5   r2   r2   r3   r=   n     z$QALoraLinearVariant.get_delta_weightr6   c                 C  r   )Nz5QALoRA for GPTQ layers does not support 'safe_merge'.r   r   r5   r6   r2   r2   r3   rH   r  r   zQALoraLinearVariant.merge_safec                 C  r   )Nz7QALoRA for GPTQ layers does not support 'merge_unsafe'.r   r   r2   r2   r3   rJ   v  r   z QALoraLinearVariant.merge_unsafec                 C  r   )Nz2QALoRA for GPTQ layers does not support 'unmerge'.r   r   r2   r2   r3   rN   z  r   zQALoraLinearVariant.unmergerP   rQ   c                 C  s   | j | j}| j| j}| j| }| j| }| j| }| jr(t|tj	s(||n|}	|	j
}
t|
dkr;|	d| j}n|	}|j
\}}|| }||||jdd}|| }||  |  | }t|
dkrv||
d d |df }|| S )N   r;   )dim)r$   r*   r%   rV   r&   r   rY   rW   r   rX   shapelenrA   r   meantsize)r   r5   rP   rQ   lora_A_weightlora_B_weightr[   r&   
group_size	x_dropped
orig_shapex_flat
batch_sizer   pooled_featuresx_pooledx_pooled_scaleddeltar2   r2   r3   r\   ~  s$   



zQALoraLinearVariant.forwardNr^   )r   r   r5   r   r   r7   r_   r`   ra   )
rc   rd   re   rf   r4   r=   rH   rJ   rN   r\   r2   r2   r2   r3   r   @  s    ,r   )
__future__r   typingr   torchaccelerate.utils.importsr   r   peft.utils.otherr   dorar   r	   r
   r   r   layerr   r   r   r   r   r   r   r   rg   rr   rx   r   r   r   r2   r2   r2   r3   <module>   s   $iTR