o
    8wÖiE<  ã                   @   s¶   d dl Z d dlmZ d dlZd dlmZ d dlm  mZ d dl	m
Z
 G dd„ deje
ƒZG dd„ deƒZG dd	„ d	eƒZG d
d„ dejjƒZG dd„ dejjƒZdd„ Zdd„ ZdS )é    N)ÚAny)ÚLycorisLayerc                       sÐ   e Zd ZdZdejf‡ fdd„Zedee	 fdd„ƒZ
de	d	ed
eedf fdd„Zde	fdd„Zde	fdd„Z	d de	d	ededededededdfdd„Zde	dejfdd„Zdejdejfdd„Z‡  ZS )!Ú	LoHaLayer©Ú	hada_w1_aÚ	hada_w1_bÚ	hada_w2_aÚ	hada_w2_bÚhada_t1Úhada_t2Ú
base_layerc                    sb   t ƒ  ¡  t | |¡ t i ¡| _t i ¡| _t i ¡| _t i ¡| _t i ¡| _	t i ¡| _
d S ©N)ÚsuperÚ__init__r   ÚnnÚParameterDictr   r   r   r	   r
   r   )Úselfr   ©Ú	__class__© úS/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/loha/layer.pyr      s   
zLoHaLayer.__init__Úreturnc                 C   s(   h | j £| j£| j£| j£| j£| j£S r   r   )r   r   r   r   Ú_available_adapters*   s   (zLoHaLayer._available_adaptersÚadapter_nameÚrÚshape.c              	   C   s<  t |ƒdkrdt t |||d |d ¡¡| j|< t t ||d ¡¡| j|< t t ||d ¡¡| j|< t t |||d |d ¡¡| j|< t t ||d ¡¡| j	|< t t ||d ¡¡| j
|< d S t t |d |¡¡| j|< t t ||d ¡¡| j|< t t |d |¡¡| j	|< t t ||d ¡¡| j
|< d S )Né   é   é   r   é   )Úlenr   Ú	ParameterÚtorchÚemptyr
   r   r   r   r   r	   )r   r   r   r   r   r   r   Úcreate_adapter_parameters.   s   $$  z#LoHaLayer.create_adapter_parametersc                 C   sÂ   || j  ¡ v r:tjj| j | t d¡d tjj| j| t d¡d tjj| j| t d¡d tj 	| j
| ¡ || j ¡ v r_tjj| j| t d¡d tjj| j| t d¡d d S d S ©Né   )Úa)r   Úkeysr   ÚinitÚkaiming_uniform_ÚmathÚsqrtr   r   Úzeros_r	   r
   r   ©r   r   r   r   r   Úreset_adapter_parameters?   s    þz"LoHaLayer.reset_adapter_parametersc                 C   sÌ   || j  ¡ v r?tjj| j | t d¡d tjj| j| t d¡d tjj| j| t d¡d tjj| j	| t d¡d || j
 ¡ v rdtjj| j
| t d¡d tjj| j| t d¡d d S d S r%   )r   r(   r   r)   r*   r+   r,   r   r   r	   r
   r   r.   r   r   r   Úreset_adapter_parameters_randomN   s    þz)LoHaLayer.reset_adapter_parameters_randomFÚalphaÚrank_dropoutÚmodule_dropoutÚinit_weightsÚuse_effective_conv2dNc                 K   s  |dkrt d|› ƒ‚|| j|< || j|< || | j|< || j|< || j|< |  ¡ }	t|	tj	ƒr7t
|	jjƒ}
n5t|	tjƒrb|oC|	jdk}|rQ|	j|	jg|	j¢R }
n|	j|	j|	jd  |	jd  f}
n
tdt|	ƒj› ƒ‚|  |||
¡ |r{|  |¡ n|  |¡ |  |¡ |  | j¡ dS )a  Internal function to create loha adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            alpha (`float`): Alpha for the added adapter.
            rank_dropout (`float`): The dropout probability for rank dimension during training.
            module_dropout (`float`): The dropout probability for disabling adapter during training.
            init_weights (`bool`): Whether to initialize weights.
            use_effective_conv2d (`bool`, *optional*, defaults to `False`):
                Use parameter effective decomposition for Conv2d with ksize > 1.
        r   z?`r` should be a positive integer value but the value passed is )r   r   r   z0LoHa is not implemented for base layers of type N)Ú
ValueErrorr   r1   Úscalingr2   r3   Úget_base_layerÚ
isinstancer   ÚLinearÚtupleÚweightr   ÚConv2dÚkernel_sizeÚout_channelsÚin_channelsÚ	TypeErrorÚtypeÚ__name__r$   r/   r0   Ú%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r   r   r   r1   r2   r3   r4   r5   Úkwargsr   r   r   r   r   Úupdate_layer]   s0   



þ

zLoHaLayer.update_layerc                 C   s  || j  ¡ v r+t| j | | j| | j| | j| | j| | j| t 	| j
| ¡d}nt| j| | j| | j| | j| t 	| j
| ¡d}|  ¡ }| |jj¡}| j| }| jrŠ|rŠt | d¡¡|k |j¡}|jdgdgt|jdd … ƒ ¢R Ž  |j¡}|| ¡  }||9 }|S )N)Úscaler   éÿÿÿÿr   )r
   r(   Úmake_weight_cpr   r   r   r   r	   r"   Útensorr7   Úmake_weightr8   Úreshaper<   r   r2   ÚtrainingÚrandÚsizeÚtoÚdtypeÚviewr    ÚdeviceÚmean)r   r   r<   r   r2   Údropr   r   r   Úget_delta_weightš   s4   ù
û

.zLoHaLayer.get_delta_weightÚxc                 O   sÔ   |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}nI| jr*| j|g|¢R i |¤Ž}n9| j|g|¢R i |¤Ž}| jD ])}|| jvrAq9| j| }| jrS| jrbt	 
d¡|krb|| j||g|¢R i |¤Ž }q9| |¡}|S )Nr   )rS   Údisable_adaptersÚmergedÚunmerger   rF   r   r3   rO   r"   rP   Ú_get_delta_activationsrR   )r   rY   ÚargsrG   Úprevious_dtypeÚresultÚactive_adapterr3   r   r   r   Úforward¿   s"   


€
zLoHaLayer.forward)F)rC   Ú
__module__Ú__qualname__Úadapter_layer_namesr   ÚModuler   ÚpropertyÚsetÚstrr   Úintr;   r$   r/   r0   ÚfloatÚboolrH   r"   ÚTensorrX   rb   Ú__classcell__r   r   r   r   r      s8    øþýüûúùø

ö=%r   c                       s‚   e Zd ZdZ						ddejdeded	ed
edede	f‡ fdd„Z
dedejdededejf
dd„Zdef‡ fdd„Z‡  ZS )r:   z LoHa implemented in Linear layerÚdefaultr   ç        Tr   r   r   r1   r2   r3   r4   c           	         s2   t ƒ  |¡ || _| j||||||fi |¤Ž d S r   ©r   r   Ú_active_adapterrH   )	r   r   r   r   r1   r2   r3   r4   rG   r   r   r   r   Ý   s    zLinear.__init__Úinputr^   rG   r   c                 O   s$   |   |¡}|  ||j¡}t ||¡S r   )rX   Ú_cast_input_dtyperS   ÚFÚlinear)r   r   rs   r^   rG   Údelta_weightr   r   r   r]   î   s   
zLinear._get_delta_activationsc                    ó   t ƒ  ¡ }d| S ©Nzloha.©r   Ú__repr__©r   Úrepr   r   r   r{   ö   ó   
zLinear.__repr__)ro   r   rp   rp   rp   T©rC   rc   rd   Ú__doc__r   rf   ri   rj   rk   rl   r   r"   rm   r   r]   r{   rn   r   r   r   r   r:   Ú   sF    øþýüûúùøÿÿÿÿ
þr:   c                       sˆ   e Zd ZdZ							ddejded	ed
ededede	de	f‡ fdd„Z
dedejdededejf
dd„Zdef‡ fdd„Z‡  ZS )r=   z LoHa implemented in Conv2d layerro   r   rp   FTr   r   r   r1   r2   r3   r5   r4   c	           
         s4   t ƒ  |¡ || _| j|||||||fi |	¤Ž d S r   rq   )
r   r   r   r   r1   r2   r3   r5   r4   rG   r   r   r   r   þ   s   ÿ
ÿzConv2d.__init__rs   r^   rG   r   c                 O   s>   |   |¡}|  ||j¡}|  ¡ }tj|||j|j|j|j	dS )N)ÚstrideÚpaddingÚdilationÚgroups)
rX   rt   rS   r8   ru   Úconv2dr   r‚   rƒ   r„   )r   r   rs   r^   rG   rw   r   r   r   r   r]     s   
úzConv2d._get_delta_activationsc                    rx   ry   rz   r|   r   r   r   r{   "  r~   zConv2d.__repr__)ro   r   rp   rp   rp   FTr   r   r   r   r   r=   û   sL    ÷þýüûúùø	÷ÿÿÿÿ
þr=   c                   @   ó.   e Zd Zee d¡fdd„ƒZedd„ ƒZdS )Ú
HadaWeightr   c                 C   s*   |   |||||¡ || ||  | }|S r   )Úsave_for_backward)ÚctxÚw1aÚw1bÚw2aÚw2brI   Údiff_weightr   r   r   rb   +  s   zHadaWeight.forwardc                 C   sh   | j \}}}}}|| }|||  }||j }|j| }	|||  }||j }
|j| }~||	|
|d fS r   )Úsaved_tensorsÚT)r‰   Úgrad_outrŠ   r‹   rŒ   r   rI   ÚtempÚgrad_w1aÚgrad_w1bÚgrad_w2aÚgrad_w2br   r   r   Úbackward1  s   



zHadaWeight.backwardN©rC   rc   rd   Ústaticmethodr"   rL   rb   r—   r   r   r   r   r‡   *  s
    r‡   c                   @   r†   )ÚHadaWeightCPr   c           
   	   C   sB   |   |||||||¡ t d|||¡}t d|||¡}	||	 | S )Nzi j k l, j r, i p -> p r k l)rˆ   r"   Úeinsum)
r‰   Út1rŠ   r‹   Út2rŒ   r   rI   Úrebuild1Úrebuild2r   r   r   rb   B  s   zHadaWeightCP.forwardc                 C   sþ   | j \}}}}}}}|| }t d||¡}	t d|	|¡}
|
| }~
t d|	|¡}t d||j¡}~~	t d||¡}t d||j¡}~t d||¡}	t d|	|¡}
|
| }~
t d|	|¡}t d||j¡}~~	t d||¡}t d||j¡}~||||||d fS )Nzi j k l, j r -> i r k lzi j k l, i r -> r j k lzr j k l, i j k l -> r izi r k l, i j k l -> r j)r   r"   r›   r   )r‰   r‘   rœ   rŠ   r‹   r   rŒ   r   rI   r’   ÚrebuildÚgrad_wr“   Ú	grad_tempr”   Úgrad_t1r•   r–   Úgrad_t2r   r   r   r—   K  s.   zHadaWeightCP.backwardNr˜   r   r   r   r   rš   A  s
    rš   c                 C   s   t  | ||||¡S r   )r‡   Úapply)rŠ   r‹   rŒ   r   rI   r   r   r   rM   n  s   rM   c              	   C   s   t  | ||||||¡S r   )rš   r¥   )rœ   rŠ   r‹   r   rŒ   r   rI   r   r   r   rK   r  s   rK   )r+   Útypingr   r"   Útorch.nnr   Útorch.nn.functionalÚ
functionalru   Úpeft.tuners.lycoris_utilsr   rf   r   r:   r=   ÚautogradÚFunctionr‡   rš   rM   rK   r   r   r   r   Ú<module>   s    B!/-