o
    NÆÏi×I  ã                   @   sÆ   d dl Z d dlmZ d dlZd dlmZ d dlm  mZ d dl	m
Z
 G dd„ deje
ƒZG dd„ deƒZG dd	„ d	eƒZG d
d„ deƒZG dd„ dejjƒZG dd„ dejjƒZdd„ Zdd„ ZdS )é    N)ÚAny)ÚLycorisLayerc                       sÖ   e Zd ZdZdejf‡ fdd„Zedee	 fdd„ƒZ
de	d	ed
eedf fdd„Zde	fdd„Zde	fdd„Z		d!de	d	edededededededdfdd„Zde	dejfdd„Zdejdejfdd „Z‡  ZS )"Ú	LoHaLayer©Ú	hada_w1_aÚ	hada_w1_bÚ	hada_w2_aÚ	hada_w2_bÚhada_t1Úhada_t2Ú
base_layerc                    sb   t ƒ  ¡  t | |¡ t i ¡| _t i ¡| _t i ¡| _t i ¡| _t i ¡| _	t i ¡| _
d S ©N)ÚsuperÚ__init__r   ÚnnÚParameterDictr   r   r   r	   r
   r   )Úselfr   ©Ú	__class__© úJ/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/loha/layer.pyr      s   
zLoHaLayer.__init__Úreturnc                 C   s(   h | j £| j£| j£| j£| j£| j£S r   r   )r   r   r   r   Ú_available_adapters*   s   (zLoHaLayer._available_adaptersÚadapter_nameÚrÚshape.c              	   C   sü  t |ƒdkrdt t |||d |d ¡¡| j|< t t ||d ¡¡| j|< t t ||d ¡¡| j|< t t |||d |d ¡¡| j|< t t ||d ¡¡| j	|< t t ||d ¡¡| j
|< d S t |ƒdkrÄt t |||d d¡¡| j|< t t ||d ¡¡| j|< t t ||d ¡¡| j|< t t |||d d¡¡| j|< t t ||d ¡¡| j	|< t t ||d ¡¡| j
|< d S t t |d |¡¡| j|< t t ||d ¡¡| j|< t t |d |¡¡| j	|< t t ||d ¡¡| j
|< d S )Né   é   é   r   é   )Úlenr   Ú	ParameterÚtorchÚemptyr
   r   r   r   r   r	   )r   r   r   r   r   r   r   Úcreate_adapter_parameters.   s$   $$     z#LoHaLayer.create_adapter_parametersc                 C   sÂ   || j  ¡ v r:tjj| j | t d¡d tjj| j| t d¡d tjj| j| t d¡d tj 	| j
| ¡ || j ¡ v r_tjj| j| t d¡d tjj| j| t d¡d d S d S ©Né   )Úa)r   Úkeysr   ÚinitÚkaiming_uniform_ÚmathÚsqrtr   r   Úzeros_r	   r
   r   ©r   r   r   r   r   Úreset_adapter_parametersG   s    þz"LoHaLayer.reset_adapter_parametersc                 C   sÌ   || j  ¡ v r?tjj| j | t d¡d tjj| j| t d¡d tjj| j| t d¡d tjj| j	| t d¡d || j
 ¡ v rdtjj| j
| t d¡d tjj| j| t d¡d d S d S r%   )r   r(   r   r)   r*   r+   r,   r   r   r	   r
   r   r.   r   r   r   Úreset_adapter_parameters_randomV   s    þz)LoHaLayer.reset_adapter_parameters_randomFÚalphaÚrank_dropoutÚmodule_dropoutÚinit_weightsÚuse_effective_conv2dÚinference_modeNc	                 K   sn  |dkrt d|› ƒ‚|| j|< || j|< || | j|< || j|< || j|< |  ¡ }
t|
tj	ƒr7t
|
jjƒ}n]t|
tjƒrb|oC|
jdk}|rQ|
j|
jg|
j¢R }nC|
j|
j|
jd  |
jd  f}n2t|
tjƒrŠ|op|
jd dk}|r~|
j|
j|
jd f}n|
j|
j|
jd  f}n
tdt|
ƒj› ƒ‚|  |||¡ |r£|  |¡ n|  |¡ |  |¡ | j| j|d dS )a  Internal function to create loha adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            alpha (`float`): Alpha for the added adapter.
            rank_dropout (`float`): The dropout probability for rank dimension during training.
            module_dropout (`float`): The dropout probability for disabling adapter during training.
            init_weights (`bool`): Whether to initialize weights.
            use_effective_conv2d (`bool`, *optional*, defaults to `False`):
                Use parameter effective decomposition for Conv2d with ksize > 1.
        r   z?`r` should be a positive integer value but the value passed is )r   r   r   z0LoHa is not implemented for base layers of type )r6   N)Ú
ValueErrorr   r1   Úscalingr2   r3   Úget_base_layerÚ
isinstancer   ÚLinearÚtupleÚweightr   ÚConv2dÚkernel_sizeÚout_channelsÚin_channelsÚConv1dÚ	TypeErrorÚtypeÚ__name__r$   r/   r0   Ú%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r   r   r   r1   r2   r3   r4   r5   r6   Úkwargsr   r   r   r   r   Úupdate_layere   s>   



þþ

zLoHaLayer.update_layerc                 C   s  || j  ¡ v r+t| j | | j| | j| | j| | j| | j| t 	| j
| ¡d}nt| j| | j| | j| | j| t 	| j
| ¡d}|  ¡ }| |jj¡}| j| }| jrŠ|rŠt | d¡¡|k |j¡}|jdgdgt|jdd … ƒ ¢R Ž  |j¡}|| ¡  }||9 }|S )N)Úscaler   éÿÿÿÿr   )r
   r(   Úmake_weight_cpr   r   r   r   r	   r"   Útensorr8   Úmake_weightr9   Úreshaper=   r   r2   ÚtrainingÚrandÚsizeÚtoÚdtypeÚviewr    ÚdeviceÚmean)r   r   r=   r   r2   Údropr   r   r   Úget_delta_weightµ   s4   ù
û

.zLoHaLayer.get_delta_weightÚxc                 O   sÔ   |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}nI| jr*| j|g|¢R i |¤Ž}n9| j|g|¢R i |¤Ž}| jD ])}|| jvrAq9| j| }| jrS| jrbt	 
d¡|krb|| j||g|¢R i |¤Ž }q9| |¡}|S )Nr   )rU   Údisable_adaptersÚmergedÚunmerger   rH   r   r3   rQ   r"   rR   Ú_get_delta_activationsrT   )r   r[   ÚargsrI   Úprevious_dtypeÚresultÚactive_adapterr3   r   r   r   ÚforwardÜ   s"   


€
zLoHaLayer.forward)FF)rE   Ú
__module__Ú__qualname__Úadapter_layer_namesr   ÚModuler   ÚpropertyÚsetÚstrr   Úintr<   r$   r/   r0   ÚfloatÚboolrJ   r"   ÚTensorrZ   rd   Ú__classcell__r   r   r   r   r      s>    ÷þýüûúùø	÷
õP'r   c                       s‚   e Zd ZdZ						ddejdeded	ed
edede	f‡ fdd„Z
dedejdededejf
dd„Zdef‡ fdd„Z‡  ZS )r;   z LoHa implemented in Linear layerÚdefaultr   ç        Tr   r   r   r1   r2   r3   r4   c           	         s2   t ƒ  |¡ || _| j||||||fi |¤Ž d S r   ©r   r   Ú_active_adapterrJ   )	r   r   r   r   r1   r2   r3   r4   rI   r   r   r   r   ú   s    zLinear.__init__Úinputr`   rI   r   c                 O   s$   |   |¡}|  ||j¡}t ||¡S r   )rZ   Ú_cast_input_dtyperU   ÚFÚlinear)r   r   ru   r`   rI   Údelta_weightr   r   r   r_     s   
zLinear._get_delta_activationsc                    ó   t ƒ  ¡ }d| S ©Nzloha.©r   Ú__repr__©r   Úrepr   r   r   r}     ó   
zLinear.__repr__)rq   r   rr   rr   rr   T©rE   re   rf   Ú__doc__r   rh   rk   rl   rm   rn   r   r"   ro   r   r_   r}   rp   r   r   r   r   r;   ÷   sF    øþýüûúùøÿÿÿÿ
þr;   c                       óˆ   e Zd ZdZ							ddejded	ed
ededede	de	f‡ fdd„Z
dedejdededejf
dd„Zdef‡ fdd„Z‡  ZS )r>   z LoHa implemented in Conv2d layerrq   r   rr   FTr   r   r   r1   r2   r3   r5   r4   c	           
         ó4   t ƒ  |¡ || _| j|||||||fi |	¤Ž d S r   rs   ©
r   r   r   r   r1   r2   r3   r5   r4   rI   r   r   r   r     ó   ÿ
ÿzConv2d.__init__ru   r`   rI   r   c                 O   ó>   |   |¡}|  ||j¡}|  ¡ }tj|||j|j|j|j	dS ©N)ÚstrideÚpaddingÚdilationÚgroups)
rZ   rv   rU   r9   rw   Úconv2dr‰   rŠ   r‹   rŒ   ©r   r   ru   r`   rI   ry   r   r   r   r   r_   /  ó   
úzConv2d._get_delta_activationsc                    rz   r{   r|   r~   r   r   r   r}   ?  r€   zConv2d.__repr__©rq   r   rr   rr   rr   FTr   r   r   r   r   r>     óL    ÷þýüûúùø	÷ÿÿÿÿ
þr>   c                       rƒ   )rB   z LoHa implemented in Conv1d layerrq   r   rr   FTr   r   r   r1   r2   r3   r5   r4   c	           
         r„   r   rs   r…   r   r   r   r   G  r†   zConv1d.__init__ru   r`   rI   r   c                 O   r‡   rˆ   )
rZ   rv   rU   r9   rw   Úconv1dr‰   rŠ   r‹   rŒ   rŽ   r   r   r   r_   [  r   zConv1d._get_delta_activationsc                    rz   r{   r|   r~   r   r   r   r}   k  r€   zConv1d.__repr__r   r   r   r   r   r   rB   D  r‘   rB   c                   @   ó.   e Zd Zee d¡fdd„ƒZedd„ ƒZdS )Ú
HadaWeightr   c                 C   s*   |   |||||¡ || ||  | }|S r   )Úsave_for_backward)ÚctxÚw1aÚw1bÚw2aÚw2brK   Údiff_weightr   r   r   rd   t  s   zHadaWeight.forwardc                 C   sh   | j \}}}}}|| }|||  }||j }|j| }	|||  }||j }
|j| }~||	|
|d fS r   )Úsaved_tensorsÚT)r–   Úgrad_outr—   r˜   r™   rš   rK   ÚtempÚgrad_w1aÚgrad_w1bÚgrad_w2aÚgrad_w2br   r   r   Úbackwardz  s   



zHadaWeight.backwardN©rE   re   rf   Ústaticmethodr"   rN   rd   r¤   r   r   r   r   r”   s  s
    r”   c                   @   r“   )ÚHadaWeightCPr   c           
   	   C   sB   |   |||||||¡ t d|||¡}t d|||¡}	||	 | S )Nzi j k l, j r, i p -> p r k l)r•   r"   Úeinsum)
r–   Út1r—   r˜   Út2r™   rš   rK   Úrebuild1Úrebuild2r   r   r   rd   ‹  s   zHadaWeightCP.forwardc                 C   sþ   | j \}}}}}}}|| }t d||¡}	t d|	|¡}
|
| }~
t d|	|¡}t d||j¡}~~	t d||¡}t d||j¡}~t d||¡}	t d|	|¡}
|
| }~
t d|	|¡}t d||j¡}~~	t d||¡}t d||j¡}~||||||d fS )Nzi j k l, j r -> i r k lzi j k l, i r -> r j k lzr j k l, i j k l -> r izi r k l, i j k l -> r j)rœ   r"   r¨   r   )r–   rž   r©   r—   r˜   rª   r™   rš   rK   rŸ   ÚrebuildÚgrad_wr    Ú	grad_tempr¡   Úgrad_t1r¢   r£   Úgrad_t2r   r   r   r¤   ”  s.   zHadaWeightCP.backwardNr¥   r   r   r   r   r§   Š  s
    r§   c                 C   s   t  | ||||¡S r   )r”   Úapply)r—   r˜   r™   rš   rK   r   r   r   rO   ·  s   rO   c              	   C   s   t  | ||||||¡S r   )r§   r²   )r©   r—   r˜   rª   r™   rš   rK   r   r   r   rM   »  s   rM   )r+   Útypingr   r"   Útorch.nnr   Útorch.nn.functionalÚ
functionalrw   Úpeft.tuners.lycoris_utilsr   rh   r   r;   r>   rB   ÚautogradÚFunctionr”   r§   rO   rM   r   r   r   r   Ú<module>   s    _!,/-