o
    NÆÏi±P  ã                	   @   sÈ   d dl Z d dlmZmZmZ d dlZd dlmZ d dlm  m	Z
 d dlmZ G dd„ dejeƒZG dd„ deƒZG dd	„ d	eƒZG d
d„ deƒZddededeeef fdd„Zdd„ Zddd„ZdS )é    N)ÚAnyÚOptionalÚUnion)ÚLycorisLayerc                       sè   e Zd ZdZdejddf‡ fdd„Zedee	 fdd„ƒZ
d	e	d
edededef
dd„Zd	e	fdd„Zd	e	fdd„Zdd„ Z	d%d	e	d
edededededededededdfdd„Zd	e	dejfd d!„Zd"ejdejfd#d$„Z‡  ZS )&Ú	LoKrLayer©Úlokr_w1Ú	lokr_w1_aÚ	lokr_w1_bÚlokr_w2Ú	lokr_w2_aÚ	lokr_w2_bÚlokr_t2Ú
base_layerÚreturnNc                    sn   t ƒ  ¡  t | |¡ t i ¡| _t i ¡| _t i ¡| _t i ¡| _t i ¡| _	t i ¡| _
t i ¡| _d S ©N)ÚsuperÚ__init__r   ÚnnÚParameterDictr   r	   r
   r   r   r   r   )Úselfr   ©Ú	__class__© úJ/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/lokr/layer.pyr   &   s   
zLoKrLayer.__init__c                 C   s.   h | j £| j£| j£| j£| j£| j£| j£S r   r   )r   r   r   r   Ú_available_adapters3   s   ÿþýüûúùzLoKrLayer._available_adaptersÚadapter_nameÚrÚuse_w1Úuse_w2Úuse_effective_conv2dc              	   C   sê  |rt  t |d d |d d ¡¡| j|< n t  t |d d |¡¡| j|< t  t ||d d ¡¡| j|< t|ƒdkr½|r]t  tj|d d |d d g|dd … ¢R Ž ¡| j|< d S |r“t  t |||d |d ¡¡| j	|< t  t ||d d ¡¡| j
|< t  t ||d d ¡¡| j|< d S t  t |d d |¡¡| j
|< t  t ||d d |d  |d  ¡¡| j|< d S t|ƒdkr:|rßt  t |d d |d d |d ¡¡| j|< d S |rt  t |||d d¡¡| j	|< t  t ||d d ¡¡| j
|< t  t ||d d ¡¡| j|< d S t  t |d d |¡¡| j
|< t  t ||d d |d  ¡¡| j|< d S |rSt  t |d d |d d ¡¡| j|< d S t  t |d d |¡¡| j
|< t  t ||d d ¡¡| j|< d S )Nr   é   é   é   é   )r   Ú	ParameterÚtorchÚemptyr   r	   r
   Úlenr   r   r   r   )r   r   r   Úshaper   r   r    r   r   r   Úcreate_adapter_parameters?   s4   	*  <$ $ 42  $ ,, $z#LoKrLayer.create_adapter_parametersc                 C   sÚ   || j v rtj | j | ¡ ntj | j| ¡ tjj| j| t d¡d || j	v r:tjj| j	| t d¡d ntjj| j
| t d¡d tjj| j| t d¡d || jv rktjj| j| t d¡d d S d S ©Né   )Úa)r   r   ÚinitÚzeros_r	   Úkaiming_uniform_r
   ÚmathÚsqrtr   r   r   r   ©r   r   r   r   r   Úreset_adapter_parametersm   s   


 ÿz"LoKrLayer.reset_adapter_parametersc                 C   sî   || j v rtjj| j | t d¡d ntjj| j| t d¡d tjj| j| t d¡d || jv rDtjj| j| t d¡d ntjj| j	| t d¡d tjj| j
| t d¡d || jv rutjj| j| t d¡d d S d S r+   )r   r   r.   r0   r1   r2   r	   r
   r   r   r   r   r3   r   r   r   Úreset_adapter_parameters_random}   s   


 ÿz)LoKrLayer.reset_adapter_parameters_randomc                 C   sÚ   || j v rtjj| j | t d¡d ntjj| j| t d¡d tjj| j| t d¡d || jv r?tj 	| j| ¡ ntj 	| j
| ¡ tjj| j| t d¡d || jv rktjj| j| t d¡d d S d S r+   )r   r   r.   r0   r1   r2   r	   r
   r   r/   r   r   r   r3   r   r   r   Ú$reset_adapter_parameters_lycoris_wayŽ   s   


 ÿz.LoKrLayer.reset_adapter_parameters_lycoris_wayFÚalphaÚrank_dropoutÚmodule_dropoutÚinit_weightsÚdecompose_bothÚdecompose_factorÚinference_modec                 K   sÚ  |dkrt d|› ƒ‚|| j|< || j|< || | j|< || j|< || j|< |d | j|< |  ¡ }t|t	j
ƒr}|j|j}}t||	ƒ\}}t||	ƒ\}}||f||ff}|of|t|d d |d d ƒd k  }|t|d d |d d ƒd k  }d}n¾t|t	jƒrÓ|j|j}}|j}t||	ƒ\}}t||	ƒ\}}||f||fg|¢R }|o¸|t|d d |d d ƒd k  }|t|d d |d d ƒd k}|oÑ|jdk}nht|t	jƒr1|j|j}}|jd f}t||	ƒ\}}t||	ƒ\}}||f||fg|¢R }|o|t|d d |d d ƒd k  }|t|d d |d d ƒd k}|o/|jd dk}n
tdt|ƒj› ƒ‚|  ||||||¡ |rY|d	krS|  |¡ n|  |¡ n|  |¡ |  |¡ | j| j|
d
 dS )a  Internal function to create lokr adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            alpha (`float`): Alpha for the added adapter.
            rank_dropout (`float`): The dropout probability for rank dimension during training
            module_dropout (`float`): The dropout probability for disabling adapter during training.
            init_weights (`bool`): Whether to initialize adapter weights.
            use_effective_conv2d (`bool`): Use parameter effective decomposition for Conv2d with ksize > 1.
            decompose_both (`bool`): Perform rank decomposition of left kronecker product matrix.
            decompose_factor (`int`): Kronecker product decomposition factor.
        r   z?`r` should be a positive integer value but the value passed is Úrank_dropout_scaler!   r#   F)r!   r!   z0LoKr is not implemented for base layers of type Úlycoris)r=   N)Ú
ValueErrorr   r7   Úscalingr8   r9   r>   Úget_base_layerÚ
isinstancer   ÚLinearÚin_featuresÚout_featuresÚfactorizationÚmaxÚConv2dÚin_channelsÚout_channelsÚkernel_sizeÚConv1dÚ	TypeErrorÚtypeÚ__name__r*   r6   r4   r5   Ú%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r   r   r   r7   r8   r9   r:   r    r;   r<   r=   Úkwargsr   Úin_dimÚout_dimÚin_mÚin_nÚout_lÚout_kr)   r   r   Úk_sizer   r   r   Úupdate_layerž   sX   



($("*"


zLoKrLayer.update_layerc                 C   s"  || j v r| j | }n
| j| | j|  }|| jv r | j| }n|| jv r5t| j| | j| | j| ƒ}n
| j| | j|  }t||| j	| ƒ}|  
¡ }| |jj¡}| j| }| jr|rt | d¡¡|k ¡ }|jdgdgt|jdd … ƒ ¢R Ž  |j¡}| j| r‹|| ¡  }||9 }|S )Nr   éÿÿÿÿr!   )r   r	   r
   r   r   Úmake_weight_cpr   r   Ú	make_kronrA   rB   ÚreshapeÚweightr)   r8   Útrainingr&   ÚrandÚsizeÚfloatÚviewr(   ÚtoÚdevicer>   Úmean)r   r   Úw1Úw2ra   r   r8   Údropr   r   r   Úget_delta_weight  s&   


 

.
zLoKrLayer.get_delta_weightÚxc                 O   sÔ   |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}nI| jr*| j|g|¢R i |¤Ž}n9| j|g|¢R i |¤Ž}| jD ])}|| jvrAq9| j| }| jrS| jrbt	 
d¡|krb|| j||g|¢R i |¤Ž }q9| |¡}|S )Nr!   )ÚdtypeÚdisable_adaptersÚmergedÚunmerger   rS   r   r9   rb   r&   rc   Ú_get_delta_activationsrg   )r   rn   ÚargsrT   Úprevious_dtypeÚresultÚactive_adapterr9   r   r   r   Úforward#  s"   


€
zLoKrLayer.forward)F)rP   Ú
__module__Ú__qualname__Úadapter_layer_namesr   ÚModuler   ÚpropertyÚsetÚstrr   ÚintÚboolr*   r4   r5   r6   re   r\   r&   ÚTensorrm   rx   Ú__classcell__r   r   r   r   r      sZ    þýûú
ù.õþýüûúùø	÷
öõ
óc"r   c                       s¢   e Zd ZdZ								ddejdeeee	j
f  d	ee	j d
edededededef‡ fdd„Zd
ede	jdedede	jf
dd„Zdef‡ fdd„Z‡  ZS )rD   z LoKr implemented in Linear layerNÚdefaultr   ç        Tr   rh   ro   r   r   r7   r8   r9   r:   c
                    s2   t ƒ  |¡ || _| j||||||	fi |
¤Ž d S r   ©r   r   Ú_active_adapterr\   )r   r   rh   ro   r   r   r7   r8   r9   r:   rT   r   r   r   r   A  s    zLinear.__init__Úinputrt   rT   r   c                 O   s$   |   |¡}|  ||j¡}t ||¡S r   )rm   Ú_cast_input_dtypero   ÚFÚlinear)r   r   rˆ   rt   rT   Údelta_weightr   r   r   rs   T  s   
zLinear._get_delta_activationsc                    ó   t ƒ  ¡ }d| S ©Nzlokr.©r   Ú__repr__©r   Úrepr   r   r   r   \  ó   
zLinear.__repr__)NNr„   r   r…   r…   r…   T©rP   ry   rz   Ú__doc__r   r|   r   r   r   r&   rh   ro   r€   re   r   r   r‚   r   rs   r   rƒ   r   r   r   r   rD   >  sR    öþýüûúùø	÷
öÿÿÿÿ
þrD   c                       ó¨   e Zd ZdZ									ddejd	eeee	j
f  d
ee	j dededededededef‡ fdd„Zdede	jdedede	jf
dd„Zdef‡ fdd„Z‡  ZS )rI   z LoKr implemented in Conv2d layerNr„   r   r…   FTr   rh   ro   r   r   r7   r8   r9   r    r:   c                    ó4   t ƒ  |¡ || _| j||||||
|	fi |¤Ž d S r   r†   ©r   r   rh   ro   r   r   r7   r8   r9   r    r:   rT   r   r   r   r   d  ó   ÿ
ÿzConv2d.__init__rˆ   rt   rT   r   c                 O   ó>   |   |¡}|  ||j¡}|  ¡ }tj|||j|j|j|j	dS ©N)ÚstrideÚpaddingÚdilationÚgroups)
rm   r‰   ro   rB   rŠ   Úconv2drœ   r   rž   rŸ   ©r   r   rˆ   rt   rT   rŒ   r   r   r   r   rs   z  ó   
úzConv2d._get_delta_activationsc                    r   rŽ   r   r‘   r   r   r   r   Š  r“   zConv2d.__repr__©	NNr„   r   r…   r…   r…   FTr”   r   r   r   r   rI   a  óX    õþýüûúùø	÷
öõÿÿÿÿ
þrI   c                       r–   )rM   z LoKr implemented in Conv1d layerNr„   r   r…   FTr   rh   ro   r   r   r7   r8   r9   r    r:   c                    r—   r   r†   r˜   r   r   r   r   ’  r™   zConv1d.__init__rˆ   rt   rT   r   c                 O   rš   r›   )
rm   r‰   ro   rB   rŠ   Úconv1drœ   r   rž   rŸ   r¡   r   r   r   rs   ¨  r¢   zConv1d._get_delta_activationsc                    r   rŽ   r   r‘   r   r   r   r   ¸  r“   zConv1d.__repr__r£   r”   r   r   r   r   rM     r¤   rM   r]   Ú	dimensionÚfactorr   c                 C   sÀ   |dkr| | dkr|}| | }||fS |dkr| }d| }}|| }||k rS|d }| | dkr;|d7 }| | dks1| | }|| |ksI||krJn	||}}||k s'||kr\||}}||fS )a¢  Factorizes the provided number into the product of two numbers

    Args:
        dimension (`int`): The number that needs to be factorized.
        factor (`int`, optional):
            Factorization divider. The algorithm will try to output two numbers, one of each will be as close to the
            factor as possible. If -1 is provided, the decomposition algorithm would try to search dividers near the
            square root of the dimension. Defaults to -1.

    Returns:
        Tuple[`int`, `int`]: A tuple of two numbers, whose product is equal to the provided number. The first number is
        always less than or equal to the second.

    Example:
        ```py
        >>> factorization(256, factor=-1)
        (16, 16)

        >>> factorization(128, factor=-1)
        (8, 16)

        >>> factorization(127, factor=-1)
        (1, 127)

        >>> factorization(128, factor=4)
        (4, 32)
        ```
    r   r]   r!   r   )r¦   r§   ÚmÚnÚlengthÚnew_mÚnew_nr   r   r   rG   À  s*   
ÿ
ø	
rG   c                 C   s   t  d| ||¡}|S )Nzi j k l, i p, j r -> p r k l)r&   Úeinsum)ÚtÚwaÚwbÚrebuild2r   r   r   r^   ô  s   r^   ç      ð?c                 C   s:   t |jƒdkr|  d¡ d¡} | ¡ }t | |¡}|| S )Nr"   r#   )r(   r)   Ú	unsqueezeÚ
contiguousr&   Úkron)rj   rk   ÚscaleÚrebuildr   r   r   r_   ù  s
   r_   )r]   )r²   )r1   Útypingr   r   r   r&   Útorch.nnr   Útorch.nn.functionalÚ
functionalrŠ   Úpeft.tuners.lycoris_utilsr   r|   r   rD   rI   rM   r€   ÚtuplerG   r^   r_   r   r   r   r   Ú<module>   s     '#. 14