o
    8wÖi*@  ã                	   @   s¸   d dl Z d dlmZmZmZ d dlZd dlmZ d dlm  m	Z
 d dlmZ G dd„ dejeƒZG dd„ deƒZG dd	„ d	eƒZddededeeef fdd„Zdd„ Zddd„ZdS )é    N)ÚAnyÚOptionalÚUnion)ÚLycorisLayerc                       sà   e Zd ZdZdejddf‡ fdd„Zedee	 fdd„ƒZ
d	e	d
edededef
dd„Zd	e	fdd„Zd	e	fdd„Zdd„ Zd	e	d
ededededededededdfdd„Zd	e	dejfdd„Zd ejdejfd!d"„Z‡  ZS )#Ú	LoKrLayer©Úlokr_w1Ú	lokr_w1_aÚ	lokr_w1_bÚlokr_w2Ú	lokr_w2_aÚ	lokr_w2_bÚlokr_t2Ú
base_layerÚreturnNc                    sn   t ƒ  ¡  t | |¡ t i ¡| _t i ¡| _t i ¡| _t i ¡| _t i ¡| _	t i ¡| _
t i ¡| _d S ©N)ÚsuperÚ__init__r   ÚnnÚParameterDictr   r	   r
   r   r   r   r   )Úselfr   ©Ú	__class__© úS/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/lokr/layer.pyr   &   s   
zLoKrLayer.__init__c                 C   s.   h | j £| j£| j£| j£| j£| j£| j£S r   r   )r   r   r   r   Ú_available_adapters3   s   ÿþýüûúùzLoKrLayer._available_adaptersÚadapter_nameÚrÚuse_w1Úuse_w2Úuse_effective_conv2dc              	   C   sî  |rt  t |d d |d d ¡¡| j|< n t  t |d d |¡¡| j|< t  t ||d d ¡¡| j|< t|ƒdkr½|r]t  tj|d d |d d g|dd … ¢R Ž ¡| j|< d S |r“t  t |||d |d ¡¡| j	|< t  t ||d d ¡¡| j
|< t  t ||d d ¡¡| j|< d S t  t |d d |¡¡| j
|< t  t ||d d |d  |d  ¡¡| j|< d S |rÕt  t |d d |d d ¡¡| j|< d S t  t |d d |¡¡| j
|< t  t ||d d ¡¡| j|< d S )Nr   é   é   é   é   )r   Ú	ParameterÚtorchÚemptyr   r	   r
   Úlenr   r   r   r   )r   r   r   Úshaper   r   r    r   r   r   Úcreate_adapter_parameters?   s"   	*  <$ $ 4, $z#LoKrLayer.create_adapter_parametersc                 C   sÚ   || j v rtj | j | ¡ ntj | j| ¡ tjj| j| t d¡d || j	v r:tjj| j	| t d¡d ntjj| j
| t d¡d tjj| j| t d¡d || jv rktjj| j| t d¡d d S d S ©Né   )Úa)r   r   ÚinitÚzeros_r	   Úkaiming_uniform_r
   ÚmathÚsqrtr   r   r   r   ©r   r   r   r   r   Úreset_adapter_parametersa   s   


 ÿz"LoKrLayer.reset_adapter_parametersc                 C   sî   || j v rtjj| j | t d¡d ntjj| j| t d¡d tjj| j| t d¡d || jv rDtjj| j| t d¡d ntjj| j	| t d¡d tjj| j
| t d¡d || jv rutjj| j| t d¡d d S d S r+   )r   r   r.   r0   r1   r2   r	   r
   r   r   r   r   r3   r   r   r   Úreset_adapter_parameters_randomq   s   


 ÿz)LoKrLayer.reset_adapter_parameters_randomc                 C   sÚ   || j v rtjj| j | t d¡d ntjj| j| t d¡d tjj| j| t d¡d || jv r?tj 	| j| ¡ ntj 	| j
| ¡ tjj| j| t d¡d || jv rktjj| j| t d¡d d S d S r+   )r   r   r.   r0   r1   r2   r	   r
   r   r/   r   r   r   r3   r   r   r   Ú$reset_adapter_parameters_lycoris_way‚   s   


 ÿz.LoKrLayer.reset_adapter_parameters_lycoris_wayÚalphaÚrank_dropoutÚmodule_dropoutÚinit_weightsÚdecompose_bothÚdecompose_factorc
                 K   s  |dkrt d|› ƒ‚|| j|< || j|< || | j|< || j|< || j|< |
d | j|< |  ¡ }t|t	j
ƒr}|j|j}}t||	ƒ\}}t||	ƒ\}}||f||ff}|of|t|d d |d d ƒd k  }|t|d d |d d ƒd k  }d}n`t|t	jƒrÓ|j|j}}|j}t||	ƒ\}}t||	ƒ\}}||f||fg|¢R }|o¸|t|d d |d d ƒd k  }|t|d d |d d ƒd k}|oÑ|jdk}n
tdt|ƒj› ƒ‚|  ||||||¡ |rù|d	kró|  |¡ n|  |¡ n|  |¡ |  |¡ |  | j¡ d
S )a  Internal function to create lokr adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            alpha (`float`): Alpha for the added adapter.
            rank_dropout (`float`): The dropout probability for rank dimension during training
            module_dropout (`float`): The dropout probability for disabling adapter during training.
            init_weights (`bool`): Whether to initialize adapter weights.
            use_effective_conv2d (`bool`): Use parameter effective decomposition for Conv2d with ksize > 1.
            decompose_both (`bool`): Perform rank decomposition of left kronecker product matrix.
            decompose_factor (`int`): Kronecker product decomposition factor.
        r   z?`r` should be a positive integer value but the value passed is Úrank_dropout_scaler!   r#   F)r!   r!   z0LoKr is not implemented for base layers of type ÚlycorisN)Ú
ValueErrorr   r7   Úscalingr8   r9   r=   Úget_base_layerÚ
isinstancer   ÚLinearÚin_featuresÚout_featuresÚfactorizationÚmaxÚConv2dÚin_channelsÚout_channelsÚkernel_sizeÚ	TypeErrorÚtypeÚ__name__r*   r6   r4   r5   Ú%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r   r   r   r7   r8   r9   r:   r    r;   r<   Úkwargsr   Úin_dimÚout_dimÚin_mÚin_nÚout_lÚout_kr)   r   r   Úk_sizer   r   r   Úupdate_layer’   sF   



($("

zLoKrLayer.update_layerc                 C   s  || j v r| j | }n
| j| | j|  }|| jv r | j| }n|| jv r5t| j| | j| | j| ƒ}n
| j| | j|  }t||| j	| ƒ}| 
|  ¡ jj¡}| j| }| jr|rt | d¡¡|k ¡ }|jdgdgt|jdd … ƒ ¢R Ž  |j¡}| j| r‰|| ¡  }||9 }|S )Nr   éÿÿÿÿr!   )r   r	   r
   r   r   Úmake_weight_cpr   r   Ú	make_kronr@   ÚreshaperA   Úweightr)   r8   Útrainingr&   ÚrandÚsizeÚfloatÚviewr(   ÚtoÚdevicer=   Úmean)r   r   Úw1Úw2r_   r8   Údropr   r   r   Úget_delta_weightà   s$   


 

.
zLoKrLayer.get_delta_weightÚxc                 O   sÔ   |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}nI| jr*| j|g|¢R i |¤Ž}n9| j|g|¢R i |¤Ž}| jD ])}|| jvrAq9| j| }| jrS| jrbt	 
d¡|krb|| j||g|¢R i |¤Ž }q9| |¡}|S )Nr!   )ÚdtypeÚdisable_adaptersÚmergedÚunmerger   rQ   r   r9   r`   r&   ra   Ú_get_delta_activationsre   )r   rl   ÚargsrR   Úprevious_dtypeÚresultÚactive_adapterr9   r   r   r   Úforwardý   s"   


€
zLoKrLayer.forward)rN   Ú
__module__Ú__qualname__Úadapter_layer_namesr   ÚModuler   ÚpropertyÚsetÚstrr   ÚintÚboolr*   r4   r5   r6   rc   rZ   r&   ÚTensorrk   rv   Ú__classcell__r   r   r   r   r      sT    þýûú
ù"þýüûúùø	÷
ö
ôNr   c                       s¢   e Zd ZdZ								ddejdeeee	j
f  d	ee	j d
edededededef‡ fdd„Zd
ede	jdedede	jf
dd„Zdef‡ fdd„Z‡  ZS )rC   z LoKr implemented in Linear layerNÚdefaultr   ç        Tr   rf   rm   r   r   r7   r8   r9   r:   c
                    s2   t ƒ  |¡ || _| j||||||	fi |
¤Ž d S r   ©r   r   Ú_active_adapterrZ   )r   r   rf   rm   r   r   r7   r8   r9   r:   rR   r   r   r   r     s    zLinear.__init__Úinputrr   rR   r   c                 O   s$   |   |¡}|  ||j¡}t ||¡S r   )rk   Ú_cast_input_dtyperm   ÚFÚlinear)r   r   r†   rr   rR   Údelta_weightr   r   r   rq   .  s   
zLinear._get_delta_activationsc                    ó   t ƒ  ¡ }d| S ©Nzlokr.©r   Ú__repr__©r   Úrepr   r   r   rŽ   6  ó   
zLinear.__repr__)NNr‚   r   rƒ   rƒ   rƒ   T©rN   rw   rx   Ú__doc__r   rz   r   r   r}   r&   rf   rm   r~   rc   r   r   r€   r   rq   rŽ   r   r   r   r   r   rC     sR    öþýüûúùø	÷
öÿÿÿÿ
þrC   c                       s¨   e Zd ZdZ									ddejd	eeee	j
f  d
ee	j dededededededef‡ fdd„Zdede	jdedede	jf
dd„Zdef‡ fdd„Z‡  ZS )rH   z LoKr implemented in Conv2d layerNr‚   r   rƒ   FTr   rf   rm   r   r   r7   r8   r9   r    r:   c                    s4   t ƒ  |¡ || _| j||||||
|	fi |¤Ž d S r   r„   )r   r   rf   rm   r   r   r7   r8   r9   r    r:   rR   r   r   r   r   >  s   ÿ
ÿzConv2d.__init__r†   rr   rR   r   c                 O   s>   |   |¡}|  ||j¡}|  ¡ }tj|||j|j|j|j	dS )N)ÚstrideÚpaddingÚdilationÚgroups)
rk   r‡   rm   rA   rˆ   Úconv2dr”   r•   r–   r—   )r   r   r†   rr   rR   rŠ   r   r   r   r   rq   T  s   
úzConv2d._get_delta_activationsc                    r‹   rŒ   r   r   r   r   r   rŽ   d  r‘   zConv2d.__repr__)	NNr‚   r   rƒ   rƒ   rƒ   FTr’   r   r   r   r   rH   ;  sX    õþýüûúùø	÷
öõÿÿÿÿ
þrH   r[   Ú	dimensionÚfactorr   c                 C   sÀ   |dkr| | dkr|}| | }||fS |dkr| }d| }}|| }||k rS|d }| | dkr;|d7 }| | dks1| | }|| |ksI||krJn	||}}||k s'||kr\||}}||fS )a¢  Factorizes the provided number into the product of two numbers

    Args:
        dimension (`int`): The number that needs to be factorized.
        factor (`int`, optional):
            Factorization divider. The algorithm will try to output two numbers, one of each will be as close to the
            factor as possible. If -1 is provided, the decomposition algorithm would try to search dividers near the
            square root of the dimension. Defaults to -1.

    Returns:
        Tuple[`int`, `int`]: A tuple of two numbers, whose product is equal to the provided number. The first number is
        always less than or equal to the second.

    Example:
        ```py
        >>> factorization(256, factor=-1)
        (16, 16)

        >>> factorization(128, factor=-1)
        (8, 16)

        >>> factorization(127, factor=-1)
        (1, 127)

        >>> factorization(128, factor=4)
        (4, 32)
        ```
    r   r[   r!   r   )r™   rš   ÚmÚnÚlengthÚnew_mÚnew_nr   r   r   rF   l  s*   
ÿ
ø	
rF   c                 C   s   t  d| ||¡}|S )Nzi j k l, i p, j r -> p r k l)r&   Úeinsum)ÚtÚwaÚwbÚrebuild2r   r   r   r\      s   r\   ç      ð?c                 C   s:   t |jƒdkr|  d¡ d¡} | ¡ }t | |¡}|| S )Nr"   r#   )r(   r)   Ú	unsqueezeÚ
contiguousr&   Úkron)rh   ri   ÚscaleÚrebuildr   r   r   r]   ¥  s
   r]   )r[   )r¥   )r1   Útypingr   r   r   r&   Útorch.nnr   Útorch.nn.functionalÚ
functionalrˆ   Úpeft.tuners.lycoris_utilsr   rz   r   rC   rH   r~   ÚtuplerF   r\   r]   r   r   r   r   Ú<module>   s     # 14