o
    8wÖifJ  ã                   @   s–   d dl Z d dlZd dlmZmZmZ d dlZd dlmZ d dl	m  m
Z d dlmZmZ G dd„ deƒZG dd„ dejeƒZG dd	„ d	ejeƒZdS )
é    N)ÚAnyÚOptionalÚUnion)ÚBaseTunerLayerÚcheck_adapters_to_mergec                
   @   s€   e Zd ZdZdZdejddfdd„Zded	e	d
e
de
ddf
dd„Zdefdd„Zdefdd„Zdeddfdd„Zdddd„ZdS )ÚHRALayer)Úhra_u)Úhra_rÚhra_apply_GSÚ
base_layerÚreturnNc                 K   s”   || _ i | _i | _t i ¡| _d| _g | _d| _|| _	|  
¡ }t|tjƒr0|j|j| _| _d S t|tjƒrA|j|j| _| _d S tdt|ƒ› ƒ‚)NFTzUnsupported layer type )r   r	   r
   ÚnnÚParameterDictr   Ú_disable_adaptersÚmerged_adaptersÚcast_input_dtype_enabledÚkwargsÚget_base_layerÚ
isinstanceÚLinearÚin_featuresÚout_featuresÚConv2dÚin_channelsÚout_channelsÚ
ValueErrorÚtype)Úselfr   r   © r   úR/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/hra/layer.pyÚ__init__    s   zHRALayer.__init__Úadapter_nameÚrÚapply_GSÚinit_weightsc                 K   sæ   |dkrt d|› ƒ‚|| j|< || j|< |  ¡ }t|tjƒr/tjt 	| j
|¡dd| j|< n*t|tjƒrOtjt 	| j
|jd  |jd  |¡dd| j|< n
tdt|ƒj› ƒ‚|ra|  |¡ n|  |¡ |  |¡ |  | j¡ dS )aJ  Internal function to create hra adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            init_weights (`bool`): Whether to initialize weights.
            apply_GS (`bool`): Whether to apply Gram-Schmidt orthogonalization or not.
        r   z?`r` should be a positive integer value but the value passed is T)Úrequires_gradz/HRA is not implemented for base layers of type N)r   r	   r
   r   r   r   r   Ú	ParameterÚtorchÚemptyr   r   r   Úkernel_sizeÚ	TypeErrorr   Ú__name__Úreset_hra_parametersÚreset_hra_parameters_randomÚ%_move_adapter_to_device_of_base_layerÚset_adapterÚactive_adapters)r   r!   r"   r#   r$   r   r   r   r   r   Úupdate_layer4   s$   

  þ

zHRALayer.update_layerc                 C   s–   | j | d dkrt d¡ tjj| j| t d¡d d S | j| j	}t
 |d |d d ¡}tjj|t d¡d t t
j|ddd¡| j|< d S )Né   r   z@The symmetric initialization can NOT be performed when r is odd!é   ©Úaé   ©Údim)r	   ÚwarningsÚwarnr   ÚinitÚkaiming_uniform_r   ÚmathÚsqrtÚshaper'   Úzerosr&   Úrepeat_interleave)r   r!   r?   Úhalf_ur   r   r   r,   `   s   
  zHRALayer.reset_hra_parametersc                 C   s    t jj| j| t d¡d d S )Nr3   r4   )r   r;   r<   r   r=   r>   )r   r!   r   r   r   r-   j   s    z$HRALayer.reset_hra_parameters_randomÚscalec                 C   s6   |dkrd S | j D ]}|| j ¡ vrq	t d¡ q	d S )Nr6   zFScaling operation for HRA not supported! Automatically set scale to 1.©r0   r   Úkeysr9   r:   ©r   rC   Úactive_adapterr   r   r   Úscale_layerm   s   
üzHRALayer.scale_layerc                 C   s*   | j D ]}|| j ¡ vrqt d¡ qd S )Nz>Unscaling operation for HRA not supported! Keeping scale at 1.rD   rF   r   r   r   Úunscale_layerw   s
   
üzHRALayer.unscale_layer©N©r   N)r+   Ú
__module__Ú__qualname__Úadapter_layer_namesÚother_param_namesr   ÚModuler    ÚstrÚintÚboolr1   r,   r-   ÚfloatrH   rI   r   r   r   r   r      s&    þýüû
ù,

r   c                       sº   e Zd ZdZ			ddedededeeef d	d
f
‡ fdd„Zddede	e
e  d	d
fdd„Zddd„Zd deded	ejfdd„Zdejdeded	ejfdd„Zd	ef‡ fdd„Z‡  ZS )!Ú	HRALinearz+
    HRA implemented in a dense layer.
    r   FTr!   r"   r#   r$   r   Nc                    s@   t ƒ  ¡  tj| |fi |¤Ž || _| j||||fi |¤Ž d S rJ   ©Úsuperr    r   Ú_active_adapterr1   ©r   r   r!   r"   r#   r$   r   ©Ú	__class__r   r   r    „   s   
	zHRALinear.__init__Ú
safe_mergeÚadapter_namesc           	      C   sÖ   t | |ƒ}|s	dS |D ]]}|| j ¡ v rh|  ¡ }|jj}|rJ|jj ¡ }|  |¡}t	 
| |j¡|¡}t	 |¡ ¡ sBtd|› dƒ‚| |¡|j_n|  |¡}t	 
|jj |j¡|¡}| |¡|j_| j |¡ qdS )áb  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If `None`, all active adapters will be merged.
                Defaults to `None`.
        Nú1NaNs detected in the merged weights. The adapter ú seems to be broken)r   r   rE   r   ÚweightÚdtypeÚdataÚcloneÚget_delta_weightr'   ÚmmÚtoÚisfiniteÚallr   r   Úappend)	r   r\   r]   rG   r   Ú
orig_dtypeÚorig_weightÚdelta_weightÚ
new_weightr   r   r   Úmerge’   s,   


ÿ
€ëzHRALinear.mergec                 C   sœ   | j s
t d¡ dS t| jƒdkrL| j ¡ }|  ¡ }|jj}|| j	 
¡ v rC|jj ¡ }| j|dd}t | |j¡|¡}| |¡|j_t| jƒdksdS dS ©zW
        This method unmerges all merged adapter layers from the base weights.
        z Already unmerged. Nothing to do.Nr   T)Úreverse)Úmergedr9   r:   Úlenr   Úpopr   ra   rb   r   rE   rc   rd   re   r'   rf   rg   )r   rG   r   rk   rl   rm   rn   r   r   r   Úunmerge»   s   

øzHRALinear.unmergerq   c                 C   ó€  | j | }| j| }| j| }|j}|r~|d d …df |d d …df  ¡   dd¡g}td|ƒD ]0}|d d …|f  dd¡}	t|ƒD ]}
|	||
  ¡ |	 ||
   }	qB| |	|	 ¡   dd¡¡ q0t	j
|dd}t	j|d |j|jdd| | ¡   }|S ||jdd }t	j|d |j|jd}|rt|d ddƒ}nt|ƒ}|D ]}|d d …|f  dd¡}	|d| |	 |	 ¡   }q£|S ©Nr   éÿÿÿÿr6   r7   )Údevicerb   r2   ©r	   r
   r   r?   ÚnormÚviewÚrangeÚtrj   r'   ÚcatÚeyery   rb   ©r   r!   rq   Úrankr#   Úopt_ur?   ra   ÚiÚuiÚjÚindicesr   r   r   re   Í   ó.   


.(õzHRALinear.get_delta_weightÚxÚargsr   c                 O   s   |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}no| jr*| j|g|¢R i |¤Ž}n_tj| j|jd}| j	D ]}|| j
 ¡ vr@q6|  |¡}t | |j ¡|¡}q6|  ¡ jj}	|  |	|j ¡}	t |	|¡}|  | jj|j ¡}
| jrw|  ||j ¡}n
| |  ¡ jjj ¡}tj|||
d}| |¡}|S )N©ry   )Úinputra   Úbias)rb   Údisable_adaptersrr   ru   r   r'   r€   r   ry   r0   r   rE   re   rf   rg   r   ra   rc   Ú_cast_input_dtyper   r   ÚFÚlinear©r   r‰   rŠ   r   Úprevious_dtypeÚresultrn   rG   rm   rl   r   r   r   r   Úforwardë   s.   


zHRALinear.forwardc                    ó   t ƒ  ¡ }d| S ©Nzhra.©rW   Ú__repr__©r   ÚreprZ   r   r   r™     ó   
zHRALinear.__repr__©r   FT©FNrK   ©F©r+   rL   rM   Ú__doc__rQ   rR   rS   r   r    r   Úlistro   ru   r'   ÚTensorre   r   r•   r™   Ú__classcell__r   r   rZ   r   rU      s*    úýüû
úø 
) rU   c                       s¶   e Zd ZdZ			ddedededeeef f‡ fd	d
„Zddede	e
e  ddfdd„Zddd„Zd dededejfdd„Zdejdededejfdd„Zdef‡ fdd„Z‡  ZS )!Ú	HRAConv2dzHRA implemented in Conv2d layerr   FTr!   r"   r#   r$   c                    s8   t ƒ  ¡  t | |¡ || _| j||||fi |¤Ž d S rJ   rV   rY   rZ   r   r   r      s   
	zHRAConv2d.__init__Nr\   r]   r   c                 C   sh  t | |ƒ}|s	dS |D ]¦}|| j ¡ v r±|  ¡ }|jj}|rm|jj ¡ }| | j	| j
|jd  | jjd  ¡}|  |¡}t | |j¡|¡}| | j	| j
|jd |jd ¡}t |¡ ¡ setd|› dƒ‚| |¡|j_n>|jj}| | j	| j
| jjd  | jjd  ¡}|  |¡}t | |j¡|¡}| | j	| j
|jd |jd ¡}| |¡|j_| j |¡ qdS )r^   Nr   r_   r`   )r   r   rE   r   ra   rb   rc   rd   r|   r   r   r)   r   re   r'   rf   rg   rh   ri   r   r   rj   )r   r\   r]   rG   r   rk   rl   rm   r   r   r   ro   !  sV   
þ
ü
ÿþ
ü€ÕzHRAConv2d.mergec                 C   sà   | j s
t d¡ dS t| jƒdkrn| j ¡ }|  ¡ }|jj}|| j	 
¡ v re|jj ¡ }| | j| j|jd  |jd  ¡}| j|dd}t | |j¡|¡}| | j| j|jd |jd ¡}| |¡|j_t| jƒdksdS dS rp   )rr   r9   r:   rs   r   rt   r   ra   rb   r   rE   rc   rd   r|   r   r   r)   re   r'   rf   rg   )r   rG   r   rk   rl   rm   r   r   r   ru   `  s(   

þÿðzHRAConv2d.unmergerq   c                 C   rv   rw   rz   r   r   r   r   re   y  rˆ   zHRAConv2d.get_delta_weightr‰   rŠ   r   c                 O   s–  |j }| jr| jr|  ¡  | j|g|¢R i |¤Ž}nª| jr*| j|g|¢R i |¤Ž}nštj| j| jjd  | jjd  |j	d}| j
D ]}|| j ¡ vrLqB|  |¡}t | |j ¡|¡}qB| jjj}	|	 | j| j| jjd  | jjd  ¡}	|  |	|j ¡}	|  | jj|j ¡}
t |	|¡}| | j| j| jjd | jjd ¡}| jr¨|  ||j ¡}n
| |  ¡ jjj ¡}tj|||
| jjd | jjd d}| |¡}|S )Nr   r‹   )rŒ   ra   r   ÚpaddingÚstride)rb   rŽ   rr   ru   r   r'   r€   r   r)   ry   r0   r   rE   re   rf   rg   ra   rc   r|   r   r   r   r   r   r   Úconv2dr¦   r§   r’   r   r   r   r•   —  sT   þ


þ

ü

û
zHRAConv2d.forwardc                    r–   r—   r˜   rš   rZ   r   r   r™   Ê  rœ   zHRAConv2d.__repr__r   rž   rK   rŸ   r    r   r   rZ   r   r¥     s&    úýüû
ú 
?3r¥   )r=   r9   Útypingr   r   r   r'   Útorch.nnr   Útorch.nn.functionalÚ
functionalr   Úpeft.tuners.tuners_utilsr   r   r   rP   rU   r¥   r   r   r   r   Ú<module>   s   e 